feat: initial goget release — modern IPv6-first download utility

internal/recursive/parser.go

@@ -0,0 +1,180 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// LinkExtractor extracts links from HTML
+type LinkExtractor struct {
+	baseURL *url.URL
+	links   []*url.URL
+}
+
+// NewLinkExtractor creates a new extractor
+func NewLinkExtractor(baseURL *url.URL) *LinkExtractor {
+	return &LinkExtractor{
+		baseURL: baseURL,
+		links:   make([]*url.URL, 0),
+	}
+}
+
+// ExtractLinks parses HTML and extracts all absolute URL links
+func ExtractLinks(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
+	doc, err := html.Parse(strings.NewReader(string(htmlContent)))
+	if err != nil {
+		return nil, err
+	}
+
+	extractor := NewLinkExtractor(baseURL)
+	extractor.traverse(doc)
+	return extractor.links, nil
+}
+
+// traverse traverses the HTML tree and extracts links
+func (e *LinkExtractor) traverse(n *html.Node) {
+	if n.Type == html.ElementNode {
+		switch n.Data {
+		case "a":
+			e.extractHref(n)
+		case "img":
+			e.extractSrc(n)
+		case "script":
+			e.extractSrc(n)
+		case "link":
+			e.extractHref(n)
+		case "iframe":
+			e.extractSrc(n)
+		case "video":
+			e.extractSrc(n)
+		case "audio":
+			e.extractSrc(n)
+		case "source":
+			e.extractSrc(n)
+		}
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		e.traverse(c)
+	}
+}
+
+// extractHref extracts the href attribute
+func (e *LinkExtractor) extractHref(n *html.Node) {
+	for _, attr := range n.Attr {
+		if attr.Key == "href" {
+			e.addLink(attr.Val)
+			break
+		}
+	}
+}
+
+// extractSrc extracts the src attribute
+func (e *LinkExtractor) extractSrc(n *html.Node) {
+	for _, attr := range n.Attr {
+		if attr.Key == "src" {
+			e.addLink(attr.Val)
+			break
+		}
+	}
+}
+
+// addLink adds a link to the list if it is valid
+func (e *LinkExtractor) addLink(href string) {
+	// Skip empty, anchors, javascript, mailto, tel, data
+	if href == "" ||
+		strings.HasPrefix(href, "#") ||
+		strings.HasPrefix(href, "javascript:") ||
+		strings.HasPrefix(href, "mailto:") ||
+		strings.HasPrefix(href, "tel:") ||
+		strings.HasPrefix(href, "data:") {
+		return
+	}
+
+	// Parse URL
+	parsed, err := url.Parse(href)
+	if err != nil {
+		return
+	}
+
+	// Resolve relative URLs against base
+	resolved := e.baseURL.ResolveReference(parsed)
+
+	// Only keep http/https
+	if resolved.Scheme != "http" && resolved.Scheme != "https" {
+		return
+	}
+
+	// Check if already added
+	for _, existing := range e.links {
+		if existing.String() == resolved.String() {
+			return
+		}
+	}
+
+	e.links = append(e.links, resolved)
+}
+
+// ExtractLinksFromHTML is an alias for ExtractLinks (for convenience)
+func ExtractLinksFromHTML(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
+	return ExtractLinks(baseURL, htmlContent)
+}
+
+var (
+	cssURLRegex    = regexp.MustCompile(`url\(\s*['"]?([^'")\s]+)['"]?\s*\)`)
+	cssImportRegex = regexp.MustCompile(`@import\s+(?:url\(\s*)?['"]?([^'");\s]+)['"]?\s*\)?`)
+)
+
+// ExtractCSSURLs extracts URLs from CSS content (url() and @import rules).
+// Returns absolute URLs resolved against the given base URL, deduplicated.
+func ExtractCSSURLs(baseURL *url.URL, cssContent []byte) []*url.URL {
+	seen := make(map[string]bool)
+	var urls []*url.URL
+
+	addURL := func(raw string) {
+		if seen[raw] {
+			return
+		}
+		seen[raw] = true
+
+		parsed, err := url.Parse(raw)
+		if err != nil {
+			return
+		}
+
+		resolved := baseURL.ResolveReference(parsed)
+		if resolved.Scheme != "http" && resolved.Scheme != "https" {
+			return
+		}
+
+		// Check duplicate resolved URL
+		resolvedStr := resolved.String()
+		if seen[resolvedStr] {
+			return
+		}
+		seen[resolvedStr] = true
+		urls = append(urls, resolved)
+	}
+
+	// Extract url() references
+	for _, match := range cssURLRegex.FindAllStringSubmatch(string(cssContent), -1) {
+		if len(match) > 1 {
+			addURL(match[1])
+		}
+	}
+
+	// Extract @import references
+	for _, match := range cssImportRegex.FindAllStringSubmatch(string(cssContent), -1) {
+		if len(match) > 1 {
+			addURL(match[1])
+		}
+	}
+
+	return urls
+}