goget/internal/recursive/parser.go

//go:build linux || freebsd
// +build linux freebsd

package recursive

import (
	"net/url"
	"regexp"
	"strings"

	"golang.org/x/net/html"
)

// LinkExtractor extracts links from HTML
type LinkExtractor struct {
	baseURL *url.URL
	links   []*url.URL
}

// NewLinkExtractor creates a new extractor
func NewLinkExtractor(baseURL *url.URL) *LinkExtractor {
	return &LinkExtractor{
		baseURL: baseURL,
		links:   make([]*url.URL, 0),
	}
}

// ExtractLinks parses HTML and extracts all absolute URL links
func ExtractLinks(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
	doc, err := html.Parse(strings.NewReader(string(htmlContent)))
	if err != nil {
		return nil, err
	}

	extractor := NewLinkExtractor(baseURL)
	extractor.traverse(doc)
	return extractor.links, nil
}

// traverse traverses the HTML tree and extracts links
func (e *LinkExtractor) traverse(n *html.Node) {
	if n.Type == html.ElementNode {
		switch n.Data {
		case "a":
			e.extractHref(n)
		case "img":
			e.extractSrc(n)
		case "script":
			e.extractSrc(n)
		case "link":
			e.extractHref(n)
		case "iframe":
			e.extractSrc(n)
		case "video":
			e.extractSrc(n)
		case "audio":
			e.extractSrc(n)
		case "source":
			e.extractSrc(n)
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		e.traverse(c)
	}
}

// extractHref extracts the href attribute
func (e *LinkExtractor) extractHref(n *html.Node) {
	for _, attr := range n.Attr {
		if attr.Key == "href" {
			e.addLink(attr.Val)
			break
		}
	}
}

// extractSrc extracts the src attribute
func (e *LinkExtractor) extractSrc(n *html.Node) {
	for _, attr := range n.Attr {
		if attr.Key == "src" {
			e.addLink(attr.Val)
			break
		}
	}
}

// addLink adds a link to the list if it is valid
func (e *LinkExtractor) addLink(href string) {
	// Skip empty, anchors, javascript, mailto, tel, data
	if href == "" ||
		strings.HasPrefix(href, "#") ||
		strings.HasPrefix(href, "javascript:") ||
		strings.HasPrefix(href, "mailto:") ||
		strings.HasPrefix(href, "tel:") ||
		strings.HasPrefix(href, "data:") {
		return
	}

	// Parse URL
	parsed, err := url.Parse(href)
	if err != nil {
		return
	}

	// Resolve relative URLs against base
	resolved := e.baseURL.ResolveReference(parsed)

	// Only keep http/https
	if resolved.Scheme != "http" && resolved.Scheme != "https" {
		return
	}

	// Check if already added
	for _, existing := range e.links {
		if existing.String() == resolved.String() {
			return
		}
	}

	e.links = append(e.links, resolved)
}

// ExtractLinksFromHTML is an alias for ExtractLinks (for convenience)
func ExtractLinksFromHTML(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
	return ExtractLinks(baseURL, htmlContent)
}

var (
	cssURLRegex    = regexp.MustCompile(`url\(\s*['"]?([^'")\s]+)['"]?\s*\)`)
	cssImportRegex = regexp.MustCompile(`@import\s+(?:url\(\s*)?['"]?([^'");\s]+)['"]?\s*\)?`)
)

// ExtractCSSURLs extracts URLs from CSS content (url() and @import rules).
// Returns absolute URLs resolved against the given base URL, deduplicated.
func ExtractCSSURLs(baseURL *url.URL, cssContent []byte) []*url.URL {
	seen := make(map[string]bool)
	var urls []*url.URL

	addURL := func(raw string) {
		if seen[raw] {
			return
		}
		seen[raw] = true

		parsed, err := url.Parse(raw)
		if err != nil {
			return
		}

		resolved := baseURL.ResolveReference(parsed)
		if resolved.Scheme != "http" && resolved.Scheme != "https" {
			return
		}

		// Check duplicate resolved URL
		resolvedStr := resolved.String()
		if seen[resolvedStr] {
			return
		}
		seen[resolvedStr] = true
		urls = append(urls, resolved)
	}

	// Extract url() references
	for _, match := range cssURLRegex.FindAllStringSubmatch(string(cssContent), -1) {
		if len(match) > 1 {
			addURL(match[1])
		}
	}

	// Extract @import references
	for _, match := range cssImportRegex.FindAllStringSubmatch(string(cssContent), -1) {
		if len(match) > 1 {
			addURL(match[1])
		}
	}

	return urls
}