//go:build linux || freebsd // +build linux freebsd package recursive import ( "net/url" "regexp" "strings" "golang.org/x/net/html" ) // LinkExtractor extracts links from HTML type LinkExtractor struct { baseURL *url.URL links []*url.URL } // NewLinkExtractor creates a new extractor func NewLinkExtractor(baseURL *url.URL) *LinkExtractor { return &LinkExtractor{ baseURL: baseURL, links: make([]*url.URL, 0), } } // ExtractLinks parses HTML and extracts all absolute URL links func ExtractLinks(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) { doc, err := html.Parse(strings.NewReader(string(htmlContent))) if err != nil { return nil, err } extractor := NewLinkExtractor(baseURL) extractor.traverse(doc) return extractor.links, nil } // traverse traverses the HTML tree and extracts links func (e *LinkExtractor) traverse(n *html.Node) { if n.Type == html.ElementNode { switch n.Data { case "a": e.extractHref(n) case "img": e.extractSrc(n) case "script": e.extractSrc(n) case "link": e.extractHref(n) case "iframe": e.extractSrc(n) case "video": e.extractSrc(n) case "audio": e.extractSrc(n) case "source": e.extractSrc(n) } } for c := n.FirstChild; c != nil; c = c.NextSibling { e.traverse(c) } } // extractHref extracts the href attribute func (e *LinkExtractor) extractHref(n *html.Node) { for _, attr := range n.Attr { if attr.Key == "href" { e.addLink(attr.Val) break } } } // extractSrc extracts the src attribute func (e *LinkExtractor) extractSrc(n *html.Node) { for _, attr := range n.Attr { if attr.Key == "src" { e.addLink(attr.Val) break } } } // addLink adds a link to the list if it is valid func (e *LinkExtractor) addLink(href string) { // Skip empty, anchors, javascript, mailto, tel, data if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") || strings.HasPrefix(href, "mailto:") || strings.HasPrefix(href, "tel:") || strings.HasPrefix(href, "data:") { return } // Parse URL parsed, err := url.Parse(href) if err != nil { return } // Resolve relative URLs against base resolved := e.baseURL.ResolveReference(parsed) // Only keep http/https if resolved.Scheme != "http" && resolved.Scheme != "https" { return } // Check if already added for _, existing := range e.links { if existing.String() == resolved.String() { return } } e.links = append(e.links, resolved) } // ExtractLinksFromHTML is an alias for ExtractLinks (for convenience) func ExtractLinksFromHTML(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) { return ExtractLinks(baseURL, htmlContent) } var ( cssURLRegex = regexp.MustCompile(`url\(\s*['"]?([^'")\s]+)['"]?\s*\)`) cssImportRegex = regexp.MustCompile(`@import\s+(?:url\(\s*)?['"]?([^'");\s]+)['"]?\s*\)?`) ) // ExtractCSSURLs extracts URLs from CSS content (url() and @import rules). // Returns absolute URLs resolved against the given base URL, deduplicated. func ExtractCSSURLs(baseURL *url.URL, cssContent []byte) []*url.URL { seen := make(map[string]bool) var urls []*url.URL addURL := func(raw string) { if seen[raw] { return } seen[raw] = true parsed, err := url.Parse(raw) if err != nil { return } resolved := baseURL.ResolveReference(parsed) if resolved.Scheme != "http" && resolved.Scheme != "https" { return } // Check duplicate resolved URL resolvedStr := resolved.String() if seen[resolvedStr] { return } seen[resolvedStr] = true urls = append(urls, resolved) } // Extract url() references for _, match := range cssURLRegex.FindAllStringSubmatch(string(cssContent), -1) { if len(match) > 1 { addURL(match[1]) } } // Extract @import references for _, match := range cssImportRegex.FindAllStringSubmatch(string(cssContent), -1) { if len(match) > 1 { addURL(match[1]) } } return urls }