feat: initial goget release — modern IPv6-first download utility

internal/recursive/crawler.go

@@ -0,0 +1,474 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"math/rand/v2"
+	"mime"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+	format "codeberg.org/petrbalvin/goget/internal/format"
+	"codeberg.org/petrbalvin/goget/internal/linkrewrite"
+	"golang.org/x/net/html"
+)
+
+// CrawlerConfig configures the crawler
+type CrawlerConfig struct {
+	MaxDepth        int
+	FollowExternal  bool
+	ExcludePatterns []string
+	IncludePatterns []string
+	AcceptPatterns  []string
+	NoParent        bool
+	PageRequisites  bool
+	ConvertLinks    bool
+	OutputDir       string
+	Verbose         bool
+	Parallel        int
+	Delay           time.Duration
+	RandomWait      bool
+	UserAgent       string
+	DryRun          bool
+}
+
+// CrawlerStats represents crawler statistics
+type CrawlerStats struct {
+	TotalURLs      int
+	DownloadedURLs int
+	FailedURLs     int
+	SkippedURLs    int
+	TotalBytes     int64
+	StartTime      time.Time
+	EndTime        time.Time
+	mu             sync.Mutex
+}
+
+// Crawler performs recursive downloading
+type Crawler struct {
+	config    *CrawlerConfig
+	proto     core.Protocol
+	filter    *URLFilter
+	visited   map[string]bool
+	visitedMu sync.Mutex
+	queue     chan *crawlTask
+	queueOnce sync.Once
+	stats     *CrawlerStats
+	rewriter  *linkrewrite.Rewriter
+	wg        sync.WaitGroup
+	ctx       context.Context
+	cancel    context.CancelFunc
+}
+
+// crawlTask represents a task for the crawler
+type crawlTask struct {
+	url   *url.URL
+	depth int
+}
+
+// NewCrawler creates new crawler
+func NewCrawler(cfg *CrawlerConfig, proto core.Protocol) *Crawler {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	filter := NewURLFilter(nil, URLFilterConfig{
+		MaxDepth:        cfg.MaxDepth,
+		FollowExternal:  cfg.FollowExternal,
+		ExcludePatterns: cfg.ExcludePatterns,
+		IncludePatterns: cfg.IncludePatterns,
+		AcceptPatterns:  cfg.AcceptPatterns,
+		NoParent:        cfg.NoParent,
+	})
+
+	crawler := &Crawler{
+		config:  cfg,
+		proto:   proto,
+		filter:  filter,
+		visited: make(map[string]bool),
+		queue:   make(chan *crawlTask, 100),
+		stats:   &CrawlerStats{StartTime: time.Now()},
+		ctx:     ctx,
+		cancel:  cancel,
+	}
+	if cfg.ConvertLinks {
+		crawler.rewriter = linkrewrite.New(nil, cfg.OutputDir)
+	}
+
+	return crawler
+}
+
+// Download recursively downloads the URL and all links
+func (c *Crawler) Download(ctx context.Context, baseURL *url.URL) (*CrawlerStats, error) {
+	c.filter.baseURL = baseURL
+	if c.rewriter != nil {
+		c.rewriter = linkrewrite.New(baseURL, c.config.OutputDir)
+	}
+
+	// Create output directory
+	if err := os.MkdirAll(c.config.OutputDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create output directory: %w", err)
+	}
+
+	// Add to WaitGroup BEFORE adding to queue
+	c.wg.Add(1)
+	c.queue <- &crawlTask{url: baseURL, depth: 0}
+
+	// Start worker goroutines
+	workers := c.config.Parallel
+	if workers <= 0 {
+		workers = 4
+	}
+
+	for i := 0; i < workers; i++ {
+		go c.worker()
+	}
+
+	// Close queue on context cancellation to unblock workers
+	closeQueue := func() {
+		c.queueOnce.Do(func() {
+			close(c.queue)
+		})
+	}
+
+	go func() {
+		<-c.ctx.Done()
+		closeQueue()
+	}()
+
+	// Wait for all tasks to complete
+	c.wg.Wait()
+	closeQueue()
+
+	c.stats.EndTime = time.Now()
+	return c.stats, nil
+}
+
+// worker processes tasks from the queue
+func (c *Crawler) worker() {
+	for {
+		select {
+		case task, ok := <-c.queue:
+			if !ok {
+				return
+			}
+			if c.ctx.Err() != nil {
+				c.wg.Done()
+				continue
+			}
+			c.processTask(task)
+
+			// Delay between requests (with optional randomization)
+			if c.config.Delay > 0 {
+				d := c.config.Delay
+				if c.config.RandomWait {
+					d = randomizeDelay(d)
+				}
+				select {
+				case <-time.After(d):
+				case <-c.ctx.Done():
+					return
+				}
+			}
+		case <-c.ctx.Done():
+			return
+		}
+	}
+}
+
+// processTask processes a single crawl task
+func (c *Crawler) processTask(task *crawlTask) {
+	defer c.wg.Done()
+
+	// Check if cancelled
+	if c.ctx.Err() != nil {
+		return
+	}
+
+	// Check if already visited
+	c.visitedMu.Lock()
+	if c.visited[task.url.String()] {
+		c.visitedMu.Unlock()
+		c.stats.mu.Lock()
+		c.stats.SkippedURLs++
+		c.stats.mu.Unlock()
+		return
+	}
+	c.visited[task.url.String()] = true
+	c.visitedMu.Unlock()
+
+	// Check filter
+	if !c.filter.ShouldDownload(task.url, task.depth) {
+		c.stats.mu.Lock()
+		c.stats.SkippedURLs++
+		c.stats.mu.Unlock()
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[skip] %s (filtered)\n", task.url.String())
+		}
+		return
+	}
+
+	outputPath := GetLocalPath(c.filter.baseURL, task.url, c.config.OutputDir)
+
+	if c.config.DryRun {
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[dry-run] %s (depth %d)\n", task.url.String(), task.depth)
+		}
+		c.stats.mu.Lock()
+		c.stats.SkippedURLs++
+		c.stats.mu.Unlock()
+		return
+	}
+
+	// Download the URL
+	if c.config.Verbose {
+		fmt.Fprintf(os.Stderr, "[download] %s (depth %d)\n", task.url.String(), task.depth)
+	}
+
+	// Create parent directory
+	if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
+		c.stats.mu.Lock()
+		c.stats.FailedURLs++
+		c.stats.mu.Unlock()
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[error] failed to create directory: %v\n", err)
+		}
+		return
+	}
+
+	// Create download request
+	req := &core.DownloadRequest{
+		URL:            task.url,
+		Output:         outputPath,
+		Resume:         true,
+		Timeout:        30 * time.Minute,
+		Verbose:        false,
+		AutoDecompress: true,
+		Headers: map[string]string{
+			"User-Agent": c.config.UserAgent,
+		},
+		Ctx: c.ctx,
+	}
+
+	// Execute download
+	result, err := c.proto.Download(c.ctx, req)
+	if err != nil {
+		c.stats.mu.Lock()
+		c.stats.FailedURLs++
+		c.stats.mu.Unlock()
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[error] %s: %v\n", task.url.String(), err)
+		}
+		return
+	}
+
+	c.stats.mu.Lock()
+	c.stats.DownloadedURLs++
+	c.stats.TotalBytes += result.BytesDownloaded
+	c.stats.mu.Unlock()
+
+	if c.config.Verbose {
+		fmt.Fprintf(os.Stderr, "[done] %s -> %s (%s)\n", task.url.String(), outputPath, format.Bytes(result.BytesDownloaded))
+	}
+
+	if c.config.ConvertLinks && c.rewriter != nil {
+		c.rewriter.Register(task.url, outputPath)
+		c.rewriteLinks(outputPath, task.url)
+	}
+
+	// Check if content is HTML by reading file content
+	if c.shouldParseForLinks(outputPath, task.url.Path) {
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[parse] %s looks like HTML, extracting links\n", outputPath)
+		}
+		c.extractAndQueue(task.url, outputPath, task.depth+1)
+	}
+
+	// Delay between requests
+	if c.config.Delay > 0 {
+		select {
+		case <-time.After(c.config.Delay):
+		case <-c.ctx.Done():
+		}
+	}
+}
+
+// Better HTML detection - check content, not just extension
+func (c *Crawler) shouldParseForLinks(outputPath, urlPath string) bool {
+	// First check extension (quick path)
+	ext := strings.ToLower(filepath.Ext(outputPath))
+	if ext == ".html" || ext == ".htm" || ext == ".php" || ext == ".asp" || ext == ".aspx" {
+		return true
+	}
+	if ext == "" && (urlPath == "" || urlPath == "/" || strings.HasSuffix(urlPath, "/")) {
+		return true
+	}
+
+	// If extension doesn't match, read only the first 512 bytes to check for HTML signature.
+	f, err := os.Open(outputPath)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	snippet := make([]byte, 512)
+	n, err := io.ReadFull(f, snippet)
+	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+		return false
+	}
+	snippet = snippet[:n]
+	if len(snippet) == 0 {
+		return false
+	}
+	content := strings.ToLower(string(snippet))
+
+	// Look for common HTML indicators
+	if strings.Contains(content, "<!doctype html") ||
+		strings.Contains(content, "<html") ||
+		strings.Contains(content, "<head") ||
+		strings.Contains(content, "<body") {
+		return true
+	}
+
+	// Check Content-Type via mime sniffing (basic)
+	if contentType := httpDetectContentType(snippet); contentType != "" {
+		mediaType, _, err := mime.ParseMediaType(contentType)
+		if err == nil && mediaType == "text/html" {
+			return true
+		}
+	}
+
+	return false
+}
+
+// httpDetectContentType basic MIME type detection from content
+func httpDetectContentType(data []byte) string {
+	if len(data) < 2 {
+		return ""
+	}
+	if bytes.HasPrefix(data, []byte("<!")) || bytes.HasPrefix(data, []byte("<html")) || bytes.HasPrefix(data, []byte("<HTML")) {
+		return "text/html"
+	}
+	return ""
+}
+
+// extractAndQueue extracts links and adds them to the queue
+func (c *Crawler) extractAndQueue(baseURL *url.URL, outputPath string, depth int) {
+	// Read HTML content
+	data, err := os.ReadFile(outputPath)
+	if err != nil {
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[error] failed to read %s for link extraction: %v\n", outputPath, err)
+		}
+		return
+	}
+
+	// Extract links
+	links, err := ExtractLinks(baseURL, data)
+	if err != nil {
+		if c.config.Verbose {
+			fmt.Fprintf(os.Stderr, "[error] failed to parse links from %s: %v\n", outputPath, err)
+		}
+		return
+	}
+
+	if c.config.Verbose {
+		fmt.Fprintf(os.Stderr, "[links] found %d links in %s\n", len(links), outputPath)
+	}
+
+	// Add valid links to queue
+	added := 0
+	for _, link := range links {
+		if c.filter.ShouldDownload(link, depth) {
+			c.visitedMu.Lock()
+			if !c.visited[link.String()] {
+				c.visitedMu.Unlock()
+				// Add to WaitGroup before adding to queue
+				c.wg.Add(1)
+				select {
+				case c.queue <- &crawlTask{url: link, depth: depth}:
+					added++
+				case <-c.ctx.Done():
+					c.wg.Done()
+					return
+				}
+			} else {
+				c.visitedMu.Unlock()
+			}
+		}
+	}
+	if c.config.Verbose && added > 0 {
+		fmt.Fprintf(os.Stderr, "[queue] added %d new URLs to queue\n", added)
+	}
+}
+
+// GetStats returns current statistics
+func (c *Crawler) GetStats() *CrawlerStats {
+	return c.stats
+}
+
+// Cancel cancels the crawler
+func (c *Crawler) Cancel() {
+	c.cancel()
+}
+
+// GetStatsSummary returns a summary of statistics
+func (s *CrawlerStats) GetStatsSummary() string {
+	duration := s.EndTime.Sub(s.StartTime)
+	if duration == 0 {
+		duration = time.Since(s.StartTime)
+	}
+
+	speed := float64(0)
+	if duration.Seconds() > 0 {
+		speed = float64(s.TotalBytes) / duration.Seconds()
+	}
+
+	return fmt.Sprintf("Downloaded: %d files, %s total, %v elapsed, %.1f KB/s avg",
+		s.DownloadedURLs,
+		format.Bytes(s.TotalBytes),
+		duration.Round(time.Second),
+		speed/1024)
+}
+
+// rewriteLinks rewrites absolute URLs to local paths in HTML files.
+func (c *Crawler) rewriteLinks(outputPath string, sourceURL *url.URL) {
+	if c.rewriter == nil {
+		return
+	}
+
+	data, err := os.ReadFile(outputPath)
+	if err != nil {
+		return
+	}
+
+	doc, err := html.Parse(bytes.NewReader(data))
+	if err != nil {
+		return
+	}
+
+	if c.rewriter.RewriteLinks(doc, outputPath) == 0 {
+		return
+	}
+
+	var buf bytes.Buffer
+	if err := html.Render(&buf, doc); err != nil {
+		return
+	}
+	_ = os.WriteFile(outputPath, buf.Bytes(), 0644)
+	_ = sourceURL
+}
+
+// randomizeDelay returns a randomized delay between 0.5x and 1.5x.
+func randomizeDelay(d time.Duration) time.Duration {
+	factor := 0.5 + rand.Float64()
+	return time.Duration(float64(d) * factor)
+}

internal/recursive/crawler_test.go

@@ -0,0 +1,157 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"context"
+	"net/url"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+	"codeberg.org/petrbalvin/goget/internal/protocol"
+)
+
+// mockProtocol is a controllable core.Protocol implementation for crawler
+// tests. The Download method honours ctx.Done() so a cancelled test can
+// observe the cancellation propagating into the worker.
+type mockProtocol struct {
+	*protocol.BaseProtocol
+	downloadDelay time.Duration
+	downloadCount int32
+}
+
+func (m *mockProtocol) Download(ctx context.Context, _ *core.DownloadRequest) (*core.DownloadResult, error) {
+	atomic.AddInt32(&m.downloadCount, 1)
+	if m.downloadDelay > 0 {
+		select {
+		case <-time.After(m.downloadDelay):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	return &core.DownloadResult{BytesDownloaded: 0}, nil
+}
+
+func newMockProtocol(delay time.Duration) *mockProtocol {
+	return &mockProtocol{
+		BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
+			Name:   "mock",
+			Scheme: "http",
+		}),
+		downloadDelay: delay,
+	}
+}
+
+// TestDownloadCancelDoesNotPanic is a regression guard for the BACKLOG
+// entry "Double close() crash in recursive crawler". Before the fix, the
+// cancellation goroutine and the main Download path both called
+// close(c.queue); whichever fired second panicked with
+// "close of closed channel". With sync.Once guarding the close, only the
+// first caller wins and the second is a no-op.
+func TestDownloadCancelDoesNotPanic(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	// Each download takes 50ms, long enough for the cancellation goroutine
+	// to fire closeQueue before wg.Wait() returns on its own.
+	proto := newMockProtocol(50 * time.Millisecond)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	done := make(chan struct{})
+	var panicVal any
+	go func() {
+		defer close(done)
+		defer func() {
+			panicVal = recover()
+		}()
+		_, _ = crawler.Download(context.Background(), baseURL)
+	}()
+
+	// Give the worker a moment to pick up the initial task and start the
+	// slow download, then cancel — this is the exact race the original
+	// bug triggered.
+	time.Sleep(20 * time.Millisecond)
+	crawler.Cancel()
+
+	select {
+	case <-done:
+	case <-time.After(3 * time.Second):
+		t.Fatal("Download did not return within 3s after Cancel")
+	}
+
+	if panicVal != nil {
+		t.Fatalf("Download panicked on Cancel: %v", panicVal)
+	}
+}
+
+// TestDownloadNormalCompletion verifies the happy path still works after
+// introducing sync.Once. Without the bug present, the only close() call
+// happens after wg.Wait() returns, and the workers must observe a closed
+// channel and exit cleanly.
+func TestDownloadNormalCompletion(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	proto := newMockProtocol(0)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	stats, err := crawler.Download(context.Background(), baseURL)
+	if err != nil {
+		t.Fatalf("Download: %v", err)
+	}
+	if stats.DownloadedURLs != 1 {
+		t.Errorf("DownloadedURLs = %d, want 1", stats.DownloadedURLs)
+	}
+	if got := atomic.LoadInt32(&proto.downloadCount); got != 1 {
+		t.Errorf("downloadCount = %d, want 1", got)
+	}
+}
+
+// TestDownloadCancelAfterAllTasksDone exercises the ordering where the
+// main path's wg.Wait() returns first and then the cancellation goroutine
+// fires. sync.Once must ignore the second close.
+func TestDownloadCancelAfterAllTasksDone(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	proto := newMockProtocol(0)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		_, _ = crawler.Download(context.Background(), baseURL)
+	}()
+
+	<-done
+
+	// Cancelling after Download returned must not panic (no goroutine is
+	// waiting on closeQueue, but Cancel triggers ctx.Done() which the
+	// goroutine observes; sync.Once already ran so the second close is a
+	// no-op).
+	crawler.Cancel()
+}

internal/recursive/filter.go

@@ -0,0 +1,211 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"mime"
+	"net/url"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// URLFilter filters URLs according to rules
+type URLFilter struct {
+	baseURL         *url.URL
+	maxDepth        int
+	followExternal  bool
+	excludePatterns []*regexp.Regexp
+	includePatterns []*regexp.Regexp
+	acceptPatterns  []string // Glob-style accept patterns
+	noParent        bool
+}
+
+// URLFilterConfig configures the URL filter
+type URLFilterConfig struct {
+	MaxDepth        int
+	FollowExternal  bool
+	ExcludePatterns []string
+	IncludePatterns []string
+	AcceptPatterns  []string // Glob-style accept patterns (e.g., "*.pdf,*.html")
+	NoParent        bool     // Do not ascend to parent directory
+}
+
+// NewURLFilter creates a new filter
+func NewURLFilter(baseURL *url.URL, cfg URLFilterConfig) *URLFilter {
+	filter := &URLFilter{
+		baseURL:         baseURL,
+		maxDepth:        cfg.MaxDepth,
+		followExternal:  cfg.FollowExternal,
+		excludePatterns: make([]*regexp.Regexp, 0),
+		includePatterns: make([]*regexp.Regexp, 0),
+		acceptPatterns:  cfg.AcceptPatterns,
+		noParent:        cfg.NoParent,
+	}
+
+	// Compile exclude patterns
+	for _, pattern := range cfg.ExcludePatterns {
+		if re, err := regexp.Compile(pattern); err == nil {
+			filter.excludePatterns = append(filter.excludePatterns, re)
+		}
+	}
+
+	// Compile include patterns
+	for _, pattern := range cfg.IncludePatterns {
+		if re, err := regexp.Compile(pattern); err == nil {
+			filter.includePatterns = append(filter.includePatterns, re)
+		}
+	}
+
+	return filter
+}
+
+// ShouldDownload determines whether a URL should be downloaded
+func (f *URLFilter) ShouldDownload(u *url.URL, currentDepth int) bool {
+	// Check depth
+	if currentDepth > f.maxDepth {
+		return false
+	}
+
+	// Check scheme
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return false
+	}
+
+	// Check domain (external links)
+	if !f.followExternal && u.Hostname() != f.baseURL.Hostname() {
+		return false
+	}
+
+	// Check exclude patterns
+	urlStr := u.String()
+	for _, re := range f.excludePatterns {
+		if re.MatchString(urlStr) {
+			return false
+		}
+	}
+
+	// Check include patterns (if specified, must match at least one)
+	if len(f.includePatterns) > 0 {
+		matched := false
+		for _, re := range f.includePatterns {
+			if re.MatchString(urlStr) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	// Check accept patterns (glob-style and MIME type)
+	if len(f.acceptPatterns) > 0 {
+		lastPath := u.Path
+		if idx := strings.LastIndex(lastPath, "/"); idx >= 0 {
+			lastPath = lastPath[idx+1:]
+		}
+		matched := false
+		for _, pattern := range f.acceptPatterns {
+			// MIME type pattern (contains "/")
+			if strings.Contains(pattern, "/") {
+				ext := filepath.Ext(lastPath)
+				if ext != "" {
+					mimeType := mime.TypeByExtension(ext)
+					if match, _ := filepath.Match(pattern, mimeType); match {
+						matched = true
+						break
+					}
+				}
+				continue
+			}
+			// Filename glob
+			if match, _ := filepath.Match(pattern, lastPath); match {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	// Check no-parent
+	if f.noParent && f.baseURL != nil {
+		basePath := strings.TrimRight(f.baseURL.Path, "/")
+		targetPath := strings.TrimRight(u.Path, "/")
+		if !strings.HasPrefix(targetPath, basePath) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// IsSameDomain determines whether the URL belongs to the same domain
+func IsSameDomain(base, target *url.URL) bool {
+	if base == nil || target == nil {
+		return false
+	}
+	return base.Hostname() == target.Hostname()
+}
+
+// IsSubPath determines whether target is a subpath of base
+func IsSubPath(base, target *url.URL) bool {
+	if base == nil || target == nil {
+		return false
+	}
+	return strings.HasPrefix(target.Path, base.Path)
+}
+
+// GetLocalPath converts a URL to a local file path
+func GetLocalPath(baseURL, targetURL *url.URL, outputDir string) string {
+	// Start with output directory
+	path := outputDir
+
+	// Add hostname as subdirectory to avoid conflicts
+	if targetURL.Hostname() != baseURL.Hostname() {
+		path = filepath.Join(path, targetURL.Hostname())
+	}
+
+	// Add URL path
+	if targetURL.Path != "" && targetURL.Path != "/" {
+		// Clean the path
+		cleanPath := strings.TrimPrefix(targetURL.Path, "/")
+		path = filepath.Join(path, cleanPath)
+	}
+
+	// If path ends with / or is empty, add index.html
+	if strings.HasSuffix(targetURL.Path, "/") || targetURL.Path == "" {
+		path = filepath.Join(path, "index.html")
+	}
+
+	// If path has no extension, treat as HTML
+	if filepath.Ext(path) == "" {
+		path = path + ".html"
+	}
+
+	return path
+}
+
+// GetFilename extracts the filename from a URL
+func GetFilename(u *url.URL) string {
+	path := u.Path
+	if path == "" || path == "/" {
+		return "index.html"
+	}
+
+	// Get last path segment
+	segments := strings.Split(strings.Trim(path, "/"), "/")
+	if len(segments) == 0 {
+		return "index.html"
+	}
+
+	filename := segments[len(segments)-1]
+	if filename == "" {
+		return "index.html"
+	}
+
+	return filename
+}

internal/recursive/parser.go

@@ -0,0 +1,180 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// LinkExtractor extracts links from HTML
+type LinkExtractor struct {
+	baseURL *url.URL
+	links   []*url.URL
+}
+
+// NewLinkExtractor creates a new extractor
+func NewLinkExtractor(baseURL *url.URL) *LinkExtractor {
+	return &LinkExtractor{
+		baseURL: baseURL,
+		links:   make([]*url.URL, 0),
+	}
+}
+
+// ExtractLinks parses HTML and extracts all absolute URL links
+func ExtractLinks(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
+	doc, err := html.Parse(strings.NewReader(string(htmlContent)))
+	if err != nil {
+		return nil, err
+	}
+
+	extractor := NewLinkExtractor(baseURL)
+	extractor.traverse(doc)
+	return extractor.links, nil
+}
+
+// traverse traverses the HTML tree and extracts links
+func (e *LinkExtractor) traverse(n *html.Node) {
+	if n.Type == html.ElementNode {
+		switch n.Data {
+		case "a":
+			e.extractHref(n)
+		case "img":
+			e.extractSrc(n)
+		case "script":
+			e.extractSrc(n)
+		case "link":
+			e.extractHref(n)
+		case "iframe":
+			e.extractSrc(n)
+		case "video":
+			e.extractSrc(n)
+		case "audio":
+			e.extractSrc(n)
+		case "source":
+			e.extractSrc(n)
+		}
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		e.traverse(c)
+	}
+}
+
+// extractHref extracts the href attribute
+func (e *LinkExtractor) extractHref(n *html.Node) {
+	for _, attr := range n.Attr {
+		if attr.Key == "href" {
+			e.addLink(attr.Val)
+			break
+		}
+	}
+}
+
+// extractSrc extracts the src attribute
+func (e *LinkExtractor) extractSrc(n *html.Node) {
+	for _, attr := range n.Attr {
+		if attr.Key == "src" {
+			e.addLink(attr.Val)
+			break
+		}
+	}
+}
+
+// addLink adds a link to the list if it is valid
+func (e *LinkExtractor) addLink(href string) {
+	// Skip empty, anchors, javascript, mailto, tel, data
+	if href == "" ||
+		strings.HasPrefix(href, "#") ||
+		strings.HasPrefix(href, "javascript:") ||
+		strings.HasPrefix(href, "mailto:") ||
+		strings.HasPrefix(href, "tel:") ||
+		strings.HasPrefix(href, "data:") {
+		return
+	}
+
+	// Parse URL
+	parsed, err := url.Parse(href)
+	if err != nil {
+		return
+	}
+
+	// Resolve relative URLs against base
+	resolved := e.baseURL.ResolveReference(parsed)
+
+	// Only keep http/https
+	if resolved.Scheme != "http" && resolved.Scheme != "https" {
+		return
+	}
+
+	// Check if already added
+	for _, existing := range e.links {
+		if existing.String() == resolved.String() {
+			return
+		}
+	}
+
+	e.links = append(e.links, resolved)
+}
+
+// ExtractLinksFromHTML is an alias for ExtractLinks (for convenience)
+func ExtractLinksFromHTML(baseURL *url.URL, htmlContent []byte) ([]*url.URL, error) {
+	return ExtractLinks(baseURL, htmlContent)
+}
+
+var (
+	cssURLRegex    = regexp.MustCompile(`url\(\s*['"]?([^'")\s]+)['"]?\s*\)`)
+	cssImportRegex = regexp.MustCompile(`@import\s+(?:url\(\s*)?['"]?([^'");\s]+)['"]?\s*\)?`)
+)
+
+// ExtractCSSURLs extracts URLs from CSS content (url() and @import rules).
+// Returns absolute URLs resolved against the given base URL, deduplicated.
+func ExtractCSSURLs(baseURL *url.URL, cssContent []byte) []*url.URL {
+	seen := make(map[string]bool)
+	var urls []*url.URL
+
+	addURL := func(raw string) {
+		if seen[raw] {
+			return
+		}
+		seen[raw] = true
+
+		parsed, err := url.Parse(raw)
+		if err != nil {
+			return
+		}
+
+		resolved := baseURL.ResolveReference(parsed)
+		if resolved.Scheme != "http" && resolved.Scheme != "https" {
+			return
+		}
+
+		// Check duplicate resolved URL
+		resolvedStr := resolved.String()
+		if seen[resolvedStr] {
+			return
+		}
+		seen[resolvedStr] = true
+		urls = append(urls, resolved)
+	}
+
+	// Extract url() references
+	for _, match := range cssURLRegex.FindAllStringSubmatch(string(cssContent), -1) {
+		if len(match) > 1 {
+			addURL(match[1])
+		}
+	}
+
+	// Extract @import references
+	for _, match := range cssImportRegex.FindAllStringSubmatch(string(cssContent), -1) {
+		if len(match) > 1 {
+			addURL(match[1])
+		}
+	}
+
+	return urls
+}