goget/internal/recursive/recursive_test.go

//go:build linux || freebsd
// +build linux freebsd

package recursive

import (
	"net/url"
	"strings"
	"testing"
	"time"
)

// ============================================================
// Filter tests
// ============================================================

func TestNewURLFilter(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/docs/")
	cfg := URLFilterConfig{
		MaxDepth:        3,
		FollowExternal:  false,
		ExcludePatterns: []string{`\.pdf$`, `\.zip$`},
		IncludePatterns: []string{`\.html$`},
	}
	filter := NewURLFilter(baseURL, cfg)

	if filter == nil {
		t.Fatal("NewURLFilter returned nil")
	}
	if filter.baseURL.String() != "https://example.com/docs/" {
		t.Errorf("expected baseURL https://example.com/docs/, got %s", filter.baseURL.String())
	}
	if filter.maxDepth != 3 {
		t.Errorf("expected maxDepth 3, got %d", filter.maxDepth)
	}
	if filter.followExternal != false {
		t.Errorf("expected followExternal false, got %v", filter.followExternal)
	}
	if len(filter.excludePatterns) != 2 {
		t.Errorf("expected 2 exclude patterns, got %d", len(filter.excludePatterns))
	}
	if len(filter.includePatterns) != 1 {
		t.Errorf("expected 1 include pattern, got %d", len(filter.includePatterns))
	}
}

func TestNewURLFilterWithPatterns(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("empty patterns", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{})
		if len(filter.excludePatterns) != 0 {
			t.Errorf("expected 0 exclude patterns, got %d", len(filter.excludePatterns))
		}
		if len(filter.includePatterns) != 0 {
			t.Errorf("expected 0 include patterns, got %d", len(filter.includePatterns))
		}
	})

	t.Run("invalid patterns are skipped silently", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			ExcludePatterns: []string{`[invalid`},
			IncludePatterns: []string{`[also invalid`},
		})
		if len(filter.excludePatterns) != 0 {
			t.Errorf("invalid exclude pattern should be skipped, got %d patterns", len(filter.excludePatterns))
		}
		if len(filter.includePatterns) != 0 {
			t.Errorf("invalid include pattern should be skipped, got %d patterns", len(filter.includePatterns))
		}
	})

	t.Run("mixed valid and invalid patterns", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			ExcludePatterns: []string{`\.pdf$`, `[bad`, `\.zip$`},
			IncludePatterns: []string{`[bad`, `\.html$`},
		})
		if len(filter.excludePatterns) != 2 {
			t.Errorf("expected 2 valid exclude patterns, got %d", len(filter.excludePatterns))
		}
		if len(filter.includePatterns) != 1 {
			t.Errorf("expected 1 valid include pattern, got %d", len(filter.includePatterns))
		}
	})
}

func TestShouldDownloadBasic(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")
	filter := NewURLFilter(baseURL, URLFilterConfig{
		MaxDepth: 5,
	})

	t.Run("http scheme allowed", func(t *testing.T) {
		u, _ := url.Parse("http://example.com/page")
		if !filter.ShouldDownload(u, 0) {
			t.Error("http URL should be allowed")
		}
	})

	t.Run("https scheme allowed", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, 0) {
			t.Error("https URL should be allowed")
		}
	})

	t.Run("same host allowed default", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/other")
		if !filter.ShouldDownload(u, 0) {
			t.Error("same host should be allowed")
		}
	})

	t.Run("nil URL panics", func(t *testing.T) {
		defer func() {
			if r := recover(); r == nil {
				t.Error("ShouldDownload with nil URL should panic")
			}
		}()
		filter.ShouldDownload(nil, 0)
	})
}

func TestShouldDownloadDepth(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("within depth", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 3})
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, 2) {
			t.Error("depth 2 should be allowed with maxDepth 3")
		}
	})

	t.Run("exactly at depth boundary", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 3})
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, 3) {
			t.Error("depth 3 should be allowed with maxDepth 3 (exact boundary)")
		}
	})

	t.Run("exceeds depth", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 3})
		u, _ := url.Parse("https://example.com/page")
		if filter.ShouldDownload(u, 4) {
			t.Error("depth 4 should NOT be allowed with maxDepth 3")
		}
	})

	t.Run("zero maxDepth means only depth 0", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 0})
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, 0) {
			t.Error("depth 0 should be allowed with maxDepth 0")
		}
		if filter.ShouldDownload(u, 1) {
			t.Error("depth 1 should NOT be allowed with maxDepth 0")
		}
	})

	t.Run("negative depth", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 0})
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, -1) {
			t.Error("negative depth should be allowed with maxDepth 0 (depth check is currentDepth > maxDepth, and -1 > 0 is false)")
		}
	})
}

func TestShouldDownloadScheme(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")
	filter := NewURLFilter(baseURL, URLFilterConfig{MaxDepth: 5})

	tests := []struct {
		urlStr string
		allow  bool
	}{
		{"http://example.com/page", true},
		{"https://example.com/page", true},
		{"ftp://example.com/file", false},
		{"file:///tmp/doc.txt", false},
		{"data:text/html,Hello", false},
		{"javascript:void(0)", false},
	}

	for _, tt := range tests {
		u, err := url.Parse(tt.urlStr)
		if err != nil {
			t.Errorf("failed to parse URL %s: %v", tt.urlStr, err)
			continue
		}
		got := filter.ShouldDownload(u, 0)
		if got != tt.allow {
			t.Errorf("ShouldDownload(%s) = %v, want %v", tt.urlStr, got, tt.allow)
		}
	}
}

func TestShouldDownloadExternal(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("external blocked by default", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:       5,
			FollowExternal: false,
		})
		u, _ := url.Parse("https://other.com/page")
		if filter.ShouldDownload(u, 0) {
			t.Error("external URL should be blocked when followExternal is false")
		}
	})

	t.Run("external allowed when followExternal is true", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:       5,
			FollowExternal: true,
		})
		u, _ := url.Parse("https://other.com/page")
		if !filter.ShouldDownload(u, 0) {
			t.Error("external URL should be allowed when followExternal is true")
		}
	})

	t.Run("same host always allowed", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:       5,
			FollowExternal: false,
		})
		u, _ := url.Parse("https://example.com/page")
		if !filter.ShouldDownload(u, 0) {
			t.Error("same host URL should always be allowed")
		}
	})

	t.Run("subdomain is considered external", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:       5,
			FollowExternal: false,
		})
		u, _ := url.Parse("https://sub.example.com/page")
		if filter.ShouldDownload(u, 0) {
			t.Error("subdomain should be considered external when followExternal is false")
		}
	})
}

func TestShouldDownloadExcludePattern(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")
	filter := NewURLFilter(baseURL, URLFilterConfig{
		MaxDepth:        5,
		ExcludePatterns: []string{`\.pdf$`, `\.zip$`, `/api/`},
	})

	tests := []struct {
		urlStr string
		allow  bool
	}{
		{"https://example.com/doc.pdf", false},
		{"https://example.com/archive.zip", false},
		{"https://example.com/api/users", false},
		{"https://example.com/page.html", true},
		{"https://example.com/image.jpg", true},
		{"https://example.com/style.css", true},
	}

	for _, tt := range tests {
		u, _ := url.Parse(tt.urlStr)
		got := filter.ShouldDownload(u, 0)
		if got != tt.allow {
			t.Errorf("ShouldDownload(%s) = %v, want %v", tt.urlStr, got, tt.allow)
		}
	}
}

func TestShouldDownloadIncludePattern(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("include only .html and .css", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:        5,
			IncludePatterns: []string{`\.html$`, `\.css$`},
		})

		tests := []struct {
			urlStr string
			allow  bool
		}{
			{"https://example.com/page.html", true},
			{"https://example.com/style.css", true},
			{"https://example.com/image.jpg", false},
			{"https://example.com/doc.pdf", false},
			{"https://example.com/script.js", false},
		}

		for _, tt := range tests {
			u, _ := url.Parse(tt.urlStr)
			got := filter.ShouldDownload(u, 0)
			if got != tt.allow {
				t.Errorf("ShouldDownload(%s) = %v, want %v", tt.urlStr, got, tt.allow)
			}
		}
	})

	t.Run("include pattern with depth check", func(t *testing.T) {
		filter := NewURLFilter(baseURL, URLFilterConfig{
			MaxDepth:        2,
			IncludePatterns: []string{`\.html$`},
		})

		u, _ := url.Parse("https://example.com/page.html")
		if filter.ShouldDownload(u, 3) {
			t.Error("include pattern should not override depth limit")
		}
	})
}

func TestIsSameDomain(t *testing.T) {
	base, _ := url.Parse("https://example.com/")

	t.Run("same host", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/page")
		if !IsSameDomain(base, target) {
			t.Error("same host should return true")
		}
	})

	t.Run("different host", func(t *testing.T) {
		target, _ := url.Parse("https://other.com/page")
		if IsSameDomain(base, target) {
			t.Error("different host should return false")
		}
	})

	t.Run("subdomain is different", func(t *testing.T) {
		target, _ := url.Parse("https://sub.example.com/page")
		if IsSameDomain(base, target) {
			t.Error("subdomain should be treated as different domain")
		}
	})

	t.Run("nil base", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/page")
		if IsSameDomain(nil, target) {
			t.Error("nil base should return false")
		}
	})

	t.Run("nil target", func(t *testing.T) {
		if IsSameDomain(base, nil) {
			t.Error("nil target should return false")
		}
	})

	t.Run("both nil", func(t *testing.T) {
		if IsSameDomain(nil, nil) {
			t.Error("both nil should return false")
		}
	})

	t.Run("same host different scheme", func(t *testing.T) {
		target, _ := url.Parse("http://example.com/page")
		if !IsSameDomain(base, target) {
			t.Error("same host with different scheme should return true")
		}
	})

	t.Run("same host different port", func(t *testing.T) {
		target, _ := url.Parse("https://example.com:8080/page")
		if !IsSameDomain(base, target) {
			t.Error("same host with different port should return true")
		}
	})

	t.Run("www prefix considered different", func(t *testing.T) {
		target, _ := url.Parse("https://www.example.com/page")
		if IsSameDomain(base, target) {
			t.Error("www subdomain should be treated as different domain")
		}
	})
}

func TestIsSubPath(t *testing.T) {
	base, _ := url.Parse("https://example.com/docs/")

	t.Run("direct subpath", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/user-guide")
		if !IsSubPath(base, target) {
			t.Error("/docs/user-guide should be subpath of /docs/")
		}
	})

	t.Run("nested subpath", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/user-guide/chapter1")
		if !IsSubPath(base, target) {
			t.Error("/docs/user-guide/chapter1 should be subpath of /docs/")
		}
	})

	t.Run("not a subpath", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/other/page")
		if IsSubPath(base, target) {
			t.Error("/other/page should NOT be subpath of /docs/")
		}
	})

	t.Run("same path", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/")
		if !IsSubPath(base, target) {
			t.Error("same path should be a subpath")
		}
	})

	t.Run("root base", func(t *testing.T) {
		rootBase, _ := url.Parse("https://example.com/")
		target, _ := url.Parse("https://example.com/anything")
		if !IsSubPath(rootBase, target) {
			t.Error("anything should be subpath of root")
		}
	})

	t.Run("nil base", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/page")
		if IsSubPath(nil, target) {
			t.Error("nil base should return false")
		}
	})

	t.Run("nil target", func(t *testing.T) {
		if IsSubPath(base, nil) {
			t.Error("nil target should return false")
		}
	})

	t.Run("both nil", func(t *testing.T) {
		if IsSubPath(nil, nil) {
			t.Error("both nil should return false")
		}
	})
}

func TestGetLocalPath(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/docs/")

	t.Run("same host with path", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/user-guide.html")
		path := GetLocalPath(baseURL, target, "/output")
		expected := "/output/docs/user-guide.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("same host no extension", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/about")
		path := GetLocalPath(baseURL, target, "/output")
		expected := "/output/docs/about.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("root path with trailing slash", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/")
		path := GetLocalPath(baseURL, target, "/output")
		expected := "/output/docs/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("empty path resolves to index.html", func(t *testing.T) {
		target, _ := url.Parse("https://example.com")
		path := GetLocalPath(baseURL, target, "/output")
		expected := "/output/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})
}

func TestGetLocalPathSameHost(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("basic path", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/page.html")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/page.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("nested path", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/a/b/c.html")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/a/b/c.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("path with trailing slash", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/blog/")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/blog/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("file at root", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/data.json")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/data.json"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})
}

func TestGetLocalPathDifferentHost(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("different host includes hostname in path", func(t *testing.T) {
		target, _ := url.Parse("https://other.com/page.html")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/other.com/page.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("different host nested path", func(t *testing.T) {
		target, _ := url.Parse("https://other.com/a/b/page.html")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/other.com/a/b/page.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("different host with trailing slash", func(t *testing.T) {
		target, _ := url.Parse("https://other.com/docs/")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/other.com/docs/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("different host root path without slash", func(t *testing.T) {
		target, _ := url.Parse("https://other.com")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/other.com/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})
}

func TestGetLocalPathNoExtension(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("path without extension gets .html appended", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/about")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/about.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("nested path without extension", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/about")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/docs/about.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("path with dot in directory name", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/v2.0/docs/about")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/v2.0/docs/about.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})

	t.Run("root path with trailing slash and no extension", func(t *testing.T) {
		target, _ := url.Parse("https://example.com/docs/")
		path := GetLocalPath(baseURL, target, "/out")
		expected := "/out/docs/index.html"
		if path != expected {
			t.Errorf("expected %s, got %s", expected, path)
		}
	})
}

func TestGetFilename(t *testing.T) {
	t.Run("normal file", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/page.html")
		if got := GetFilename(u); got != "page.html" {
			t.Errorf("expected page.html, got %s", got)
		}
	})

	t.Run("root path", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/")
		if got := GetFilename(u); got != "index.html" {
			t.Errorf("expected index.html, got %s", got)
		}
	})

	t.Run("empty path", func(t *testing.T) {
		u, _ := url.Parse("https://example.com")
		if got := GetFilename(u); got != "index.html" {
			t.Errorf("expected index.html, got %s", got)
		}
	})

	t.Run("nested file", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/a/b/file.pdf")
		if got := GetFilename(u); got != "file.pdf" {
			t.Errorf("expected file.pdf, got %s", got)
		}
	})

	t.Run("path ending with slash", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/blog/")
		if got := GetFilename(u); got != "blog" {
			t.Errorf("expected blog, got %s", got)
		}
	})

	t.Run("file with query parameters", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/script.js?ver=2")
		if got := GetFilename(u); got != "script.js" {
			t.Errorf("expected script.js, got %s", got)
		}
	})

	t.Run("deeply nested path", func(t *testing.T) {
		u, _ := url.Parse("https://example.com/a/b/c/d/e/f/image.png")
		if got := GetFilename(u); got != "image.png" {
			t.Errorf("expected image.png, got %s", got)
		}
	})
}

// ============================================================
// Parser tests
// ============================================================

func TestNewLinkExtractor(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")
	extractor := NewLinkExtractor(baseURL)

	if extractor == nil {
		t.Fatal("NewLinkExtractor returned nil")
	}
	if extractor.baseURL.String() != "https://example.com/" {
		t.Errorf("expected baseURL https://example.com/, got %s", extractor.baseURL.String())
	}
	if extractor.links == nil {
		t.Error("expected links slice to be initialized")
	}
	if len(extractor.links) != 0 {
		t.Errorf("expected empty links slice, got %d items", len(extractor.links))
	}
}

func TestExtractLinksBasic(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("single anchor tag", func(t *testing.T) {
		html := `<a href="/page.html">Page</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/page.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})

	t.Run("multiple anchors", func(t *testing.T) {
		html := `
			<a href="/page1.html">Page 1</a>
			<a href="/page2.html">Page 2</a>
			<a href="/page3.html">Page 3</a>
		`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 3 {
			t.Fatalf("expected 3 links, got %d", len(links))
		}
	})

	t.Run("absolute URL in href", func(t *testing.T) {
		html := `<a href="https://example.com/absolute.html">Absolute</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/absolute.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})
}

func TestExtractLinksAllTags(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	html := `
		<!DOCTYPE html>
		<html>
		<head>
			<link rel="stylesheet" href="/style.css">
		</head>
		<body>
			<a href="/page.html">Link</a>
			<img src="/image.png" alt="Image">
			<script src="/script.js"></script>
			<iframe src="/iframe.html"></iframe>
			<video src="/video.mp4"></video>
			<audio src="/audio.mp3"></audio>
			<source src="/source.webm">
		</body>
		</html>
	`

	links, err := ExtractLinks(baseURL, []byte(html))
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	// source tag is handled by extractSrc, so all 8 should be extracted
	expectedCount := 8
	if len(links) != expectedCount {
		t.Fatalf("expected %d links, got %d", expectedCount, len(links))
	}

	// Verify each expected URL is present
	expectedURLs := []string{
		"https://example.com/style.css",
		"https://example.com/page.html",
		"https://example.com/image.png",
		"https://example.com/script.js",
		"https://example.com/iframe.html",
		"https://example.com/video.mp4",
		"https://example.com/audio.mp3",
		"https://example.com/source.webm",
	}

	for _, expected := range expectedURLs {
		found := false
		for _, link := range links {
			if link.String() == expected {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("expected URL %s not found in extracted links", expected)
		}
	}
}

func TestExtractLinksSkipInvalid(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	html := `
		<a href="#section">Anchor</a>
		<a href="javascript:void(0)">JS</a>
		<a href="mailto:user@example.com">Email</a>
		<a href="tel:+1234567890">Phone</a>
		<a href="data:text/html,Hello">Data</a>
		<a href="/valid.html">Valid</a>
		<a href="">Empty</a>
	`

	links, err := ExtractLinks(baseURL, []byte(html))
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	if len(links) != 1 {
		t.Fatalf("expected 1 valid link, got %d", len(links))
	}

	expected := "https://example.com/valid.html"
	if links[0].String() != expected {
		t.Errorf("expected %s, got %s", expected, links[0].String())
	}
}

func TestExtractLinksRelativeURLs(t *testing.T) {
	t.Run("relative path with ..", func(t *testing.T) {
		baseURL, _ := url.Parse("https://example.com/docs/")
		html := `<a href="../page.html">Up</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/page.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})

	t.Run("relative path without leading slash", func(t *testing.T) {
		baseURL, _ := url.Parse("https://example.com/docs/")
		html := `<a href="guide.html">Guide</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/docs/guide.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})

	t.Run("root-relative path", func(t *testing.T) {
		baseURL, _ := url.Parse("https://example.com/docs/")
		html := `<a href="/about.html">About</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/about.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})

	t.Run("protocol-relative URL", func(t *testing.T) {
		baseURL, _ := url.Parse("https://example.com/")
		html := `<a href="//other.com/page.html">Protocol Relative</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://other.com/page.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})
}

func TestExtractLinksEmpty(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	t.Run("empty byte slice", func(t *testing.T) {
		links, err := ExtractLinks(baseURL, []byte{})
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 0 {
			t.Errorf("expected 0 links from empty HTML, got %d", len(links))
		}
	})

	t.Run("nil byte slice", func(t *testing.T) {
		_, err := ExtractLinks(baseURL, nil)
		if err != nil {
			t.Fatalf("unexpected error from nil input: %v", err)
		}
	})

	t.Run("whitespace only", func(t *testing.T) {
		links, err := ExtractLinks(baseURL, []byte("   \n   \t   "))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 0 {
			t.Errorf("expected 0 links from whitespace, got %d", len(links))
		}
	})
}

func TestExtractLinksNoLinks(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	html := `
		<!DOCTYPE html>
		<html>
		<head><title>Test</title></head>
		<body>
			<h1>Hello World</h1>
			<p>No links here!</p>
		</body>
		</html>
	`

	links, err := ExtractLinks(baseURL, []byte(html))
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if len(links) != 0 {
		t.Errorf("expected 0 links from HTML without links, got %d", len(links))
	}
}

func TestExtractLinksBaseURLWithPath(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/docs/guide/")

	t.Run("relative URL resolves relative to base path", func(t *testing.T) {
		html := `<a href="chapter1.html">Chapter 1</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/docs/guide/chapter1.html"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})

	t.Run("parent path resolution", func(t *testing.T) {
		html := `<a href="../images/logo.png">Logo</a>`
		links, err := ExtractLinks(baseURL, []byte(html))
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(links) != 1 {
			t.Fatalf("expected 1 link, got %d", len(links))
		}
		expected := "https://example.com/docs/images/logo.png"
		if links[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, links[0].String())
		}
	})
}

func TestExtractLinksDuplicateRemoval(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")

	html := `
		<a href="/page.html">Page 1</a>
		<a href="/page.html">Page 2</a>
		<a href="https://example.com/page.html">Page 3</a>
		<a href="/other.html">Other</a>
	`

	links, err := ExtractLinks(baseURL, []byte(html))
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	if len(links) != 2 {
		t.Fatalf("expected 2 unique links, got %d", len(links))
	}

	expectedURLs := []string{
		"https://example.com/page.html",
		"https://example.com/other.html",
	}

	for _, expected := range expectedURLs {
		found := false
		for _, link := range links {
			if link.String() == expected {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("expected URL %s not found", expected)
		}
	}
}

func TestExtractLinksFromHTML(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/")
	html := `<a href="/test.html">Test</a>`

	links1, err1 := ExtractLinks(baseURL, []byte(html))
	links2, err2 := ExtractLinksFromHTML(baseURL, []byte(html))

	if err1 != nil || err2 != nil {
		t.Fatal("both functions should return no error")
	}

	if len(links1) != len(links2) {
		t.Fatalf("ExtractLinks returned %d links, ExtractLinksFromHTML returned %d", len(links1), len(links2))
	}

	if len(links1) > 0 && links1[0].String() != links2[0].String() {
		t.Errorf("ExtractLinks returned %s, ExtractLinksFromHTML returned %s", links1[0].String(), links2[0].String())
	}
}

// ============================================================
// Crawler tests
// ============================================================

func TestNewCrawler(t *testing.T) {
	cfg := &CrawlerConfig{
		MaxDepth:       3,
		FollowExternal: false,
		OutputDir:      "/tmp/test",
		Parallel:       2,
		UserAgent:      "TestBot/1.0",
	}

	crawler := NewCrawler(cfg, nil)

	if crawler == nil {
		t.Fatal("NewCrawler returned nil")
	}
	if crawler.config.MaxDepth != 3 {
		t.Errorf("expected MaxDepth 3, got %d", crawler.config.MaxDepth)
	}
	if crawler.config.FollowExternal != false {
		t.Errorf("expected FollowExternal false, got %v", crawler.config.FollowExternal)
	}
	if crawler.config.OutputDir != "/tmp/test" {
		t.Errorf("expected OutputDir /tmp/test, got %s", crawler.config.OutputDir)
	}
	if crawler.config.UserAgent != "TestBot/1.0" {
		t.Errorf("expected UserAgent TestBot/1.0, got %s", crawler.config.UserAgent)
	}
	if crawler.config.Parallel != 2 {
		t.Errorf("expected Parallel 2, got %d", crawler.config.Parallel)
	}
	if crawler.filter == nil {
		t.Error("expected filter to be initialized")
	}
	if crawler.visited == nil {
		t.Error("expected visited map to be initialized")
	}
	if crawler.queue == nil {
		t.Error("expected queue to be initialized")
	}
	if crawler.stats == nil {
		t.Error("expected stats to be initialized")
	}
	if crawler.ctx == nil {
		t.Error("expected context to be initialized")
	}
}

func TestNewCrawlerNilURLFilterConfig(t *testing.T) {
	cfg := &CrawlerConfig{
		MaxDepth:        0,
		FollowExternal:  false,
		ExcludePatterns: nil,
		IncludePatterns: nil,
	}

	crawler := NewCrawler(cfg, nil)

	if crawler == nil {
		t.Fatal("NewCrawler returned nil")
	}
	if crawler.filter == nil {
		t.Fatal("expected filter to be initialized")
	}
	if len(crawler.filter.excludePatterns) != 0 {
		t.Errorf("expected 0 exclude patterns, got %d", len(crawler.filter.excludePatterns))
	}
	if len(crawler.filter.includePatterns) != 0 {
		t.Errorf("expected 0 include patterns, got %d", len(crawler.filter.includePatterns))
	}
}

func TestNewCrawlerDefaultConfig(t *testing.T) {
	cfg := &CrawlerConfig{}
	crawler := NewCrawler(cfg, nil)

	if crawler == nil {
		t.Fatal("NewCrawler returned nil")
	}
	if crawler.config.MaxDepth != 0 {
		t.Errorf("expected MaxDepth 0 as default, got %d", crawler.config.MaxDepth)
	}
	if crawler.config.Parallel != 0 {
		t.Errorf("expected Parallel 0 as default, got %d", crawler.config.Parallel)
	}
	if crawler.stats == nil {
		t.Error("expected stats to be initialized")
	}
	if crawler.stats.StartTime.IsZero() {
		t.Error("expected StartTime to be set")
	}
}

func TestGetStats(t *testing.T) {
	cfg := &CrawlerConfig{}
	crawler := NewCrawler(cfg, nil)

	stats := crawler.GetStats()

	if stats == nil {
		t.Fatal("GetStats returned nil")
	}
	if stats.TotalURLs != 0 {
		t.Errorf("expected TotalURLs 0, got %d", stats.TotalURLs)
	}
	if stats.DownloadedURLs != 0 {
		t.Errorf("expected DownloadedURLs 0, got %d", stats.DownloadedURLs)
	}
	if stats.FailedURLs != 0 {
		t.Errorf("expected FailedURLs 0, got %d", stats.FailedURLs)
	}
	if stats.SkippedURLs != 0 {
		t.Errorf("expected SkippedURLs 0, got %d", stats.SkippedURLs)
	}
	if stats.TotalBytes != 0 {
		t.Errorf("expected TotalBytes 0, got %d", stats.TotalBytes)
	}
	if stats.StartTime.IsZero() {
		t.Error("expected StartTime to be set")
	}

	// Verify it returns the same pointer (not a copy)
	if stats != crawler.stats {
		t.Error("GetStats should return the same stats instance")
	}
}

func TestGetStatsSummary(t *testing.T) {
	stats := &CrawlerStats{
		DownloadedURLs: 10,
		TotalBytes:     1500000,
		StartTime:      time.Now().Add(-5 * time.Second),
		EndTime:        time.Now(),
	}

	summary := stats.GetStatsSummary()

	if !strings.Contains(summary, "10") {
		t.Errorf("summary should contain 10 downloaded files, got: %s", summary)
	}
	if !strings.Contains(summary, "1.5 MB") {
		t.Errorf("summary should contain 1.5 MB, got: %s", summary)
	}
	if !strings.Contains(summary, "Downloaded:") {
		t.Errorf("summary should start with 'Downloaded:', got: %s", summary)
	}

	// Test with EndTime set
	stats.EndTime = time.Now()
	summary = stats.GetStatsSummary()
	if !strings.Contains(summary, "Downloaded:") {
		t.Errorf("summary should be valid when EndTime is set, got: %s", summary)
	}

	// Test zero values
	emptyStats := &CrawlerStats{}
	summary = emptyStats.GetStatsSummary()
	if summary == "" {
		t.Error("summary should not be empty even for zero stats")
	}
}

func TestHttpDetectContentType(t *testing.T) {
	t.Run("HTML declaration", func(t *testing.T) {
		data := []byte("<!DOCTYPE html><html><head></head><body></body></html>")
		contentType := httpDetectContentType(data)
		if contentType != "text/html" {
			t.Errorf("expected text/html, got %s", contentType)
		}
	})

	t.Run("HTML tag", func(t *testing.T) {
		data := []byte("<html><head></head><body></body></html>")
		contentType := httpDetectContentType(data)
		if contentType != "text/html" {
			t.Errorf("expected text/html, got %s", contentType)
		}
	})

	t.Run("uppercase HTML", func(t *testing.T) {
		data := []byte("<HTML><HEAD></HEAD><BODY></BODY></HTML>")
		contentType := httpDetectContentType(data)
		if contentType != "text/html" {
			t.Errorf("expected text/html, got %s", contentType)
		}
	})

	t.Run("non-HTML content", func(t *testing.T) {
		data := []byte("<?xml version=\"1.0\"?><root></root>")
		contentType := httpDetectContentType(data)
		if contentType != "" {
			t.Errorf("expected empty string, got %s", contentType)
		}
	})

	t.Run("empty data", func(t *testing.T) {
		contentType := httpDetectContentType([]byte{})
		if contentType != "" {
			t.Errorf("expected empty string, got %s", contentType)
		}
	})

	t.Run("nil data", func(t *testing.T) {
		contentType := httpDetectContentType(nil)
		if contentType != "" {
			t.Errorf("expected empty string, got %s", contentType)
		}
	})

	t.Run("single byte", func(t *testing.T) {
		contentType := httpDetectContentType([]byte{'x'})
		if contentType != "" {
			t.Errorf("expected empty string, got %s", contentType)
		}
	})

	t.Run("binary data", func(t *testing.T) {
		data := []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}
		contentType := httpDetectContentType(data)
		if contentType != "" {
			t.Errorf("expected empty string for binary data, got %s", contentType)
		}
	})
}

// TestExtractCSSURLs tests CSS URL extraction.
func TestExtractCSSURLs(t *testing.T) {
	baseURL, _ := url.Parse("https://example.com/css/")

	t.Run("url function", func(t *testing.T) {
		css := []byte(`body { background: url("bg.png"); }`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 1 {
			t.Fatalf("expected 1 URL, got %d", len(urls))
		}
		expected := "https://example.com/css/bg.png"
		if urls[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, urls[0].String())
		}
	})

	t.Run("import rule", func(t *testing.T) {
		css := []byte(`@import "style.css";`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 1 {
			t.Fatalf("expected 1 URL, got %d", len(urls))
		}
		expected := "https://example.com/css/style.css"
		if urls[0].String() != expected {
			t.Errorf("expected %s, got %s", expected, urls[0].String())
		}
	})

	t.Run("multiple urls", func(t *testing.T) {
		css := []byte(`
			body { background: url("bg.png"); }
			div { background: url('icon.svg'); }
			@import "theme.css";
		`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 3 {
			t.Fatalf("expected 3 URLs, got %d", len(urls))
		}
	})

	t.Run("deduplicates", func(t *testing.T) {
		css := []byte(`body { background: url("bg.png"); background: url("bg.png"); }`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 1 {
			t.Fatalf("expected 1 unique URL, got %d", len(urls))
		}
	})

	t.Run("empty", func(t *testing.T) {
		urls := ExtractCSSURLs(baseURL, []byte(``))
		if len(urls) != 0 {
			t.Errorf("expected 0 URLs, got %d", len(urls))
		}
	})

	t.Run("no urls", func(t *testing.T) {
		css := []byte(`body { color: red; }`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 0 {
			t.Errorf("expected 0 URLs, got %d", len(urls))
		}
	})

	t.Run("skips non-http", func(t *testing.T) {
		css := []byte(`body { background: url("data:image/png;base64,abc"); }`)
		urls := ExtractCSSURLs(baseURL, css)
		if len(urls) != 0 {
			t.Errorf("expected 0 URLs for data: URI, got %d", len(urls))
		}
	})
}