goget/internal/mirror/mirror_test.go

//go:build linux || freebsd
// +build linux freebsd

package mirror

import (
	"net/url"
	"testing"
	"time"
)

func TestNewRobotsChecker(t *testing.T) {
	t.Run("creates checker with user agent", func(t *testing.T) {
		ua := "MyBot/1.0"
		rc := NewRobotsChecker(ua)
		if rc == nil {
			t.Fatal("NewRobotsChecker returned nil")
		}
		if rc.userAgent != ua {
			t.Errorf("userAgent = %q; want %q", rc.userAgent, ua)
		}
		if rc.policies == nil {
			t.Error("policies map should be initialized")
		}
	})

	t.Run("creates checker with empty user agent", func(t *testing.T) {
		rc := NewRobotsChecker("")
		if rc == nil {
			t.Fatal("NewRobotsChecker('') returned nil")
		}
		if rc.userAgent != "" {
			t.Errorf("userAgent = %q; want empty string", rc.userAgent)
		}
	})
}

func TestRobotsCheckerCanFetch(t *testing.T) {
	t.Run("no policy returns true", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/page")
		if !rc.CanFetch(u) {
			t.Error("CanFetch with no policy should return true")
		}
	})

	t.Run("empty policy returns true", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/page")

		// Add an empty policy directly
		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{}
		rc.mu.Unlock()

		if !rc.CanFetch(u) {
			t.Error("CanFetch with empty policy should return true")
		}
	})

	t.Run("disallowed path returns false", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/private/data")

		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{
			disallowed: []string{"/private"},
		}
		rc.mu.Unlock()

		if rc.CanFetch(u) {
			t.Error("CanFetch with disallowed path should return false")
		}
	})

	t.Run("allowed path returns true even with disallowed rules", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/public/data")

		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{
			disallowed: []string{"/private"},
		}
		rc.mu.Unlock()

		if !rc.CanFetch(u) {
			t.Error("CanFetch with non-disallowed path should return true")
		}
	})

	t.Run("root path with empty string path in URL", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		// URL with empty path
		u, _ := url.Parse("https://example.com")

		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{
			disallowed: []string{"/"},
		}
		rc.mu.Unlock()

		if rc.CanFetch(u) {
			t.Error("CanFetch with disallowed root should return false")
		}
	})

	t.Run("different host uses different policy", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		uAllowed, _ := url.Parse("https://allowed.com/page")
		uDisallowed, _ := url.Parse("https://disallowed.com/secret")

		rc.mu.Lock()
		rc.policies["allowed.com"] = &robotsPolicy{}
		rc.policies["disallowed.com"] = &robotsPolicy{
			disallowed: []string{"/secret"},
		}
		rc.mu.Unlock()

		if !rc.CanFetch(uAllowed) {
			t.Error("CanFetch on allowed host should return true")
		}
		if rc.CanFetch(uDisallowed) {
			t.Error("CanFetch on disallowed path should return false")
		}
	})
}

func TestRobotsCheckerGetCrawlDelay(t *testing.T) {
	t.Run("no policy returns zero", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/")
		if d := rc.GetCrawlDelay(u); d != 0 {
			t.Errorf("GetCrawlDelay without policy = %v; want 0", d)
		}
	})

	t.Run("returns configured delay", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/")

		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{
			crawlDelay: 5 * time.Second,
		}
		rc.mu.Unlock()

		if d := rc.GetCrawlDelay(u); d != 5*time.Second {
			t.Errorf("GetCrawlDelay = %v; want 5s", d)
		}
	})

	t.Run("zero delay returns zero", func(t *testing.T) {
		rc := NewRobotsChecker("Bot")
		u, _ := url.Parse("https://example.com/")

		rc.mu.Lock()
		rc.policies["example.com"] = &robotsPolicy{
			crawlDelay: 0,
		}
		rc.mu.Unlock()

		if d := rc.GetCrawlDelay(u); d != 0 {
			t.Errorf("GetCrawlDelay with 0 delay = %v; want 0", d)
		}
	})
}