//go:build linux || freebsd // +build linux freebsd package recursive import ( "context" "net/url" "sync/atomic" "testing" "time" "codeberg.org/petrbalvin/goget/internal/core" "codeberg.org/petrbalvin/goget/internal/protocol" ) // mockProtocol is a controllable core.Protocol implementation for crawler // tests. The Download method honours ctx.Done() so a cancelled test can // observe the cancellation propagating into the worker. type mockProtocol struct { *protocol.BaseProtocol downloadDelay time.Duration downloadCount int32 } func (m *mockProtocol) Download(ctx context.Context, _ *core.DownloadRequest) (*core.DownloadResult, error) { atomic.AddInt32(&m.downloadCount, 1) if m.downloadDelay > 0 { select { case <-time.After(m.downloadDelay): case <-ctx.Done(): return nil, ctx.Err() } } return &core.DownloadResult{BytesDownloaded: 0}, nil } func newMockProtocol(delay time.Duration) *mockProtocol { return &mockProtocol{ BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{ Name: "mock", Scheme: "http", }), downloadDelay: delay, } } // TestDownloadCancelDoesNotPanic is a regression guard for the BACKLOG // entry "Double close() crash in recursive crawler". Before the fix, the // cancellation goroutine and the main Download path both called // close(c.queue); whichever fired second panicked with // "close of closed channel". With sync.Once guarding the close, only the // first caller wins and the second is a no-op. func TestDownloadCancelDoesNotPanic(t *testing.T) { dir := t.TempDir() cfg := &CrawlerConfig{ OutputDir: dir, Parallel: 1, } // Each download takes 50ms, long enough for the cancellation goroutine // to fire closeQueue before wg.Wait() returns on its own. proto := newMockProtocol(50 * time.Millisecond) crawler := NewCrawler(cfg, proto) baseURL, err := url.Parse("http://example.com/") if err != nil { t.Fatalf("parse: %v", err) } done := make(chan struct{}) var panicVal any go func() { defer close(done) defer func() { panicVal = recover() }() _, _ = crawler.Download(context.Background(), baseURL) }() // Give the worker a moment to pick up the initial task and start the // slow download, then cancel — this is the exact race the original // bug triggered. time.Sleep(20 * time.Millisecond) crawler.Cancel() select { case <-done: case <-time.After(3 * time.Second): t.Fatal("Download did not return within 3s after Cancel") } if panicVal != nil { t.Fatalf("Download panicked on Cancel: %v", panicVal) } } // TestDownloadNormalCompletion verifies the happy path still works after // introducing sync.Once. Without the bug present, the only close() call // happens after wg.Wait() returns, and the workers must observe a closed // channel and exit cleanly. func TestDownloadNormalCompletion(t *testing.T) { dir := t.TempDir() cfg := &CrawlerConfig{ OutputDir: dir, Parallel: 1, } proto := newMockProtocol(0) crawler := NewCrawler(cfg, proto) baseURL, err := url.Parse("http://example.com/") if err != nil { t.Fatalf("parse: %v", err) } stats, err := crawler.Download(context.Background(), baseURL) if err != nil { t.Fatalf("Download: %v", err) } if stats.DownloadedURLs != 1 { t.Errorf("DownloadedURLs = %d, want 1", stats.DownloadedURLs) } if got := atomic.LoadInt32(&proto.downloadCount); got != 1 { t.Errorf("downloadCount = %d, want 1", got) } } // TestDownloadCancelAfterAllTasksDone exercises the ordering where the // main path's wg.Wait() returns first and then the cancellation goroutine // fires. sync.Once must ignore the second close. func TestDownloadCancelAfterAllTasksDone(t *testing.T) { dir := t.TempDir() cfg := &CrawlerConfig{ OutputDir: dir, Parallel: 1, } proto := newMockProtocol(0) crawler := NewCrawler(cfg, proto) baseURL, err := url.Parse("http://example.com/") if err != nil { t.Fatalf("parse: %v", err) } done := make(chan struct{}) go func() { defer close(done) _, _ = crawler.Download(context.Background(), baseURL) }() <-done // Cancelling after Download returned must not panic (no goroutine is // waiting on closeQueue, but Cancel triggers ctx.Done() which the // goroutine observes; sync.Once already ran so the second close is a // no-op). crawler.Cancel() }