goget/internal/recursive/crawler_test.go

//go:build linux || freebsd
// +build linux freebsd

package recursive

import (
	"context"
	"net/url"
	"sync/atomic"
	"testing"
	"time"

	"codeberg.org/petrbalvin/goget/internal/core"
	"codeberg.org/petrbalvin/goget/internal/protocol"
)

// mockProtocol is a controllable core.Protocol implementation for crawler
// tests. The Download method honours ctx.Done() so a cancelled test can
// observe the cancellation propagating into the worker.
type mockProtocol struct {
	*protocol.BaseProtocol
	downloadDelay time.Duration
	downloadCount int32
}

func (m *mockProtocol) Download(ctx context.Context, _ *core.DownloadRequest) (*core.DownloadResult, error) {
	atomic.AddInt32(&m.downloadCount, 1)
	if m.downloadDelay > 0 {
		select {
		case <-time.After(m.downloadDelay):
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}
	return &core.DownloadResult{BytesDownloaded: 0}, nil
}

func newMockProtocol(delay time.Duration) *mockProtocol {
	return &mockProtocol{
		BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
			Name:   "mock",
			Scheme: "http",
		}),
		downloadDelay: delay,
	}
}

// TestDownloadCancelDoesNotPanic is a regression guard for the BACKLOG
// entry "Double close() crash in recursive crawler". Before the fix, the
// cancellation goroutine and the main Download path both called
// close(c.queue); whichever fired second panicked with
// "close of closed channel". With sync.Once guarding the close, only the
// first caller wins and the second is a no-op.
func TestDownloadCancelDoesNotPanic(t *testing.T) {
	dir := t.TempDir()
	cfg := &CrawlerConfig{
		OutputDir: dir,
		Parallel:  1,
	}
	// Each download takes 50ms, long enough for the cancellation goroutine
	// to fire closeQueue before wg.Wait() returns on its own.
	proto := newMockProtocol(50 * time.Millisecond)
	crawler := NewCrawler(cfg, proto)

	baseURL, err := url.Parse("http://example.com/")
	if err != nil {
		t.Fatalf("parse: %v", err)
	}

	done := make(chan struct{})
	var panicVal any
	go func() {
		defer close(done)
		defer func() {
			panicVal = recover()
		}()
		_, _ = crawler.Download(context.Background(), baseURL)
	}()

	// Give the worker a moment to pick up the initial task and start the
	// slow download, then cancel — this is the exact race the original
	// bug triggered.
	time.Sleep(20 * time.Millisecond)
	crawler.Cancel()

	select {
	case <-done:
	case <-time.After(3 * time.Second):
		t.Fatal("Download did not return within 3s after Cancel")
	}

	if panicVal != nil {
		t.Fatalf("Download panicked on Cancel: %v", panicVal)
	}
}

// TestDownloadNormalCompletion verifies the happy path still works after
// introducing sync.Once. Without the bug present, the only close() call
// happens after wg.Wait() returns, and the workers must observe a closed
// channel and exit cleanly.
func TestDownloadNormalCompletion(t *testing.T) {
	dir := t.TempDir()
	cfg := &CrawlerConfig{
		OutputDir: dir,
		Parallel:  1,
	}
	proto := newMockProtocol(0)
	crawler := NewCrawler(cfg, proto)

	baseURL, err := url.Parse("http://example.com/")
	if err != nil {
		t.Fatalf("parse: %v", err)
	}

	stats, err := crawler.Download(context.Background(), baseURL)
	if err != nil {
		t.Fatalf("Download: %v", err)
	}
	if stats.DownloadedURLs != 1 {
		t.Errorf("DownloadedURLs = %d, want 1", stats.DownloadedURLs)
	}
	if got := atomic.LoadInt32(&proto.downloadCount); got != 1 {
		t.Errorf("downloadCount = %d, want 1", got)
	}
}

// TestDownloadCancelAfterAllTasksDone exercises the ordering where the
// main path's wg.Wait() returns first and then the cancellation goroutine
// fires. sync.Once must ignore the second close.
func TestDownloadCancelAfterAllTasksDone(t *testing.T) {
	dir := t.TempDir()
	cfg := &CrawlerConfig{
		OutputDir: dir,
		Parallel:  1,
	}
	proto := newMockProtocol(0)
	crawler := NewCrawler(cfg, proto)

	baseURL, err := url.Parse("http://example.com/")
	if err != nil {
		t.Fatalf("parse: %v", err)
	}

	done := make(chan struct{})
	go func() {
		defer close(done)
		_, _ = crawler.Download(context.Background(), baseURL)
	}()

	<-done

	// Cancelling after Download returned must not panic (no goroutine is
	// waiting on closeQueue, but Cancel triggers ctx.Done() which the
	// goroutine observes; sync.Once already ran so the second close is a
	// no-op).
	crawler.Cancel()
}