feat: initial goget release — modern IPv6-first download utility

internal/recursive/crawler_test.go

@@ -0,0 +1,157 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"context"
+	"net/url"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+	"codeberg.org/petrbalvin/goget/internal/protocol"
+)
+
+// mockProtocol is a controllable core.Protocol implementation for crawler
+// tests. The Download method honours ctx.Done() so a cancelled test can
+// observe the cancellation propagating into the worker.
+type mockProtocol struct {
+	*protocol.BaseProtocol
+	downloadDelay time.Duration
+	downloadCount int32
+}
+
+func (m *mockProtocol) Download(ctx context.Context, _ *core.DownloadRequest) (*core.DownloadResult, error) {
+	atomic.AddInt32(&m.downloadCount, 1)
+	if m.downloadDelay > 0 {
+		select {
+		case <-time.After(m.downloadDelay):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	return &core.DownloadResult{BytesDownloaded: 0}, nil
+}
+
+func newMockProtocol(delay time.Duration) *mockProtocol {
+	return &mockProtocol{
+		BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
+			Name:   "mock",
+			Scheme: "http",
+		}),
+		downloadDelay: delay,
+	}
+}
+
+// TestDownloadCancelDoesNotPanic is a regression guard for the BACKLOG
+// entry "Double close() crash in recursive crawler". Before the fix, the
+// cancellation goroutine and the main Download path both called
+// close(c.queue); whichever fired second panicked with
+// "close of closed channel". With sync.Once guarding the close, only the
+// first caller wins and the second is a no-op.
+func TestDownloadCancelDoesNotPanic(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	// Each download takes 50ms, long enough for the cancellation goroutine
+	// to fire closeQueue before wg.Wait() returns on its own.
+	proto := newMockProtocol(50 * time.Millisecond)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	done := make(chan struct{})
+	var panicVal any
+	go func() {
+		defer close(done)
+		defer func() {
+			panicVal = recover()
+		}()
+		_, _ = crawler.Download(context.Background(), baseURL)
+	}()
+
+	// Give the worker a moment to pick up the initial task and start the
+	// slow download, then cancel — this is the exact race the original
+	// bug triggered.
+	time.Sleep(20 * time.Millisecond)
+	crawler.Cancel()
+
+	select {
+	case <-done:
+	case <-time.After(3 * time.Second):
+		t.Fatal("Download did not return within 3s after Cancel")
+	}
+
+	if panicVal != nil {
+		t.Fatalf("Download panicked on Cancel: %v", panicVal)
+	}
+}
+
+// TestDownloadNormalCompletion verifies the happy path still works after
+// introducing sync.Once. Without the bug present, the only close() call
+// happens after wg.Wait() returns, and the workers must observe a closed
+// channel and exit cleanly.
+func TestDownloadNormalCompletion(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	proto := newMockProtocol(0)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	stats, err := crawler.Download(context.Background(), baseURL)
+	if err != nil {
+		t.Fatalf("Download: %v", err)
+	}
+	if stats.DownloadedURLs != 1 {
+		t.Errorf("DownloadedURLs = %d, want 1", stats.DownloadedURLs)
+	}
+	if got := atomic.LoadInt32(&proto.downloadCount); got != 1 {
+		t.Errorf("downloadCount = %d, want 1", got)
+	}
+}
+
+// TestDownloadCancelAfterAllTasksDone exercises the ordering where the
+// main path's wg.Wait() returns first and then the cancellation goroutine
+// fires. sync.Once must ignore the second close.
+func TestDownloadCancelAfterAllTasksDone(t *testing.T) {
+	dir := t.TempDir()
+	cfg := &CrawlerConfig{
+		OutputDir: dir,
+		Parallel:  1,
+	}
+	proto := newMockProtocol(0)
+	crawler := NewCrawler(cfg, proto)
+
+	baseURL, err := url.Parse("http://example.com/")
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		_, _ = crawler.Download(context.Background(), baseURL)
+	}()
+
+	<-done
+
+	// Cancelling after Download returned must not panic (no goroutine is
+	// waiting on closeQueue, but Cancel triggers ctx.Done() which the
+	// goroutine observes; sync.Once already ran so the second close is a
+	// no-op).
+	crawler.Cancel()
+}