feat: initial goget release — modern IPv6-first download utility

2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
@@ -0,0 +1,559 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package http
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+	"codeberg.org/petrbalvin/goget/internal/transport"
+)
+
+// testWriter implements io.Writer for testing
+type testWriter struct {
+	buf strings.Builder
+}
+
+func (w *testWriter) Write(p []byte) (int, error) {
+	return w.buf.Write(p)
+}
+
+func TestDownloadSequentialIntegration(t *testing.T) {
+	// Create a test server that returns known content
+	testContent := "Hello, this is test content for download integration testing!"
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(testContent))
+	}))
+	defer server.Close()
+
+	// Parse the server URL
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("Failed to parse server URL: %v", err)
+	}
+
+	// Create the HTTP client
+	client, err := NewClient(DefaultConfig())
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+
+	// Create download request
+	req := &core.DownloadRequest{
+		URL:     parsedURL,
+		Output:  "",
+		Writer:  &testWriter{},
+		Verbose: false,
+		Headers: make(map[string]string),
+	}
+
+	// Execute download
+	result, err := client.Download(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Download failed: %v", err)
+	}
+
+	if result.BytesDownloaded <= 0 {
+		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
+	}
+
+	if result.BytesDownloaded != int64(len(testContent)) {
+		t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded)
+	}
+}
+
+func TestDownloadParallelIntegration(t *testing.T) {
+	// Create test content for parallel download
+	testContent := strings.Repeat("B", 1024*1024) // 1MB
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Support range requests
+		w.Header().Set("Accept-Ranges", "bytes")
+		w.Header().Set("Content-Type", "application/octet-stream")
+
+		rangeHeader := r.Header.Get("Range")
+		if rangeHeader != "" {
+			var start, end int64
+			n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
+			if n >= 1 {
+				if end == 0 || end >= int64(len(testContent)) {
+					end = int64(len(testContent)) - 1
+				}
+				w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent)))
+				w.WriteHeader(http.StatusPartialContent)
+				w.Write([]byte(testContent[start : end+1]))
+				return
+			}
+		}
+
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(testContent))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("Failed to parse server URL: %v", err)
+	}
+
+	cfg := DefaultConfig()
+	cfg.Parallel = &core.ParallelConfig{
+		Connections:  4,
+		MinSize:      100, // Force parallel for small files
+		MinChunkSize: 100,
+		MaxChunkSize: 1024 * 1024,
+	}
+
+	client, err := NewClient(cfg)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+
+	req := &core.DownloadRequest{
+		URL:      parsedURL,
+		Output:   "",
+		Writer:   &testWriter{},
+		Verbose:  false,
+		Headers:  make(map[string]string),
+		Parallel: cfg.Parallel,
+	}
+
+	result, err := client.Download(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Download failed: %v", err)
+	}
+
+	if result.BytesDownloaded <= 0 {
+		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
+	}
+
+	if !result.Parallel {
+		t.Log("Download was not parallel (may be expected depending on server negotiation)")
+	}
+}
+
+func TestDownloadResumeIntegration(t *testing.T) {
+	testContent := "This is content for resume testing!"
+	var requestCount int
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+
+		w.Header().Set("Accept-Ranges", "bytes")
+		w.Header().Set("ETag", "\"test-etag-123\"")
+
+		rangeHeader := r.Header.Get("Range")
+		if rangeHeader != "" {
+			var start int64
+			fmt.Sscanf(rangeHeader, "bytes=%d-", &start)
+			w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent)))
+			w.WriteHeader(http.StatusPartialContent)
+			w.Write([]byte(testContent[start:]))
+			return
+		}
+
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(testContent))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("Failed to parse server URL: %v", err)
+	}
+
+	client, err := NewClient(DefaultConfig())
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+
+	// Create resume info simulating a partially downloaded file
+	resumeInfo := &core.ResumeInfo{
+		DownloadedBytes: 10,
+		ETag:            "\"test-etag-123\"",
+		URL:             server.URL,
+	}
+
+	req := &core.DownloadRequest{
+		URL:        parsedURL,
+		Output:     "",
+		Writer:     &testWriter{},
+		Resume:     true,
+		ResumeInfo: resumeInfo,
+		Verbose:    false,
+		Headers:    make(map[string]string),
+	}
+
+	result, err := client.Download(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Download failed: %v", err)
+	}
+
+	if result.BytesDownloaded <= 0 {
+		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
+	}
+
+	if result.Resumed {
+		t.Log("Download was resumed successfully")
+	}
+}
+
+func TestUploadIntegration(t *testing.T) {
+	var receivedBody string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, "Failed to read body", http.StatusInternalServerError)
+			return
+		}
+		receivedBody = string(body)
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	}))
+	defer server.Close()
+
+	client, err := NewClient(DefaultConfig())
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+
+	// Test POST upload with form data
+	formReq := &UploadRequest{
+		URL:      server.URL,
+		Method:   "POST",
+		FormData: map[string]string{"key": "value"},
+		Timeout:  0,
+	}
+
+	result, err := client.Upload(context.Background(), formReq)
+	if err != nil {
+		t.Fatalf("Upload failed: %v", err)
+	}
+
+	if result.StatusCode != http.StatusOK {
+		t.Errorf("Expected status 200, got %d", result.StatusCode)
+	}
+
+	if result.BytesUploaded <= 0 {
+		t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded)
+	}
+
+	if !strings.Contains(receivedBody, "key=value") {
+		t.Errorf("Expected body to contain form data, got: %s", receivedBody)
+	}
+}
+
+func TestDownloadFollowsRedirectIntegration(t *testing.T) {
+	const finalBody = "redirected content"
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/redirect" {
+			http.Redirect(w, r, "/target", http.StatusFound)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(finalBody))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL + "/redirect")
+	if err != nil {
+		t.Fatalf("parse url: %v", err)
+	}
+
+	client, err := NewClient(DefaultConfig())
+	if err != nil {
+		t.Fatalf("create client: %v", err)
+	}
+
+	result, err := client.Download(context.Background(), &core.DownloadRequest{
+		URL:     parsedURL,
+		Writer:  &testWriter{},
+		Headers: map[string]string{},
+	})
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+	if result.BytesDownloaded != int64(len(finalBody)) {
+		t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
+	}
+}
+
+// TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a
+// server that always answers 500. It verifies the loop actually attempts the
+// configured number of retries and that each attempt's response body is
+// closed — guarding against the regression where http.Client.Do returned a
+// non-nil resp alongside an error and the body was never closed, exhausting
+// the connection pool.
+func TestRetryAllErrorsClosesBody(t *testing.T) {
+	var requestCount int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&requestCount, 1)
+		w.WriteHeader(http.StatusInternalServerError)
+		// Write a non-empty body so a leaking response body would matter
+		// (an empty body is closed trivially by net/http's chunked reader).
+		_, _ = w.Write([]byte("server error"))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("parse url: %v", err)
+	}
+
+	cfg := DefaultConfig()
+	cfg.Transport.RetryAllErrors = true
+	cfg.Transport.MaxRetries = 3
+	cfg.Transport.Retry = &transport.RetryConfig{
+		MaxRetries:   3,
+		InitialDelay: time.Millisecond,
+	}
+
+	client, err := NewClient(cfg)
+	if err != nil {
+		t.Fatalf("create client: %v", err)
+	}
+
+	_, err = client.Download(context.Background(), &core.DownloadRequest{
+		URL:     parsedURL,
+		Writer:  &testWriter{},
+		Headers: map[string]string{},
+	})
+	if err == nil {
+		t.Fatal("expected error after exhausting retries on 500 responses")
+	}
+
+	// We expect at least 1 (initial transport.Retry attempt) + MaxRetries
+	// (RetryAllErrors loop). The exact number may be higher because Go's
+	// HTTP transport may also retry once on its own when an attempt fails
+	// (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert
+	// the lower bound only — the goal is to confirm the retry loop runs and
+	// that every failed response body is closed, not to pin a Go-internal
+	// detail.
+	minRequests := int32(1 + cfg.Transport.MaxRetries)
+	if got := atomic.LoadInt32(&requestCount); got < minRequests {
+		t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries)
+	}
+}
+
+// TestUploadRateLimiterAndProgressAreChained is a regression guard for the
+// BACKLOG entry "Rate limiter silently dropped when progress wrapper
+// enabled". The two wrappers must compose — httpReq.Body must end up
+// reading through the rate-limited reader, with progress reported on top.
+func TestUploadRateLimiterAndProgressAreChained(t *testing.T) {
+	const payload = "the quick brown fox jumps over the lazy dog"
+
+	var receivedBody string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, "read error", http.StatusInternalServerError)
+			return
+		}
+		receivedBody = string(body)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	// Write payload to a temp file (Upload's PUT path opens the file).
+	tmpDir := t.TempDir()
+	tmpFile := filepath.Join(tmpDir, "payload.txt")
+	if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil {
+		t.Fatalf("write tmp: %v", err)
+	}
+
+	cfg := DefaultConfig()
+	// Generous rate limit so the test isn't dominated by the limiter.
+	// limitedReader.Read asks for len(p) tokens on every call, and the
+	// HTTP transport uses a 32 KB buffer, so burst must comfortably exceed
+	// that or Allow() blocks forever.
+	cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024)
+	cfg.Transport.MaxRetries = 0
+	cfg.Transport.Retry = &transport.RetryConfig{
+		MaxRetries:   0,
+		InitialDelay: time.Millisecond,
+	}
+
+	client, err := NewClient(cfg)
+	if err != nil {
+		t.Fatalf("create client: %v", err)
+	}
+
+	progressCalls := 0
+	var lastCurrent int64
+	progressCb := func(current, total int64, _ float64) {
+		progressCalls++
+		if current < lastCurrent {
+			t.Errorf("progress went backwards: %d -> %d", lastCurrent, current)
+		}
+		lastCurrent = current
+	}
+
+	_, err = client.Upload(context.Background(), &UploadRequest{
+		URL:              server.URL,
+		Method:           "PUT",
+		FilePath:         tmpFile,
+		ProgressCallback: progressCb,
+	})
+	if err != nil {
+		t.Fatalf("upload: %v", err)
+	}
+
+	// The server must receive the full, unmodified payload. If the rate
+	// limiter wrapper was silently dropped (e.g. httpReq.Body not updated
+	// after body was reassigned), the server would still get the data via
+	// the original reader — so this check alone doesn't expose the bug.
+	if receivedBody != payload {
+		t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload)
+	}
+
+	// Progress callback must fire at least once and reach the full payload
+	// length. The exact call count depends on the read-buffer behaviour of
+	// net/http (a 43-byte payload may be read in a single Read), but a count
+	// of zero would prove the progress reader was bypassed — which is the
+	// exact regression the BACKLOG entry warns about.
+	if progressCalls < 1 {
+		t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls)
+	}
+	if lastCurrent != int64(len(payload)) {
+		t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload))
+	}
+}
+
+// TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few
+// 500s short-circuits the retry loop and produces a successful download —
+// another code path that must close every preceding failed response body
+// before jumping to okStatus.
+func TestRetryAllErrorsRecoversOn200(t *testing.T) {
+	const finalBody = "ok after retries"
+	var requestCount int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&requestCount, 1)
+		if n < 3 {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte("server error"))
+			return
+		}
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody)))
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(finalBody))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("parse url: %v", err)
+	}
+
+	cfg := DefaultConfig()
+	cfg.Transport.RetryAllErrors = true
+	cfg.Transport.MaxRetries = 5
+	cfg.Transport.Retry = &transport.RetryConfig{
+		MaxRetries:   5,
+		InitialDelay: time.Millisecond,
+	}
+
+	client, err := NewClient(cfg)
+	if err != nil {
+		t.Fatalf("create client: %v", err)
+	}
+
+	result, err := client.Download(context.Background(), &core.DownloadRequest{
+		URL:     parsedURL,
+		Writer:  &testWriter{},
+		Headers: map[string]string{},
+	})
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+	if result.BytesDownloaded != int64(len(finalBody)) {
+		t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
+	}
+	if got := atomic.LoadInt32(&requestCount); got != 3 {
+		t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got)
+	}
+}
+
+// TestTraceTimeTTFBOnReusedConnection is a regression guard for the
+// BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on
+// reused connections". The previous implementation measured TTFB as
+// time.Since(dnsStart); on a pooled HTTP connection the DNSStart
+// callback is silent, dnsStart stayed zero, and the result was ~55 years
+// instead of a plausible request-time duration. The fix anchors TTFB to
+// a requestStart timestamp captured before the trace is attached, so
+// the value is meaningful whether or not the connection is reused.
+func TestTraceTimeTTFBOnReusedConnection(t *testing.T) {
+	var connectionCount int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&connectionCount, 1)
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer server.Close()
+
+	parsedURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("Failed to parse server URL: %v", err)
+	}
+
+	cfg := DefaultConfig()
+	cfg.Output.TraceTime = true
+	client, err := NewClient(cfg)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+
+	reqTemplate := func() *core.DownloadRequest {
+		return &core.DownloadRequest{
+			URL:     parsedURL,
+			Writer:  &testWriter{},
+			Headers: make(map[string]string),
+		}
+	}
+
+	// First request opens the connection — DNSStart/ConnectStart/
+	// TLSHandshakeStart all fire, populating the trace timestamps.
+	if _, err := client.Download(context.Background(), reqTemplate()); err != nil {
+		t.Fatalf("first download failed: %v", err)
+	}
+
+	// Second request should be served from the pool. Whether or not
+	// keep-alive actually reuses the connection in this test depends
+	// on the transport's MaxIdleConnsPerHost, but the assertion is the
+	// same: TTFB must be a plausible duration in either case.
+	result, err := client.Download(context.Background(), reqTemplate())
+	if err != nil {
+		t.Fatalf("second download failed: %v", err)
+	}
+
+	if result.TraceTimings == nil {
+		t.Fatal("TraceTimings is nil despite TraceTime=true")
+	}
+
+	// 1 hour is a generous upper bound that flags the original ~55-year
+	// regression (time.Since(time.Time{}) on a fresh time.Time) without
+	// being flaky on a busy CI runner.
+	if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour {
+		t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb)
+	}
+	if ttfb := result.TraceTimings.TTFB; ttfb < 0 {
+		t.Errorf("TTFB = %v; want non-negative", ttfb)
+	}
+}