//go:build linux || freebsd // +build linux freebsd package http import ( "context" "fmt" "io" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "codeberg.org/petrbalvin/goget/internal/core" "codeberg.org/petrbalvin/goget/internal/transport" ) // testWriter implements io.Writer for testing type testWriter struct { buf strings.Builder } func (w *testWriter) Write(p []byte) (int, error) { return w.buf.Write(p) } func TestDownloadSequentialIntegration(t *testing.T) { // Create a test server that returns known content testContent := "Hello, this is test content for download integration testing!" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent))) w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) defer server.Close() // Parse the server URL parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("Failed to parse server URL: %v", err) } // Create the HTTP client client, err := NewClient(DefaultConfig()) if err != nil { t.Fatalf("Failed to create client: %v", err) } // Create download request req := &core.DownloadRequest{ URL: parsedURL, Output: "", Writer: &testWriter{}, Verbose: false, Headers: make(map[string]string), } // Execute download result, err := client.Download(context.Background(), req) if err != nil { t.Fatalf("Download failed: %v", err) } if result.BytesDownloaded <= 0 { t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded) } if result.BytesDownloaded != int64(len(testContent)) { t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded) } } func TestDownloadParallelIntegration(t *testing.T) { // Create test content for parallel download testContent := strings.Repeat("B", 1024*1024) // 1MB server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Support range requests w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("Content-Type", "application/octet-stream") rangeHeader := r.Header.Get("Range") if rangeHeader != "" { var start, end int64 n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end) if n >= 1 { if end == 0 || end >= int64(len(testContent)) { end = int64(len(testContent)) - 1 } w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent))) w.WriteHeader(http.StatusPartialContent) w.Write([]byte(testContent[start : end+1])) return } } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent))) w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) defer server.Close() parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("Failed to parse server URL: %v", err) } cfg := DefaultConfig() cfg.Parallel = &core.ParallelConfig{ Connections: 4, MinSize: 100, // Force parallel for small files MinChunkSize: 100, MaxChunkSize: 1024 * 1024, } client, err := NewClient(cfg) if err != nil { t.Fatalf("Failed to create client: %v", err) } req := &core.DownloadRequest{ URL: parsedURL, Output: "", Writer: &testWriter{}, Verbose: false, Headers: make(map[string]string), Parallel: cfg.Parallel, } result, err := client.Download(context.Background(), req) if err != nil { t.Fatalf("Download failed: %v", err) } if result.BytesDownloaded <= 0 { t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded) } if !result.Parallel { t.Log("Download was not parallel (may be expected depending on server negotiation)") } } func TestDownloadResumeIntegration(t *testing.T) { testContent := "This is content for resume testing!" var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount++ w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("ETag", "\"test-etag-123\"") rangeHeader := r.Header.Get("Range") if rangeHeader != "" { var start int64 fmt.Sscanf(rangeHeader, "bytes=%d-", &start) w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent))) w.WriteHeader(http.StatusPartialContent) w.Write([]byte(testContent[start:])) return } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent))) w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) defer server.Close() parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("Failed to parse server URL: %v", err) } client, err := NewClient(DefaultConfig()) if err != nil { t.Fatalf("Failed to create client: %v", err) } // Create resume info simulating a partially downloaded file resumeInfo := &core.ResumeInfo{ DownloadedBytes: 10, ETag: "\"test-etag-123\"", URL: server.URL, } req := &core.DownloadRequest{ URL: parsedURL, Output: "", Writer: &testWriter{}, Resume: true, ResumeInfo: resumeInfo, Verbose: false, Headers: make(map[string]string), } result, err := client.Download(context.Background(), req) if err != nil { t.Fatalf("Download failed: %v", err) } if result.BytesDownloaded <= 0 { t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded) } if result.Resumed { t.Log("Download was resumed successfully") } } func TestUploadIntegration(t *testing.T) { var receivedBody string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read body", http.StatusInternalServerError) return } receivedBody = string(body) w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) })) defer server.Close() client, err := NewClient(DefaultConfig()) if err != nil { t.Fatalf("Failed to create client: %v", err) } // Test POST upload with form data formReq := &UploadRequest{ URL: server.URL, Method: "POST", FormData: map[string]string{"key": "value"}, Timeout: 0, } result, err := client.Upload(context.Background(), formReq) if err != nil { t.Fatalf("Upload failed: %v", err) } if result.StatusCode != http.StatusOK { t.Errorf("Expected status 200, got %d", result.StatusCode) } if result.BytesUploaded <= 0 { t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded) } if !strings.Contains(receivedBody, "key=value") { t.Errorf("Expected body to contain form data, got: %s", receivedBody) } } func TestDownloadFollowsRedirectIntegration(t *testing.T) { const finalBody = "redirected content" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/redirect" { http.Redirect(w, r, "/target", http.StatusFound) return } w.WriteHeader(http.StatusOK) w.Write([]byte(finalBody)) })) defer server.Close() parsedURL, err := url.Parse(server.URL + "/redirect") if err != nil { t.Fatalf("parse url: %v", err) } client, err := NewClient(DefaultConfig()) if err != nil { t.Fatalf("create client: %v", err) } result, err := client.Download(context.Background(), &core.DownloadRequest{ URL: parsedURL, Writer: &testWriter{}, Headers: map[string]string{}, }) if err != nil { t.Fatalf("download: %v", err) } if result.BytesDownloaded != int64(len(finalBody)) { t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody)) } } // TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a // server that always answers 500. It verifies the loop actually attempts the // configured number of retries and that each attempt's response body is // closed — guarding against the regression where http.Client.Do returned a // non-nil resp alongside an error and the body was never closed, exhausting // the connection pool. func TestRetryAllErrorsClosesBody(t *testing.T) { var requestCount int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&requestCount, 1) w.WriteHeader(http.StatusInternalServerError) // Write a non-empty body so a leaking response body would matter // (an empty body is closed trivially by net/http's chunked reader). _, _ = w.Write([]byte("server error")) })) defer server.Close() parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("parse url: %v", err) } cfg := DefaultConfig() cfg.Transport.RetryAllErrors = true cfg.Transport.MaxRetries = 3 cfg.Transport.Retry = &transport.RetryConfig{ MaxRetries: 3, InitialDelay: time.Millisecond, } client, err := NewClient(cfg) if err != nil { t.Fatalf("create client: %v", err) } _, err = client.Download(context.Background(), &core.DownloadRequest{ URL: parsedURL, Writer: &testWriter{}, Headers: map[string]string{}, }) if err == nil { t.Fatal("expected error after exhausting retries on 500 responses") } // We expect at least 1 (initial transport.Retry attempt) + MaxRetries // (RetryAllErrors loop). The exact number may be higher because Go's // HTTP transport may also retry once on its own when an attempt fails // (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert // the lower bound only — the goal is to confirm the retry loop runs and // that every failed response body is closed, not to pin a Go-internal // detail. minRequests := int32(1 + cfg.Transport.MaxRetries) if got := atomic.LoadInt32(&requestCount); got < minRequests { t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries) } } // TestUploadRateLimiterAndProgressAreChained is a regression guard for the // BACKLOG entry "Rate limiter silently dropped when progress wrapper // enabled". The two wrappers must compose — httpReq.Body must end up // reading through the rate-limited reader, with progress reported on top. func TestUploadRateLimiterAndProgressAreChained(t *testing.T) { const payload = "the quick brown fox jumps over the lazy dog" var receivedBody string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "read error", http.StatusInternalServerError) return } receivedBody = string(body) w.WriteHeader(http.StatusOK) })) defer server.Close() // Write payload to a temp file (Upload's PUT path opens the file). tmpDir := t.TempDir() tmpFile := filepath.Join(tmpDir, "payload.txt") if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil { t.Fatalf("write tmp: %v", err) } cfg := DefaultConfig() // Generous rate limit so the test isn't dominated by the limiter. // limitedReader.Read asks for len(p) tokens on every call, and the // HTTP transport uses a 32 KB buffer, so burst must comfortably exceed // that or Allow() blocks forever. cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024) cfg.Transport.MaxRetries = 0 cfg.Transport.Retry = &transport.RetryConfig{ MaxRetries: 0, InitialDelay: time.Millisecond, } client, err := NewClient(cfg) if err != nil { t.Fatalf("create client: %v", err) } progressCalls := 0 var lastCurrent int64 progressCb := func(current, total int64, _ float64) { progressCalls++ if current < lastCurrent { t.Errorf("progress went backwards: %d -> %d", lastCurrent, current) } lastCurrent = current } _, err = client.Upload(context.Background(), &UploadRequest{ URL: server.URL, Method: "PUT", FilePath: tmpFile, ProgressCallback: progressCb, }) if err != nil { t.Fatalf("upload: %v", err) } // The server must receive the full, unmodified payload. If the rate // limiter wrapper was silently dropped (e.g. httpReq.Body not updated // after body was reassigned), the server would still get the data via // the original reader — so this check alone doesn't expose the bug. if receivedBody != payload { t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload) } // Progress callback must fire at least once and reach the full payload // length. The exact call count depends on the read-buffer behaviour of // net/http (a 43-byte payload may be read in a single Read), but a count // of zero would prove the progress reader was bypassed — which is the // exact regression the BACKLOG entry warns about. if progressCalls < 1 { t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls) } if lastCurrent != int64(len(payload)) { t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload)) } } // TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few // 500s short-circuits the retry loop and produces a successful download — // another code path that must close every preceding failed response body // before jumping to okStatus. func TestRetryAllErrorsRecoversOn200(t *testing.T) { const finalBody = "ok after retries" var requestCount int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { n := atomic.AddInt32(&requestCount, 1) if n < 3 { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("server error")) return } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody))) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(finalBody)) })) defer server.Close() parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("parse url: %v", err) } cfg := DefaultConfig() cfg.Transport.RetryAllErrors = true cfg.Transport.MaxRetries = 5 cfg.Transport.Retry = &transport.RetryConfig{ MaxRetries: 5, InitialDelay: time.Millisecond, } client, err := NewClient(cfg) if err != nil { t.Fatalf("create client: %v", err) } result, err := client.Download(context.Background(), &core.DownloadRequest{ URL: parsedURL, Writer: &testWriter{}, Headers: map[string]string{}, }) if err != nil { t.Fatalf("download: %v", err) } if result.BytesDownloaded != int64(len(finalBody)) { t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody)) } if got := atomic.LoadInt32(&requestCount); got != 3 { t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got) } } // TestTraceTimeTTFBOnReusedConnection is a regression guard for the // BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on // reused connections". The previous implementation measured TTFB as // time.Since(dnsStart); on a pooled HTTP connection the DNSStart // callback is silent, dnsStart stayed zero, and the result was ~55 years // instead of a plausible request-time duration. The fix anchors TTFB to // a requestStart timestamp captured before the trace is attached, so // the value is meaningful whether or not the connection is reused. func TestTraceTimeTTFBOnReusedConnection(t *testing.T) { var connectionCount int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&connectionCount, 1) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("hello")) })) defer server.Close() parsedURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("Failed to parse server URL: %v", err) } cfg := DefaultConfig() cfg.Output.TraceTime = true client, err := NewClient(cfg) if err != nil { t.Fatalf("Failed to create client: %v", err) } reqTemplate := func() *core.DownloadRequest { return &core.DownloadRequest{ URL: parsedURL, Writer: &testWriter{}, Headers: make(map[string]string), } } // First request opens the connection — DNSStart/ConnectStart/ // TLSHandshakeStart all fire, populating the trace timestamps. if _, err := client.Download(context.Background(), reqTemplate()); err != nil { t.Fatalf("first download failed: %v", err) } // Second request should be served from the pool. Whether or not // keep-alive actually reuses the connection in this test depends // on the transport's MaxIdleConnsPerHost, but the assertion is the // same: TTFB must be a plausible duration in either case. result, err := client.Download(context.Background(), reqTemplate()) if err != nil { t.Fatalf("second download failed: %v", err) } if result.TraceTimings == nil { t.Fatal("TraceTimings is nil despite TraceTime=true") } // 1 hour is a generous upper bound that flags the original ~55-year // regression (time.Since(time.Time{}) on a fresh time.Time) without // being flaky on a busy CI runner. if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour { t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb) } if ttfb := result.TraceTimings.TTFB; ttfb < 0 { t.Errorf("TTFB = %v; want non-negative", ttfb) } }