goget/internal/protocol/http/integration_test.go

//go:build linux || freebsd
// +build linux freebsd

package http

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"codeberg.org/petrbalvin/goget/internal/core"
	"codeberg.org/petrbalvin/goget/internal/transport"
)

// testWriter implements io.Writer for testing
type testWriter struct {
	buf strings.Builder
}

func (w *testWriter) Write(p []byte) (int, error) {
	return w.buf.Write(p)
}

func TestDownloadSequentialIntegration(t *testing.T) {
	// Create a test server that returns known content
	testContent := "Hello, this is test content for download integration testing!"
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(testContent))
	}))
	defer server.Close()

	// Parse the server URL
	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("Failed to parse server URL: %v", err)
	}

	// Create the HTTP client
	client, err := NewClient(DefaultConfig())
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	// Create download request
	req := &core.DownloadRequest{
		URL:     parsedURL,
		Output:  "",
		Writer:  &testWriter{},
		Verbose: false,
		Headers: make(map[string]string),
	}

	// Execute download
	result, err := client.Download(context.Background(), req)
	if err != nil {
		t.Fatalf("Download failed: %v", err)
	}

	if result.BytesDownloaded <= 0 {
		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
	}

	if result.BytesDownloaded != int64(len(testContent)) {
		t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded)
	}
}

func TestDownloadParallelIntegration(t *testing.T) {
	// Create test content for parallel download
	testContent := strings.Repeat("B", 1024*1024) // 1MB

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Support range requests
		w.Header().Set("Accept-Ranges", "bytes")
		w.Header().Set("Content-Type", "application/octet-stream")

		rangeHeader := r.Header.Get("Range")
		if rangeHeader != "" {
			var start, end int64
			n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
			if n >= 1 {
				if end == 0 || end >= int64(len(testContent)) {
					end = int64(len(testContent)) - 1
				}
				w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent)))
				w.WriteHeader(http.StatusPartialContent)
				w.Write([]byte(testContent[start : end+1]))
				return
			}
		}

		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(testContent))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("Failed to parse server URL: %v", err)
	}

	cfg := DefaultConfig()
	cfg.Parallel = &core.ParallelConfig{
		Connections:  4,
		MinSize:      100, // Force parallel for small files
		MinChunkSize: 100,
		MaxChunkSize: 1024 * 1024,
	}

	client, err := NewClient(cfg)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	req := &core.DownloadRequest{
		URL:      parsedURL,
		Output:   "",
		Writer:   &testWriter{},
		Verbose:  false,
		Headers:  make(map[string]string),
		Parallel: cfg.Parallel,
	}

	result, err := client.Download(context.Background(), req)
	if err != nil {
		t.Fatalf("Download failed: %v", err)
	}

	if result.BytesDownloaded <= 0 {
		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
	}

	if !result.Parallel {
		t.Log("Download was not parallel (may be expected depending on server negotiation)")
	}
}

func TestDownloadResumeIntegration(t *testing.T) {
	testContent := "This is content for resume testing!"
	var requestCount int

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requestCount++

		w.Header().Set("Accept-Ranges", "bytes")
		w.Header().Set("ETag", "\"test-etag-123\"")

		rangeHeader := r.Header.Get("Range")
		if rangeHeader != "" {
			var start int64
			fmt.Sscanf(rangeHeader, "bytes=%d-", &start)
			w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent)))
			w.WriteHeader(http.StatusPartialContent)
			w.Write([]byte(testContent[start:]))
			return
		}

		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(testContent))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("Failed to parse server URL: %v", err)
	}

	client, err := NewClient(DefaultConfig())
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	// Create resume info simulating a partially downloaded file
	resumeInfo := &core.ResumeInfo{
		DownloadedBytes: 10,
		ETag:            "\"test-etag-123\"",
		URL:             server.URL,
	}

	req := &core.DownloadRequest{
		URL:        parsedURL,
		Output:     "",
		Writer:     &testWriter{},
		Resume:     true,
		ResumeInfo: resumeInfo,
		Verbose:    false,
		Headers:    make(map[string]string),
	}

	result, err := client.Download(context.Background(), req)
	if err != nil {
		t.Fatalf("Download failed: %v", err)
	}

	if result.BytesDownloaded <= 0 {
		t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
	}

	if result.Resumed {
		t.Log("Download was resumed successfully")
	}
}

func TestUploadIntegration(t *testing.T) {
	var receivedBody string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, err := io.ReadAll(r.Body)
		if err != nil {
			http.Error(w, "Failed to read body", http.StatusInternalServerError)
			return
		}
		receivedBody = string(body)
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("OK"))
	}))
	defer server.Close()

	client, err := NewClient(DefaultConfig())
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	// Test POST upload with form data
	formReq := &UploadRequest{
		URL:      server.URL,
		Method:   "POST",
		FormData: map[string]string{"key": "value"},
		Timeout:  0,
	}

	result, err := client.Upload(context.Background(), formReq)
	if err != nil {
		t.Fatalf("Upload failed: %v", err)
	}

	if result.StatusCode != http.StatusOK {
		t.Errorf("Expected status 200, got %d", result.StatusCode)
	}

	if result.BytesUploaded <= 0 {
		t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded)
	}

	if !strings.Contains(receivedBody, "key=value") {
		t.Errorf("Expected body to contain form data, got: %s", receivedBody)
	}
}

func TestDownloadFollowsRedirectIntegration(t *testing.T) {
	const finalBody = "redirected content"
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/redirect" {
			http.Redirect(w, r, "/target", http.StatusFound)
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(finalBody))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL + "/redirect")
	if err != nil {
		t.Fatalf("parse url: %v", err)
	}

	client, err := NewClient(DefaultConfig())
	if err != nil {
		t.Fatalf("create client: %v", err)
	}

	result, err := client.Download(context.Background(), &core.DownloadRequest{
		URL:     parsedURL,
		Writer:  &testWriter{},
		Headers: map[string]string{},
	})
	if err != nil {
		t.Fatalf("download: %v", err)
	}
	if result.BytesDownloaded != int64(len(finalBody)) {
		t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
	}
}

// TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a
// server that always answers 500. It verifies the loop actually attempts the
// configured number of retries and that each attempt's response body is
// closed — guarding against the regression where http.Client.Do returned a
// non-nil resp alongside an error and the body was never closed, exhausting
// the connection pool.
func TestRetryAllErrorsClosesBody(t *testing.T) {
	var requestCount int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		atomic.AddInt32(&requestCount, 1)
		w.WriteHeader(http.StatusInternalServerError)
		// Write a non-empty body so a leaking response body would matter
		// (an empty body is closed trivially by net/http's chunked reader).
		_, _ = w.Write([]byte("server error"))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("parse url: %v", err)
	}

	cfg := DefaultConfig()
	cfg.Transport.RetryAllErrors = true
	cfg.Transport.MaxRetries = 3
	cfg.Transport.Retry = &transport.RetryConfig{
		MaxRetries:   3,
		InitialDelay: time.Millisecond,
	}

	client, err := NewClient(cfg)
	if err != nil {
		t.Fatalf("create client: %v", err)
	}

	_, err = client.Download(context.Background(), &core.DownloadRequest{
		URL:     parsedURL,
		Writer:  &testWriter{},
		Headers: map[string]string{},
	})
	if err == nil {
		t.Fatal("expected error after exhausting retries on 500 responses")
	}

	// We expect at least 1 (initial transport.Retry attempt) + MaxRetries
	// (RetryAllErrors loop). The exact number may be higher because Go's
	// HTTP transport may also retry once on its own when an attempt fails
	// (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert
	// the lower bound only — the goal is to confirm the retry loop runs and
	// that every failed response body is closed, not to pin a Go-internal
	// detail.
	minRequests := int32(1 + cfg.Transport.MaxRetries)
	if got := atomic.LoadInt32(&requestCount); got < minRequests {
		t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries)
	}
}

// TestUploadRateLimiterAndProgressAreChained is a regression guard for the
// BACKLOG entry "Rate limiter silently dropped when progress wrapper
// enabled". The two wrappers must compose — httpReq.Body must end up
// reading through the rate-limited reader, with progress reported on top.
func TestUploadRateLimiterAndProgressAreChained(t *testing.T) {
	const payload = "the quick brown fox jumps over the lazy dog"

	var receivedBody string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, err := io.ReadAll(r.Body)
		if err != nil {
			http.Error(w, "read error", http.StatusInternalServerError)
			return
		}
		receivedBody = string(body)
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	// Write payload to a temp file (Upload's PUT path opens the file).
	tmpDir := t.TempDir()
	tmpFile := filepath.Join(tmpDir, "payload.txt")
	if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil {
		t.Fatalf("write tmp: %v", err)
	}

	cfg := DefaultConfig()
	// Generous rate limit so the test isn't dominated by the limiter.
	// limitedReader.Read asks for len(p) tokens on every call, and the
	// HTTP transport uses a 32 KB buffer, so burst must comfortably exceed
	// that or Allow() blocks forever.
	cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024)
	cfg.Transport.MaxRetries = 0
	cfg.Transport.Retry = &transport.RetryConfig{
		MaxRetries:   0,
		InitialDelay: time.Millisecond,
	}

	client, err := NewClient(cfg)
	if err != nil {
		t.Fatalf("create client: %v", err)
	}

	progressCalls := 0
	var lastCurrent int64
	progressCb := func(current, total int64, _ float64) {
		progressCalls++
		if current < lastCurrent {
			t.Errorf("progress went backwards: %d -> %d", lastCurrent, current)
		}
		lastCurrent = current
	}

	_, err = client.Upload(context.Background(), &UploadRequest{
		URL:              server.URL,
		Method:           "PUT",
		FilePath:         tmpFile,
		ProgressCallback: progressCb,
	})
	if err != nil {
		t.Fatalf("upload: %v", err)
	}

	// The server must receive the full, unmodified payload. If the rate
	// limiter wrapper was silently dropped (e.g. httpReq.Body not updated
	// after body was reassigned), the server would still get the data via
	// the original reader — so this check alone doesn't expose the bug.
	if receivedBody != payload {
		t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload)
	}

	// Progress callback must fire at least once and reach the full payload
	// length. The exact call count depends on the read-buffer behaviour of
	// net/http (a 43-byte payload may be read in a single Read), but a count
	// of zero would prove the progress reader was bypassed — which is the
	// exact regression the BACKLOG entry warns about.
	if progressCalls < 1 {
		t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls)
	}
	if lastCurrent != int64(len(payload)) {
		t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload))
	}
}

// TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few
// 500s short-circuits the retry loop and produces a successful download —
// another code path that must close every preceding failed response body
// before jumping to okStatus.
func TestRetryAllErrorsRecoversOn200(t *testing.T) {
	const finalBody = "ok after retries"
	var requestCount int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		n := atomic.AddInt32(&requestCount, 1)
		if n < 3 {
			w.WriteHeader(http.StatusInternalServerError)
			_, _ = w.Write([]byte("server error"))
			return
		}
		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody)))
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte(finalBody))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("parse url: %v", err)
	}

	cfg := DefaultConfig()
	cfg.Transport.RetryAllErrors = true
	cfg.Transport.MaxRetries = 5
	cfg.Transport.Retry = &transport.RetryConfig{
		MaxRetries:   5,
		InitialDelay: time.Millisecond,
	}

	client, err := NewClient(cfg)
	if err != nil {
		t.Fatalf("create client: %v", err)
	}

	result, err := client.Download(context.Background(), &core.DownloadRequest{
		URL:     parsedURL,
		Writer:  &testWriter{},
		Headers: map[string]string{},
	})
	if err != nil {
		t.Fatalf("download: %v", err)
	}
	if result.BytesDownloaded != int64(len(finalBody)) {
		t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
	}
	if got := atomic.LoadInt32(&requestCount); got != 3 {
		t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got)
	}
}

// TestTraceTimeTTFBOnReusedConnection is a regression guard for the
// BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on
// reused connections". The previous implementation measured TTFB as
// time.Since(dnsStart); on a pooled HTTP connection the DNSStart
// callback is silent, dnsStart stayed zero, and the result was ~55 years
// instead of a plausible request-time duration. The fix anchors TTFB to
// a requestStart timestamp captured before the trace is attached, so
// the value is meaningful whether or not the connection is reused.
func TestTraceTimeTTFBOnReusedConnection(t *testing.T) {
	var connectionCount int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		atomic.AddInt32(&connectionCount, 1)
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("hello"))
	}))
	defer server.Close()

	parsedURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("Failed to parse server URL: %v", err)
	}

	cfg := DefaultConfig()
	cfg.Output.TraceTime = true
	client, err := NewClient(cfg)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	reqTemplate := func() *core.DownloadRequest {
		return &core.DownloadRequest{
			URL:     parsedURL,
			Writer:  &testWriter{},
			Headers: make(map[string]string),
		}
	}

	// First request opens the connection — DNSStart/ConnectStart/
	// TLSHandshakeStart all fire, populating the trace timestamps.
	if _, err := client.Download(context.Background(), reqTemplate()); err != nil {
		t.Fatalf("first download failed: %v", err)
	}

	// Second request should be served from the pool. Whether or not
	// keep-alive actually reuses the connection in this test depends
	// on the transport's MaxIdleConnsPerHost, but the assertion is the
	// same: TTFB must be a plausible duration in either case.
	result, err := client.Download(context.Background(), reqTemplate())
	if err != nil {
		t.Fatalf("second download failed: %v", err)
	}

	if result.TraceTimings == nil {
		t.Fatal("TraceTimings is nil despite TraceTime=true")
	}

	// 1 hour is a generous upper bound that flags the original ~55-year
	// regression (time.Since(time.Time{}) on a fresh time.Time) without
	// being flaky on a busy CI runner.
	if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour {
		t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb)
	}
	if ttfb := result.TraceTimings.TTFB; ttfb < 0 {
		t.Errorf("TTFB = %v; want non-negative", ttfb)
	}
}