goget/internal/protocol/http/client.go

//go:build linux || freebsd
// +build linux freebsd

package http

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"codeberg.org/petrbalvin/goget/internal/archive"
	"codeberg.org/petrbalvin/goget/internal/auth"
	"codeberg.org/petrbalvin/goget/internal/compression"
	"codeberg.org/petrbalvin/goget/internal/cookie"
	"codeberg.org/petrbalvin/goget/internal/core"
	format "codeberg.org/petrbalvin/goget/internal/format"
	"codeberg.org/petrbalvin/goget/internal/hsts"
	"codeberg.org/petrbalvin/goget/internal/mirror"
	"codeberg.org/petrbalvin/goget/internal/output"
	"codeberg.org/petrbalvin/goget/internal/protocol"
	"codeberg.org/petrbalvin/goget/internal/recursive"
	"codeberg.org/petrbalvin/goget/internal/transport"
)

type Client struct {
	protocol.BaseProtocol
	httpClient *http.Client
	transport  *http.Transport
	dialer     *transport.Dialer
	config     *Config
}

type Config struct {
	Transport TransportConfig
	Auth      AuthConfig
	Recursive RecursiveConfig
	Output    OutputConfig

	// HTTP request construction
	UserAgent       string
	Headers         map[string]string
	CustomHeaders   map[string]string
	PostData        string
	PostFile        string
	FormData        []string
	Range           string
	Compressed      bool
	Referer         string
	FailOnHTTPError bool
	Spider          bool

	// Download mechanics
	Parallel    *core.ParallelConfig
	RateLimiter *transport.TokenBucket
	RateLimit   int64

	// Dependencies
	CookieJar *cookie.Jar
}

func DefaultConfig() *Config {
	return &Config{
		Transport: TransportConfig{
			Timeout:         30 * time.Minute,
			BufferSize:      32 * 1024,
			FollowRedirects: true,
			MaxRedirects:    10,
			TLS:             transport.DefaultTLSConfig(),
			Dialer:          transport.DefaultDialerConfig(),
			Retry:           transport.DefaultRetryConfig(),
		},
		Auth: AuthConfig{
			AuthType: "auto",
		},
		Recursive: RecursiveConfig{
			MaxDepth:        3,
			ExcludePatterns: []string{},
			MirrorAssets:    true,
			ConvertLinks:    true,
			RespectRobots:   true,
		},
		Output: OutputConfig{
			OutputDir: ".",
		},
		UserAgent: core.Name + "/" + core.Version,
		Headers:   make(map[string]string),
		Parallel:  core.DefaultParallelConfig(),
	}
}

func NewClient(cfg *Config) (*Client, error) {
	if cfg == nil {
		cfg = DefaultConfig()
	}

	if cfg.Transport.Dialer == nil {
		cfg.Transport.Dialer = transport.DefaultDialerConfig()
	}
	if cfg.Transport.BindInterface != "" {
		cfg.Transport.Dialer.BindInterface = cfg.Transport.BindInterface
	}
	if cfg.Transport.BlockPrivateIPs {
		cfg.Transport.Dialer.BlockPrivateIPs = true
	}

	dialer := transport.NewDialer(cfg.Transport.Dialer)

	if len(cfg.Transport.DNSServers) > 0 {
		dialer.SetCustomDNS(cfg.Transport.DNSServers)
	}

	tlsCfg, err := cfg.Transport.TLS.ToTLSConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to create tls config: %w", err)
	}

	httpTransport := &http.Transport{
		DialContext:         dialer.DialContext,
		TLSClientConfig:     tlsCfg,
		Proxy:               http.ProxyFromEnvironment,
		DisableCompression:  true,
		MaxIdleConns:        100,
		MaxIdleConnsPerHost: 10,
		IdleConnTimeout:     90 * time.Second,
	}

	if cfg.Transport.Proxy != nil {
		proxyTransport, err := transport.NewProxyTransport(cfg.Transport.Proxy, httpTransport)
		if err != nil {
			return nil, fmt.Errorf("failed to configure proxy: %w", err)
		}
		httpTransport = proxyTransport
	}

	httpTransport.ForceAttemptHTTP2 = true

	httpClient := &http.Client{
		Transport: httpTransport,
		Timeout:   cfg.Transport.Timeout,
	}

	if cfg.CookieJar != nil {
		httpClient.Jar = cfg.CookieJar
	}

	if !cfg.Transport.FollowRedirects {
		httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
	} else {
		maxRedirects := cfg.Transport.MaxRedirects
		if maxRedirects <= 0 {
			maxRedirects = 50 // unlimited redirects, capped for safety
		}
		httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			if len(via) >= maxRedirects {
				return fmt.Errorf("stopped after %d redirects", maxRedirects)
			}
			return nil
		}
	}

	if cfg.Transport.MaxRetries > 0 {
		cfg.Transport.Retry.MaxRetries = cfg.Transport.MaxRetries
	}
	if cfg.Transport.RetryDelay > 0 {
		cfg.Transport.Retry.InitialDelay = cfg.Transport.RetryDelay
	}

	client := &Client{
		BaseProtocol: *protocol.NewBaseProtocol(protocol.ProtocolInfo{
			Name:        "HTTP/HTTPS",
			Scheme:      "http",
			Version:     "1.1/2.0",
			Operations:  []string{"download", "upload", "head", "recursive"},
			Features:    []string{"resume", "compression", "redirects", "tls", "parallel", "auth", "cookies", "recursive", "extract"},
			DefaultPort: 80,
		}),
		httpClient: httpClient,
		transport:  httpTransport,
		dialer:     dialer,
		config:     cfg,
	}

	return client, nil
}

func (c *Client) Scheme() string { return "http" }
func (c *Client) CanHandle(u *url.URL) bool {
	return u != nil && (u.Scheme == "http" || u.Scheme == "https")
}
func (c *Client) Capabilities() []string {
	return []string{"download", "resume", "compression", "redirects", "tls", "proxy", "parallel", "auth", "cookies", "recursive", "extract"}
}
func (c *Client) SupportsResume() bool          { return true }
func (c *Client) SupportsCompression() bool     { return true }
func (c *Client) SupportsParallel() bool        { return true }
func (c *Client) SupportsRecursive() bool       { return true }
func (c *Client) SetHeader(key, value string)   { c.config.Headers[key] = value } // unchanged - flat field
func (c *Client) GetTransport() *http.Transport { return c.transport }

// SetHSTSCache sets the HSTS cache for automatic HTTPS upgrades.
func (c *Client) SetHSTSCache(cache *hsts.Cache) {
	c.config.Transport.HSTSCache = cache
}

func (c *Client) updateHSTS(resp *http.Response) {
	if c.config.Transport.HSTSCache != nil && resp != nil {
		c.config.Transport.HSTSCache.Update(resp)
	}
}

func (c *Client) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
	if !c.CanHandle(req.URL) {
		return nil, core.NewProtocolError("url not supported by http protocol", nil, req.URL.String())
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[http] Download() called: url=%s, output=%s, recursive=%v, mirror=%v, extract=%v\n",
			req.URL.String(), req.Output, req.Recursive, c.config.Recursive.Mirror, c.config.Output.AutoExtract)
	}

	// HSTS: upgrade http to https if host is known
	if c.config.Transport.HSTSCache != nil {
		if c.config.Transport.HSTSCache.Apply(req.URL) && c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[hsts] upgraded to %s\n", req.URL.String())
		}
	}

	// Mirror mode takes precedence
	if c.config.Recursive.Mirror {
		return c.downloadMirror(ctx, req)
	}

	if req.Recursive {
		return c.downloadRecursive(ctx, req)
	}

	result, err := c.downloadSingle(ctx, req)
	if err != nil {
		return nil, err
	}

	// Auto-extract after successful download
	if c.config.Output.AutoExtract && req.Output != "" && req.Output != "-" && archive.IsArchiveFormat(req.Output) {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[auto-extract] Detected archive: %s\n", req.Output)
		}

		extractDir := c.config.Output.ExtractDir
		if extractDir == "" {
			extractDir = req.Output + ".extracted"
		}

		if err := os.MkdirAll(extractDir, 0755); err != nil {
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[auto-extract] Warning: failed to create extract dir: %v\n", err)
			}
		} else {
			// Open the downloaded file for extraction
			srcFile, err := os.Open(req.Output)
			if err != nil {
				if c.config.Output.Verbose {
					fmt.Fprintf(os.Stderr, "[auto-extract] Warning: failed to open archive: %v\n", err)
				}
			} else {
				// Auto-detect format and extract
				if err := archive.AutoExtract(req.Output, srcFile, extractDir); err != nil {
					if c.config.Output.Verbose {
						fmt.Fprintf(os.Stderr, "[auto-extract] Warning: failed to extract: %v\n", err)
					}
				} else if c.config.Output.Verbose {
					fmt.Fprintf(os.Stderr, "[auto-extract] Extracted to: %s\n", extractDir)
					// List extracted files for verification
					if entries, err := os.ReadDir(extractDir); err == nil {
						for _, entry := range entries {
							fmt.Fprintf(os.Stderr, "[auto-extract]   - %s\n", entry.Name())
						}
					}
				}
				srcFile.Close()
			}
		}
	}

	return result, nil
}

func (c *Client) downloadRecursive(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
	startTime := time.Now()

	outputDir := c.config.Output.OutputDir
	if outputDir == "" {
		outputDir = "."
	}
	if req.Output != "" && req.Output != "-" && !req.Recursive {
		outputDir = filepath.Dir(req.Output)
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[recursive] outputDir=%s, url=%s\n", outputDir, req.URL.String())
	}

	// --recursive-parallel takes precedence over --parallel when set; both
	// ultimately control the Crawler worker count, but the recursive variant
	// is the user-facing knob for recursive downloads.
	crawlerParallel := c.config.Recursive.Parallel
	if crawlerParallel <= 0 {
		crawlerParallel = c.config.Parallel.Connections
	}

	crawlerCfg := &recursive.CrawlerConfig{
		MaxDepth:        c.config.Recursive.MaxDepth,
		FollowExternal:  c.config.Recursive.FollowExternal,
		ExcludePatterns: c.config.Recursive.ExcludePatterns,
		IncludePatterns: []string{},
		AcceptPatterns:  parseAcceptPatterns(c.config.Recursive.Accept),
		NoParent:        c.config.Recursive.NoParent,
		PageRequisites:  c.config.Recursive.PageRequisites,
		ConvertLinks:    c.config.Recursive.ConvertLinks,
		OutputDir:       outputDir,
		Verbose:         c.config.Output.Verbose,
		Parallel:        crawlerParallel,
		Delay:           c.config.Recursive.Wait,
		RandomWait:      c.config.Recursive.RandomWait,
		UserAgent:       c.config.UserAgent,
		DryRun:          c.config.Recursive.DryRun,
	}
	if c.config.Recursive.Wait == 0 {
		crawlerCfg.Delay = 100 * time.Millisecond // default
	}

	crawler := recursive.NewCrawler(crawlerCfg, c)

	stats, err := crawler.Download(ctx, req.URL)
	if err != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[recursive] error: %v\n", err)
		}
		return nil, core.NewNetworkError("recursive download failed", err, req.URL.String())
	}

	duration := time.Since(startTime)

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[recursive] done: %d files, %s total\n",
			stats.DownloadedURLs, format.Bytes(stats.TotalBytes))
	}

	result := &core.DownloadResult{
		BytesDownloaded: stats.TotalBytes,
		TotalSize:       stats.TotalBytes,
		Duration:        duration,
		Protocol:        "HTTP/HTTPS (recursive)",
		IPVersion:       6,
		Speed:           float64(stats.TotalBytes) / duration.Seconds(),
		OutputPath:      outputDir,
		Resumed:         false,
		Parallel:        true,
		ChunksCount:     stats.DownloadedURLs,
	}

	return result, nil
}

func (c *Client) downloadMirror(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
	startTime := time.Now()

	outputDir := c.config.Output.OutputDir
	if outputDir == "" {
		outputDir = "."
	}
	if req.Output != "" && req.Output != "-" {
		outputDir = filepath.Dir(req.Output)
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[mirror] outputDir=%s, url=%s, convertLinks=%v, downloadAssets=%v\n",
			outputDir, req.URL.String(), c.config.Recursive.ConvertLinks, c.config.Recursive.MirrorAssets)
	}

	mirrorCfg := &mirror.MirrorConfig{
		BaseURL:         req.URL,
		OutputDir:       outputDir,
		MaxDepth:        0, // 0 = infinite for mirror mode
		FollowExternal:  false,
		ExcludePatterns: c.config.Recursive.ExcludePatterns,
		IncludePatterns: []string{},
		Verbose:         c.config.Output.Verbose,
		Parallel:        8, // More workers for mirror mode
		Delay:           50 * time.Millisecond,
		UserAgent:       c.config.UserAgent,
		ConvertLinks:    c.config.Recursive.ConvertLinks,
		DownloadAssets:  c.config.Recursive.MirrorAssets,
		PruneEmpty:      true,
		BackupConverted: c.config.Recursive.BackupConverted,
		RestrictFiles:   true,
		Timeout:         c.config.Transport.Timeout,
		RespectRobots:   c.config.Recursive.RespectRobots,
		DryRun:          c.config.Recursive.DryRun,
	}

	// Add rate limiter if configured
	if c.config.RateLimit > 0 {
		mirrorCfg.RateLimiter = transport.NewTokenBucket(c.config.RateLimit, c.config.RateLimit)
	}

	m := mirror.NewMirror(mirrorCfg, c)

	stats, err := m.Download(ctx)
	if err != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[mirror] error: %v\n", err)
		}
		return nil, core.NewNetworkError("mirror download failed", err, req.URL.String())
	}

	duration := time.Since(startTime)

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[mirror] done: %d files, %s total, %d links converted\n",
			stats.DownloadedURLs, format.Bytes(stats.TotalBytes), stats.ConvertedLinks)
	}

	result := &core.DownloadResult{
		BytesDownloaded: stats.TotalBytes,
		TotalSize:       stats.TotalBytes,
		Duration:        duration,
		Protocol:        "HTTP/HTTPS (mirror)",
		IPVersion:       6,
		Speed:           float64(stats.TotalBytes) / duration.Seconds(),
		OutputPath:      outputDir,
		Resumed:         false,
		Parallel:        true,
		ChunksCount:     int(stats.DownloadedURLs),
	}

	return result, nil
}

func (c *Client) downloadSingle(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
	startTime := time.Now()

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[single] downloadSingle() start: url=%s, output=%s\n",
			req.URL.String(), req.Output)
	}

	username, password := auth.GetAuthForURL("", "", c.config.Auth.Username, c.config.Auth.Password, req.URL)

	headReq, err := http.NewRequestWithContext(ctx, "HEAD", req.URL.String(), nil)
	if err != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[single] HEAD request error: %v\n", err)
		}
		return nil, core.NewNetworkError("failed to create HEAD request", err, req.URL.String())
	}
	headReq.Header.Set("User-Agent", c.config.UserAgent)
	if username != "" {
		auth.BasicAuth(headReq, username, password)
	}

	headResp, err := c.httpClient.Do(headReq)
	if err != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[single] HEAD request failed: %v\n", err)
		}
	} else {
		defer headResp.Body.Close()
		c.updateHSTS(headResp)
	}

	totalSize := int64(-1)
	supportsRange := false
	etag := ""
	lastModified := ""

	if headResp != nil && headResp.StatusCode == http.StatusOK {
		totalSize = headResp.ContentLength
		supportsRange = strings.ToLower(headResp.Header.Get("Accept-Ranges")) == "bytes"
		etag = headResp.Header.Get("ETag")
		lastModified = headResp.Header.Get("Last-Modified")

		// Content-Disposition: use server-provided filename
		if c.config.Output.ContentDisposition && req.Output == "" {
			if cd := headResp.Header.Get("Content-Disposition"); cd != "" {
				if filename := ParseContentDisposition(cd); filename != "" {
					safe := SanitizeFilename(filename)
					if safe != "" {
						req.Output = safe
						if c.config.Output.Verbose {
							fmt.Fprintf(os.Stderr, "[single] Content-Disposition: filename=%s\n", safe)
						}
					}
				}
			}
		}

		// Show headers
		if c.config.Output.ShowHeaders {
			fmt.Fprintf(os.Stderr, "\n--- Response Headers ---\n")
			fmt.Fprintf(os.Stderr, "HTTP/%d.%d %d %s\n",
				headResp.ProtoMajor, headResp.ProtoMinor, headResp.StatusCode, headResp.Status)
			for key, values := range headResp.Header {
				for _, value := range values {
					fmt.Fprintf(os.Stderr, "%s: %s\n", key, value)
				}
			}
			fmt.Fprintf(os.Stderr, "Content-Length: %d\n", totalSize)
			fmt.Fprintf(os.Stderr, "------------------------\n")
		}

		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[single] HEAD: size=%d, supportsRange=%v\n", totalSize, supportsRange)
		}
	}

	useParallel := req.Parallel != nil && supportsRange && totalSize > 0 && totalSize >= req.Parallel.MinSize
	connections := 1
	if useParallel {
		connections = req.Parallel.GetConnections(totalSize)
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[single] Using parallel: %d connections\n", connections)
		}
	}

	var resumeChunks map[int]int64
	if req.Resume && req.Parallel != nil && useParallel {
		meta, _ := output.LoadResumeMetadata(req.Output)
		if meta != nil && meta.Downloaded > 0 {
			if len(meta.Chunks) > 0 {
				resumeChunks = meta.Chunks
			} else {
				resumeChunks = make(map[int]int64)
			}
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[single] Resume: %d bytes from metadata (chunks: %d)\n", meta.Downloaded, len(resumeChunks))
			}
		}
	}

	var result *core.DownloadResult
	var downloadErr error

	if useParallel {
		result, downloadErr = c.downloadParallel(ctx, req, totalSize, connections, resumeChunks, username, password)
	} else {
		result, downloadErr = c.downloadSequential(ctx, req, totalSize, etag, lastModified, username, password)
	}

	if downloadErr != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[single] download error: %v\n", downloadErr)
		}
		return nil, downloadErr
	}

	result.Duration = time.Since(startTime)
	result.Speed = float64(result.BytesDownloaded) / result.Duration.Seconds()

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[single] done: %s downloaded, speed=%s/s\n",
			format.Bytes(result.BytesDownloaded), format.Speed(result.Speed))
	}

	return result, nil
}

// buildRequestBody builds the request body from form data or POST data configuration.
// Returns the HTTP method, body reader, content type, and any error.
func (c *Client) buildRequestBody(urlStr string) (method string, body io.Reader, contentType string, err error) {
	method = "GET"
	contentType = ""

	if len(c.config.FormData) > 0 {
		method = "POST"
		var buf bytes.Buffer
		writer := multipart.NewWriter(&buf)

		for _, field := range c.config.FormData {
			parts := strings.SplitN(field, "=", 2)
			name := parts[0]
			value := ""
			if len(parts) > 1 {
				value = parts[1]
			}

			if strings.HasPrefix(value, "@") {
				// File upload: field=@/path/to/file
				filePath := strings.TrimPrefix(value, "@")
				f, err := os.Open(filePath)
				if err != nil {
					return "", nil, "", core.NewFileError(fmt.Sprintf("form file not found: %s", filePath), err)
				}
				part, err := writer.CreateFormFile(name, filepath.Base(filePath))
				if err != nil {
					f.Close()
					return "", nil, "", core.NewNetworkError("failed to create form file field", err, urlStr)
				}
				if _, err := io.Copy(part, f); err != nil {
					f.Close()
					return "", nil, "", core.NewNetworkError("failed to copy form file", err, urlStr)
				}
				f.Close()
			} else {
				// Regular field: field=value
				if err := writer.WriteField(name, value); err != nil {
					return "", nil, "", core.NewNetworkError("failed to write form field", err, urlStr)
				}
			}
		}
		writer.Close()
		body = &buf
		contentType = writer.FormDataContentType()
	} else if c.config.PostData != "" {
		method = "POST"
		body = strings.NewReader(c.config.PostData)
	}

	return method, body, contentType, nil
}

// applyRequestAuth applies authentication to the HTTP request based on the client config.
// It handles OAuth 2.0 tokens (with auto-refresh), Basic auth, and sets the Range header
// for resume.
func (c *Client) applyRequestAuth(ctx context.Context, httpReq *http.Request, username, password string, startOffset int64, etag, lastModified string) error {
	// Determine auth type
	authType := c.config.Auth.AuthType
	if authType == "" {
		authType = "auto"
	}

	// Apply OAuth 2.0 token if configured
	if c.config.Auth.OAuth != nil && c.config.Auth.OAuth.AccessToken != "" {
		oauthClient := auth.NewOAuthClient(c.config.Auth.OAuth)
		if oauthClient.IsTokenValid() {
			oauthClient.ApplyToken(httpReq)
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Applied OAuth 2.0 token\n")
			}
		} else if c.config.Auth.OAuth.TokenURL != "" {
			// Try to request a new token
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] OAuth token expired, requesting new one...\n")
			}
			tokenResp, err := oauthClient.RequestToken(ctx)
			if err == nil {
				oauthClient.UpdateToken(tokenResp)
				oauthClient.ApplyToken(httpReq)
				if c.config.Output.Verbose {
					fmt.Fprintf(os.Stderr, "[seq] Applied new OAuth 2.0 token\n")
				}
			} else if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Failed to request OAuth token: %v\n", err)
			}
		}
	}

	// For auto/basic, try with credentials first
	if username != "" && (authType == "auto" || authType == "basic") {
		auth.BasicAuth(httpReq, username, password)
	}

	if startOffset > 0 {
		httpReq.Header.Set("Range", fmt.Sprintf("bytes=%d-", startOffset))
		if etag != "" {
			httpReq.Header.Set("If-Range", etag)
		} else if lastModified != "" {
			httpReq.Header.Set("If-Range", lastModified)
		}
	}

	return nil
}

// openOutputFile opens the output file for writing, either for append (resume) or create.
// When req.Output is "-" or empty with no Writer, it returns os.Stdout.
func (c *Client) openOutputFile(req *core.DownloadRequest, startOffset int64) (io.Writer, *os.File, error) {
	if req.Writer != nil {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[seq] Using provided Writer\n")
		}
		return req.Writer, nil, nil
	}

	if req.Output != "" && req.Output != "-" {
		var file *os.File
		var err error
		if startOffset > 0 {
			file, err = os.OpenFile(req.Output, os.O_APPEND|os.O_WRONLY, 0644)
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Opening file for append: %s\n", req.Output)
			}
		} else {
			if err := os.MkdirAll(filepath.Dir(req.Output), 0755); err != nil {
				return nil, nil, core.NewFileError("failed to create output directory", err)
			}
			file, err = os.Create(req.Output)
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Creating new file: %s\n", req.Output)
			}
		}
		if err != nil {
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Failed to open/create file: %v\n", err)
			}
			return nil, nil, core.NewFileError("failed to open output file", err)
		}
		return file, file, nil
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[seq] Writing to stdout\n")
	}
	return os.Stdout, nil, nil
}

func (c *Client) downloadSequential(ctx context.Context, req *core.DownloadRequest, totalSize int64, etag, lastModified, username, password string) (*core.DownloadResult, error) {
	startTime := time.Now()
	resumed := false
	startOffset := int64(0)

	if req.Resume && req.Output != "" && req.Output != "-" {
		canResume, meta, err := output.CanResume(req.Output, req.URL.String())
		if err != nil {
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Resume check failed: %v\n", err)
			}
			output.CleanupResumeFiles(req.Output)
		} else if canResume && meta != nil {
			startOffset = meta.Downloaded
			resumed = true
			if c.config.Output.Verbose {
				fmt.Fprintf(os.Stderr, "[seq] Resuming from byte %d\n", startOffset)
			}
		}
	}

	// Max time: apply context deadline
	if c.config.Transport.MaxTime > 0 {
		deadlineCtx, deadlineCancel := context.WithTimeout(ctx, c.config.Transport.MaxTime)
		defer deadlineCancel()
		ctx = deadlineCtx
	}

	// Max file size: check Content-Length from HEAD response
	if c.config.Transport.MaxFileSize > 0 && totalSize > 0 && totalSize > c.config.Transport.MaxFileSize {
		return nil, core.NewNetworkError(
			fmt.Sprintf("file too large: %s (max %s)",
				format.Bytes(totalSize), format.Bytes(c.config.Transport.MaxFileSize)),
			nil, core.SafeURL(req.URL))
	}

	// Build request body from form data or POST data
	method, body, contentType, err := c.buildRequestBody(core.SafeURL(req.URL))
	if err != nil {
		return nil, err
	}

	httpReq, err := http.NewRequestWithContext(ctx, method, req.URL.String(), body)
	if err != nil {
		return nil, core.NewNetworkError("failed to create request", err, core.SafeURL(req.URL))
	}

	httpReq.Header.Set("User-Agent", c.config.UserAgent)
	if c.config.Compressed {
		httpReq.Header.Set("Accept-Encoding", "gzip, deflate")
	} else {
		httpReq.Header.Set("Accept-Encoding", "identity")
	}
	// Referer header
	if c.config.Referer != "" {
		httpReq.Header.Set("Referer", c.config.Referer)
	}
	// Range header (partial download)
	if c.config.Range != "" {
		httpReq.Header.Set("Range", "bytes="+c.config.Range)
	}

	// Set Content-Type for POST data
	if contentType != "" {
		httpReq.Header.Set("Content-Type", contentType)
	} else if method == "POST" {
		if httpReq.Header.Get("Content-Type") == "" {
			httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		}
	}

	// Apply custom headers (-H/--header)
	for key, val := range c.config.CustomHeaders {
		httpReq.Header.Set(key, val)
	}

	// Handle authentication based on AuthType
	authType := c.config.Auth.AuthType
	if authType == "" {
		authType = "auto"
	}

	c.applyRequestAuth(ctx, httpReq, username, password, startOffset, etag, lastModified)

	for key, value := range req.Headers {
		httpReq.Header.Set(key, value)
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[seq] Sending GET request to %s (auth=%s)\n", core.SafeURL(req.URL), authType)
	}

	// HTTP trace timing
	// tt is allocated up front (rather than inside the deferred closure
	// below) because Go evaluates the return-statement expressions before
	// running deferred functions. If tt were nil when
	// `return &core.DownloadResult{TraceTimings: tt, ...}` is evaluated,
	// the caller would receive a nil TraceTimings pointer even though the
	// defer would then populate tt. Allocating here ensures the return
	// value captures a non-nil pointer that the defer can fill in place.
	var tt *core.TraceTimings
	if c.config.Output.TraceTime {
		tt = &core.TraceTimings{}

		// requestStart is captured before the trace is attached so it is
		// never zero, even on connections that the HTTP client reuses
		// from its pool (where DNS, TCP and TLS callbacks all stay
		// silent and dnsStart/tcpStart/tlsStart remain zero-valued).
		// Without it, TTFB = time.Since(dnsStart) would be the duration
		// since year 1, i.e. ~55 years, which is plainly bogus.
		requestStart := time.Now()

		var (
			dnsStart     time.Time
			dnsEnd       time.Time
			tcpStart     time.Time
			tcpEnd       time.Time
			tlsStart     time.Time
			tlsEnd       time.Time
			gotFirstByte time.Time
		)

		trace := &httptrace.ClientTrace{
			DNSStart:             func(_ httptrace.DNSStartInfo) { dnsStart = time.Now() },
			DNSDone:              func(_ httptrace.DNSDoneInfo) { dnsEnd = time.Now() },
			ConnectStart:         func(_, _ string) { tcpStart = time.Now() },
			ConnectDone:          func(_, _ string, _ error) { tcpEnd = time.Now() },
			TLSHandshakeStart:    func() { tlsStart = time.Now() },
			TLSHandshakeDone:     func(_ tls.ConnectionState, _ error) { tlsEnd = time.Now() },
			GotFirstResponseByte: func() { gotFirstByte = time.Now() },
		}
		httpReq = httpReq.WithContext(httptrace.WithClientTrace(httpReq.Context(), trace))

		defer func() {
			if !dnsEnd.IsZero() && !dnsStart.IsZero() {
				tt.DNSLookup = dnsEnd.Sub(dnsStart)
			}
			if !tcpEnd.IsZero() && !tcpStart.IsZero() {
				tt.TCPConnect = tcpEnd.Sub(tcpStart)
			}
			if !tlsEnd.IsZero() && !tlsStart.IsZero() {
				tt.TLSHandshake = tlsEnd.Sub(tlsStart)
			}
			if !gotFirstByte.IsZero() {
				// Always measure TTFB from requestStart, not dnsStart.
				// On reused connections dnsStart is the zero value and
				// time.Since(zero) would yield ~55 years.
				tt.TTFB = gotFirstByte.Sub(requestStart)
			}
			tt.Total = tt.DNSLookup + tt.TCPConnect + tt.TLSHandshake + tt.TTFB
		}()
	}

	var resp *http.Response
	retryErr := transport.Retry(ctx, func(ctx context.Context, attempt int) error {
		var err error
		resp, err = c.httpClient.Do(httpReq)
		return err
	}, c.config.Transport.Retry)

	if retryErr != nil {
		return nil, core.NewNetworkError("request failed after retries", retryErr, core.SafeURL(req.URL))
	}
	defer resp.Body.Close()
	c.updateHSTS(resp)

	if c.config.Output.ShowHeaders {
		fmt.Fprintf(os.Stderr, "\n--- Response Headers ---\n")
		for key, values := range resp.Header {
			for _, value := range values {
				fmt.Fprintf(os.Stderr, "%s: %s\n", key, value)
			}
		}
		fmt.Fprintf(os.Stderr, "------------------------\n")
	}

	if c.config.Output.Verbose {
		fmt.Fprintf(os.Stderr, "[seq] Response: status=%s, content-length=%d\n",
			resp.Status, resp.ContentLength)
	}

	// Handle 401 Unauthorized - try Digest Auth if auto mode
	if resp.StatusCode == http.StatusUnauthorized && authType == "auto" && username != "" {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[seq] Got 401, trying Digest Auth...\n")
		}

		// Use auth package's DigestAuthHandler instead of inline logic
		digestHandler := auth.NewDigestAuthHandler(username, password)
		authedReq, err := digestHandler.HandleResponse(httpReq, resp)
		if err != nil {
			resp.Body.Close()
			return nil, core.NewNetworkError("digest auth failed", err, core.SafeURL(req.URL))
		}

		// The old body was consumed by HandleResponse; close the original response
		_ = resp.Body.Close()

		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[seq] Retrying with Digest Auth\n")
		}

		// Retry request with digest auth
		resp, err = c.httpClient.Do(authedReq)
		if err != nil {
			return nil, core.NewNetworkError("digest auth request failed", err, core.SafeURL(req.URL))
		}
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
		// Retry on any HTTP error if RetryAllErrors is enabled
		if c.config.Transport.RetryAllErrors {
			resp.Body.Close()
			for attempt := 0; attempt < c.config.Transport.Retry.MaxRetries; attempt++ {
				if c.config.Output.Verbose {
					fmt.Fprintf(os.Stderr, "[seq] Retrying on HTTP %d (attempt %d/%d)\n",
						resp.StatusCode, attempt+1, c.config.Transport.Retry.MaxRetries)
				}
				time.Sleep(c.config.Transport.Retry.InitialDelay)
				resp, err = c.httpClient.Do(httpReq)
				if err != nil {
					// http.Client.Do may return a non-nil resp alongside an
					// error (e.g. read error after the response was received).
					// Close the body so the connection can be returned to the
					// pool, otherwise we leak connections on every failed retry.
					if resp != nil {
						resp.Body.Close()
					}
					continue
				}
				if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusPartialContent {
					goto okStatus
				}
				resp.Body.Close()
			}
		}
		return nil, core.NewProtocolError(
			fmt.Sprintf("HTTP error: %s", resp.Status),
			nil,
			core.SafeURL(req.URL),
		)
	}
okStatus:

	if resp.StatusCode == http.StatusPartialContent && startOffset == 0 {
		return nil, core.NewProtocolError("unexpected partial content", nil, core.SafeURL(req.URL))
	}

	writer, file, err := c.openOutputFile(req, startOffset)
	if err != nil {
		return nil, err
	}
	if file != nil {
		defer file.Close()
	}

	var reader io.Reader = resp.Body

	if c.config.RateLimiter != nil {
		reader = c.config.RateLimiter.WrapReader(reader)
	}

	// Decompression
	if req.AutoDecompress && startOffset == 0 {
		contentEncoding := resp.Header.Get("Content-Encoding")

		compressionType := compression.GetCompressionFromHeader(contentEncoding)

		if compressionType != "identity" {
			decompressor, ok := compression.Get(compressionType)
			if ok {
				wrappedReader, err := decompressor.Reader(resp.Body)
				if err == nil {
					reader = wrappedReader
					if c.config.Output.Verbose {
						fmt.Fprintf(os.Stderr, "[seq] Decompressing with %s\n", compressionType)
					}
				}
			}
		}
	}

	if req.ProgressCallback != nil {
		reader = &progressReader{
			r:        reader,
			callback: req.ProgressCallback,
			total:    totalSize,
			start:    startTime,
			offset:   startOffset,
		}
	}

	buf := make([]byte, c.config.Transport.BufferSize)
	var bytesDownloaded int64 = 0

	for {
		if ctx.Err() != nil {
			if req.Resume && req.Output != "" && bytesDownloaded > 0 {
				meta := output.NewResumeMetadata(
					req.URL.String(),
					resp.Header.Get("ETag"),
					resp.Header.Get("Last-Modified"),
					startOffset+bytesDownloaded,
					totalSize,
				)
				meta.Save(req.Output)
			}
			return nil, ctx.Err()
		}

		n, err := reader.Read(buf)
		if n > 0 {
			_, wErr := writer.Write(buf[:n])
			if wErr != nil {
				return nil, core.NewFileError("failed to write data", wErr)
			}
			bytesDownloaded += int64(n)
		}

		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, core.NewNetworkError("failed to read response body", err, core.SafeURL(req.URL))
		}
	}

	if req.Output != "" && req.Output != "-" {
		if c.config.Output.Verbose {
			fmt.Fprintf(os.Stderr, "[seq] Deleting resume metadata for %s\n", req.Output)
		}
		output.DeleteResumeMetadata(req.Output)
	}

	if c.config.Output.KeepTimestamps && req.Output != "" && req.Output != "-" {
		lastModified := resp.Header.Get("Last-Modified")
		if lastModified != "" {
			modTime, err := time.Parse(http.TimeFormat, lastModified)
			if err == nil {
				if err := os.Chtimes(req.Output, time.Now(), modTime); err != nil {
					if c.config.Output.Verbose {
						fmt.Fprintf(os.Stderr, "[timestamps] failed to set modtime: %v\n", err)
					}
				} else if c.config.Output.Verbose {
					fmt.Fprintf(os.Stderr, "[timestamps] set modtime to %s\n", modTime.Format(time.RFC3339))
				}
			}
		}
	}

	// --fail: treat HTTP 4xx/5xx as download failure
	if c.config.FailOnHTTPError && resp.StatusCode >= 400 {
		return nil, core.NewProtocolError(
			fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status), nil, core.SafeURL(req.URL))
	}

	duration := time.Since(startTime)
	speed := float64(0)
	if duration.Seconds() > 0 {
		speed = float64(bytesDownloaded) / duration.Seconds()
	}

	return &core.DownloadResult{
		BytesDownloaded: startOffset + bytesDownloaded,
		TotalSize:       totalSize,
		Duration:        duration,
		Speed:           speed,
		Protocol:        "HTTP/HTTPS",
		IPVersion:       6,
		OutputPath:      req.Output,
		Resumed:         resumed,
		Parallel:        false,
		HTTPStatusCode:  resp.StatusCode,
		TraceTimings:    tt,
	}, nil
}

type progressReader struct {
	r        io.Reader
	callback func(current, total int64, speed float64)
	total    int64
	read     int64
	start    time.Time
	offset   int64
}

func (pr *progressReader) Read(p []byte) (int, error) {
	n, err := pr.r.Read(p)
	if n > 0 && pr.callback != nil {
		pr.read += int64(n)
		duration := time.Since(pr.start)
		speed := float64(pr.offset+pr.read) / duration.Seconds()
		pr.callback(pr.offset+pr.read, pr.total, speed)
	}
	return n, err
}

// parseAcceptPatterns parses comma-separated accept patterns into a slice.
func parseAcceptPatterns(s string) []string {
	if s == "" {
		return nil
	}
	var patterns []string
	for _, p := range strings.Split(s, ",") {
		p = strings.TrimSpace(p)
		if p != "" {
			patterns = append(patterns, p)
		}
	}
	return patterns
}