feat: initial goget release — modern IPv6-first download utility
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
"codeberg.org/petrbalvin/goget/internal/hsts"
|
||||
"codeberg.org/petrbalvin/goget/internal/transport"
|
||||
)
|
||||
|
||||
// TransportConfig groups network, TLS, proxy, and connection settings.
|
||||
type TransportConfig struct {
|
||||
// Connection
|
||||
Timeout time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
MaxTime time.Duration
|
||||
MaxFileSize int64
|
||||
BufferSize int
|
||||
FollowRedirects bool
|
||||
MaxRedirects int
|
||||
NoClobber bool
|
||||
|
||||
// TLS and certificates
|
||||
TLS *transport.TLSConfig
|
||||
CACertFile string
|
||||
HSTSCache *hsts.Cache
|
||||
|
||||
// Proxy and dialer
|
||||
Proxy *transport.ProxyConfig
|
||||
Dialer *transport.DialerConfig
|
||||
BindInterface string
|
||||
|
||||
// DNS and IP filtering
|
||||
DNSServers []string
|
||||
BlockPrivateIPs bool
|
||||
|
||||
// Retry
|
||||
Retry *transport.RetryConfig
|
||||
MaxRetries int
|
||||
RetryDelay time.Duration
|
||||
RetryHTTPCodes []int
|
||||
RetryAllErrors bool
|
||||
}
|
||||
|
||||
// AuthConfig groups authentication settings.
|
||||
type AuthConfig struct {
|
||||
Username string
|
||||
Password string
|
||||
AuthType string // "basic", "digest", "auto"
|
||||
OAuth *auth.OAuthConfig
|
||||
}
|
||||
|
||||
// RecursiveConfig groups recursive download and mirroring settings.
|
||||
type RecursiveConfig struct {
|
||||
// Core recursion
|
||||
Enabled bool
|
||||
MaxDepth int
|
||||
FollowExternal bool
|
||||
ExcludePatterns []string
|
||||
Accept string
|
||||
Reject []string
|
||||
|
||||
// Parallel worker count for recursive downloads. 0 falls back to
|
||||
// Parallel.Connections (or the default). Applies to all recursive
|
||||
// protocols that support parallel file processing.
|
||||
Parallel int
|
||||
|
||||
// Scope control
|
||||
NoParent bool
|
||||
SpanHosts bool
|
||||
PageRequisites bool
|
||||
Domains []string
|
||||
|
||||
// Rate limiting for recursion
|
||||
Wait time.Duration
|
||||
RandomWait bool
|
||||
|
||||
// Output control
|
||||
CutDirs int
|
||||
AdjustExt bool
|
||||
DryRun bool
|
||||
|
||||
// Mirror mode
|
||||
Mirror bool
|
||||
MirrorAssets bool
|
||||
ConvertLinks bool
|
||||
RespectRobots bool
|
||||
}
|
||||
|
||||
// OutputConfig groups output, progress, and display settings.
|
||||
type OutputConfig struct {
|
||||
OutputDir string
|
||||
Verbose bool
|
||||
JSONOutput bool
|
||||
ShowHeaders bool
|
||||
ContentDisposition bool
|
||||
KeepTimestamps bool
|
||||
WriteOut string
|
||||
TraceTime bool
|
||||
OnComplete string
|
||||
WarcFile string
|
||||
AutoExtract bool
|
||||
ExtractDir string
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// URLInfo contains extracted information from an HTTP URL
|
||||
type URLInfo struct {
|
||||
Scheme string
|
||||
Host string
|
||||
Port string
|
||||
Path string
|
||||
Query string
|
||||
Fragment string
|
||||
User *url.Userinfo
|
||||
}
|
||||
|
||||
// Handler contains HTTP-specific logic
|
||||
type Handler struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewHandler creates new handler
|
||||
func NewHandler(client *Client) *Handler {
|
||||
return &Handler{client: client}
|
||||
}
|
||||
|
||||
// ParseHTTPURL parses an HTTP URL and extracts information
|
||||
func ParseHTTPURL(u *url.URL) (*URLInfo, error) {
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid http url scheme: %s", u.Scheme)
|
||||
}
|
||||
|
||||
host := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
if u.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
return &URLInfo{
|
||||
Scheme: u.Scheme,
|
||||
Host: host,
|
||||
Port: port,
|
||||
Path: u.Path,
|
||||
Query: u.RawQuery,
|
||||
Fragment: u.Fragment,
|
||||
User: u.User,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetContentType extracts Content-Type from the response
|
||||
func GetContentType(resp *http.Response) string {
|
||||
return resp.Header.Get("Content-Type")
|
||||
}
|
||||
|
||||
// GetContentLength extracts Content-Length from the response
|
||||
func GetContentLength(resp *http.Response) int64 {
|
||||
return resp.ContentLength
|
||||
}
|
||||
|
||||
// GetContentEncoding extracts Content-Encoding from the response
|
||||
func GetContentEncoding(resp *http.Response) string {
|
||||
return resp.Header.Get("Content-Encoding")
|
||||
}
|
||||
|
||||
// GetETag extracts ETag from the response
|
||||
func GetETag(resp *http.Response) string {
|
||||
return resp.Header.Get("ETag")
|
||||
}
|
||||
|
||||
// GetLastModified extracts Last-Modified from the response
|
||||
func GetLastModified(resp *http.Response) string {
|
||||
return resp.Header.Get("Last-Modified")
|
||||
}
|
||||
|
||||
// SupportsRangeCheck checks whether the server supports range requests
|
||||
func SupportsRangeCheck(resp *http.Response) bool {
|
||||
acceptRanges := resp.Header.Get("Accept-Ranges")
|
||||
return strings.ToLower(acceptRanges) == "bytes"
|
||||
}
|
||||
|
||||
// IsRedirect checks whether the response is a redirect
|
||||
func IsRedirect(statusCode int) bool {
|
||||
return statusCode >= 300 && statusCode < 400
|
||||
}
|
||||
|
||||
// GetRedirectLocation gets the Location header from a redirect response
|
||||
func GetRedirectLocation(resp *http.Response) string {
|
||||
return resp.Header.Get("Location")
|
||||
}
|
||||
|
||||
// BuildURLWithQuery creates a URL with query parameters
|
||||
func BuildURLWithQuery(base *url.URL, params map[string]string) (*url.URL, error) {
|
||||
if base == nil {
|
||||
return nil, fmt.Errorf("base url cannot be nil")
|
||||
}
|
||||
|
||||
result := *base
|
||||
query := result.Query()
|
||||
|
||||
for key, value := range params {
|
||||
query.Set(key, value)
|
||||
}
|
||||
|
||||
result.RawQuery = query.Encode()
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// GetUserinfo extracts authentication credentials from the URL
|
||||
func GetUserinfo(u *url.URL) (username, password string, ok bool) {
|
||||
if u.User == nil {
|
||||
return "", "", false
|
||||
}
|
||||
username = u.User.Username()
|
||||
password, ok = u.User.Password()
|
||||
return username, password, true
|
||||
}
|
||||
|
||||
// IsSecure checks whether the URL uses HTTPS
|
||||
func IsSecure(u *url.URL) bool {
|
||||
return u.Scheme == "https"
|
||||
}
|
||||
|
||||
// GetDefaultPortForScheme returns the default port for the scheme
|
||||
func GetDefaultPortForScheme(scheme string) string {
|
||||
switch scheme {
|
||||
case "https":
|
||||
return "443"
|
||||
case "http":
|
||||
return "80"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ParseContentDisposition extracts filename from Content-Disposition header
|
||||
func ParseContentDisposition(headerValue string) string {
|
||||
if headerValue == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
_, params, err := mime.ParseMediaType(headerValue)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Try filename* first (RFC 5987, extended filename)
|
||||
if filename, ok := params["filename*"]; ok {
|
||||
return filename
|
||||
}
|
||||
|
||||
// Fall back to filename
|
||||
if filename, ok := params["filename"]; ok {
|
||||
return filename
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// SanitizeFilename removes path components from a filename for security
|
||||
func SanitizeFilename(name string) string {
|
||||
name = filepath.Base(name)
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/transport"
|
||||
)
|
||||
|
||||
// testWriter implements io.Writer for testing
|
||||
type testWriter struct {
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (w *testWriter) Write(p []byte) (int, error) {
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
|
||||
func TestDownloadSequentialIntegration(t *testing.T) {
|
||||
// Create a test server that returns known content
|
||||
testContent := "Hello, this is test content for download integration testing!"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Parse the server URL
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
// Create the HTTP client
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Create download request
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Execute download
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded != int64(len(testContent)) {
|
||||
t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadParallelIntegration(t *testing.T) {
|
||||
// Create test content for parallel download
|
||||
testContent := strings.Repeat("B", 1024*1024) // 1MB
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Support range requests
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start, end int64
|
||||
n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
|
||||
if n >= 1 {
|
||||
if end == 0 || end >= int64(len(testContent)) {
|
||||
end = int64(len(testContent)) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write([]byte(testContent[start : end+1]))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Parallel = &core.ParallelConfig{
|
||||
Connections: 4,
|
||||
MinSize: 100, // Force parallel for small files
|
||||
MinChunkSize: 100,
|
||||
MaxChunkSize: 1024 * 1024,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
Parallel: cfg.Parallel,
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if !result.Parallel {
|
||||
t.Log("Download was not parallel (may be expected depending on server negotiation)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadResumeIntegration(t *testing.T) {
|
||||
testContent := "This is content for resume testing!"
|
||||
var requestCount int
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("ETag", "\"test-etag-123\"")
|
||||
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start int64
|
||||
fmt.Sscanf(rangeHeader, "bytes=%d-", &start)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write([]byte(testContent[start:]))
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Create resume info simulating a partially downloaded file
|
||||
resumeInfo := &core.ResumeInfo{
|
||||
DownloadedBytes: 10,
|
||||
ETag: "\"test-etag-123\"",
|
||||
URL: server.URL,
|
||||
}
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Resume: true,
|
||||
ResumeInfo: resumeInfo,
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if result.Resumed {
|
||||
t.Log("Download was resumed successfully")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadIntegration(t *testing.T) {
|
||||
var receivedBody string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read body", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
receivedBody = string(body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Test POST upload with form data
|
||||
formReq := &UploadRequest{
|
||||
URL: server.URL,
|
||||
Method: "POST",
|
||||
FormData: map[string]string{"key": "value"},
|
||||
Timeout: 0,
|
||||
}
|
||||
|
||||
result, err := client.Upload(context.Background(), formReq)
|
||||
if err != nil {
|
||||
t.Fatalf("Upload failed: %v", err)
|
||||
}
|
||||
|
||||
if result.StatusCode != http.StatusOK {
|
||||
t.Errorf("Expected status 200, got %d", result.StatusCode)
|
||||
}
|
||||
|
||||
if result.BytesUploaded <= 0 {
|
||||
t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded)
|
||||
}
|
||||
|
||||
if !strings.Contains(receivedBody, "key=value") {
|
||||
t.Errorf("Expected body to contain form data, got: %s", receivedBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFollowsRedirectIntegration(t *testing.T) {
|
||||
const finalBody = "redirected content"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/redirect" {
|
||||
http.Redirect(w, r, "/target", http.StatusFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(finalBody))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL + "/redirect")
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if result.BytesDownloaded != int64(len(finalBody)) {
|
||||
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a
|
||||
// server that always answers 500. It verifies the loop actually attempts the
|
||||
// configured number of retries and that each attempt's response body is
|
||||
// closed — guarding against the regression where http.Client.Do returned a
|
||||
// non-nil resp alongside an error and the body was never closed, exhausting
|
||||
// the connection pool.
|
||||
func TestRetryAllErrorsClosesBody(t *testing.T) {
|
||||
var requestCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
// Write a non-empty body so a leaking response body would matter
|
||||
// (an empty body is closed trivially by net/http's chunked reader).
|
||||
_, _ = w.Write([]byte("server error"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Transport.RetryAllErrors = true
|
||||
cfg.Transport.MaxRetries = 3
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 3,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error after exhausting retries on 500 responses")
|
||||
}
|
||||
|
||||
// We expect at least 1 (initial transport.Retry attempt) + MaxRetries
|
||||
// (RetryAllErrors loop). The exact number may be higher because Go's
|
||||
// HTTP transport may also retry once on its own when an attempt fails
|
||||
// (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert
|
||||
// the lower bound only — the goal is to confirm the retry loop runs and
|
||||
// that every failed response body is closed, not to pin a Go-internal
|
||||
// detail.
|
||||
minRequests := int32(1 + cfg.Transport.MaxRetries)
|
||||
if got := atomic.LoadInt32(&requestCount); got < minRequests {
|
||||
t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadRateLimiterAndProgressAreChained is a regression guard for the
|
||||
// BACKLOG entry "Rate limiter silently dropped when progress wrapper
|
||||
// enabled". The two wrappers must compose — httpReq.Body must end up
|
||||
// reading through the rate-limited reader, with progress reported on top.
|
||||
func TestUploadRateLimiterAndProgressAreChained(t *testing.T) {
|
||||
const payload = "the quick brown fox jumps over the lazy dog"
|
||||
|
||||
var receivedBody string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "read error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
receivedBody = string(body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Write payload to a temp file (Upload's PUT path opens the file).
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "payload.txt")
|
||||
if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil {
|
||||
t.Fatalf("write tmp: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
// Generous rate limit so the test isn't dominated by the limiter.
|
||||
// limitedReader.Read asks for len(p) tokens on every call, and the
|
||||
// HTTP transport uses a 32 KB buffer, so burst must comfortably exceed
|
||||
// that or Allow() blocks forever.
|
||||
cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024)
|
||||
cfg.Transport.MaxRetries = 0
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 0,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
progressCalls := 0
|
||||
var lastCurrent int64
|
||||
progressCb := func(current, total int64, _ float64) {
|
||||
progressCalls++
|
||||
if current < lastCurrent {
|
||||
t.Errorf("progress went backwards: %d -> %d", lastCurrent, current)
|
||||
}
|
||||
lastCurrent = current
|
||||
}
|
||||
|
||||
_, err = client.Upload(context.Background(), &UploadRequest{
|
||||
URL: server.URL,
|
||||
Method: "PUT",
|
||||
FilePath: tmpFile,
|
||||
ProgressCallback: progressCb,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
|
||||
// The server must receive the full, unmodified payload. If the rate
|
||||
// limiter wrapper was silently dropped (e.g. httpReq.Body not updated
|
||||
// after body was reassigned), the server would still get the data via
|
||||
// the original reader — so this check alone doesn't expose the bug.
|
||||
if receivedBody != payload {
|
||||
t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload)
|
||||
}
|
||||
|
||||
// Progress callback must fire at least once and reach the full payload
|
||||
// length. The exact call count depends on the read-buffer behaviour of
|
||||
// net/http (a 43-byte payload may be read in a single Read), but a count
|
||||
// of zero would prove the progress reader was bypassed — which is the
|
||||
// exact regression the BACKLOG entry warns about.
|
||||
if progressCalls < 1 {
|
||||
t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls)
|
||||
}
|
||||
if lastCurrent != int64(len(payload)) {
|
||||
t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few
|
||||
// 500s short-circuits the retry loop and produces a successful download —
|
||||
// another code path that must close every preceding failed response body
|
||||
// before jumping to okStatus.
|
||||
func TestRetryAllErrorsRecoversOn200(t *testing.T) {
|
||||
const finalBody = "ok after retries"
|
||||
var requestCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&requestCount, 1)
|
||||
if n < 3 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte("server error"))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(finalBody))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Transport.RetryAllErrors = true
|
||||
cfg.Transport.MaxRetries = 5
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 5,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if result.BytesDownloaded != int64(len(finalBody)) {
|
||||
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
|
||||
}
|
||||
if got := atomic.LoadInt32(&requestCount); got != 3 {
|
||||
t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTraceTimeTTFBOnReusedConnection is a regression guard for the
|
||||
// BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on
|
||||
// reused connections". The previous implementation measured TTFB as
|
||||
// time.Since(dnsStart); on a pooled HTTP connection the DNSStart
|
||||
// callback is silent, dnsStart stayed zero, and the result was ~55 years
|
||||
// instead of a plausible request-time duration. The fix anchors TTFB to
|
||||
// a requestStart timestamp captured before the trace is attached, so
|
||||
// the value is meaningful whether or not the connection is reused.
|
||||
func TestTraceTimeTTFBOnReusedConnection(t *testing.T) {
|
||||
var connectionCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&connectionCount, 1)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Output.TraceTime = true
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
reqTemplate := func() *core.DownloadRequest {
|
||||
return &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// First request opens the connection — DNSStart/ConnectStart/
|
||||
// TLSHandshakeStart all fire, populating the trace timestamps.
|
||||
if _, err := client.Download(context.Background(), reqTemplate()); err != nil {
|
||||
t.Fatalf("first download failed: %v", err)
|
||||
}
|
||||
|
||||
// Second request should be served from the pool. Whether or not
|
||||
// keep-alive actually reuses the connection in this test depends
|
||||
// on the transport's MaxIdleConnsPerHost, but the assertion is the
|
||||
// same: TTFB must be a plausible duration in either case.
|
||||
result, err := client.Download(context.Background(), reqTemplate())
|
||||
if err != nil {
|
||||
t.Fatalf("second download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.TraceTimings == nil {
|
||||
t.Fatal("TraceTimings is nil despite TraceTime=true")
|
||||
}
|
||||
|
||||
// 1 hour is a generous upper bound that flags the original ~55-year
|
||||
// regression (time.Since(time.Time{}) on a fresh time.Time) without
|
||||
// being flaky on a busy CI runner.
|
||||
if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour {
|
||||
t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb)
|
||||
}
|
||||
if ttfb := result.TraceTimings.TTFB; ttfb < 0 {
|
||||
t.Errorf("TTFB = %v; want non-negative", ttfb)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
func (c *Client) downloadParallel(ctx context.Context, req *core.DownloadRequest, totalSize int64, connections int, resumeChunks map[int]int64, username, password string) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
chunks := createChunks(totalSize, connections, c.config.Parallel)
|
||||
if c.config.Output.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[parallel] Split into %d chunks\n", len(chunks))
|
||||
}
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "goget-chunks-*")
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to create temp dir", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// completedChunks tracks per-chunk downloaded bytes for resume metadata.
|
||||
completedChunks := make(map[int]int64)
|
||||
var completedChunksMu sync.Mutex
|
||||
|
||||
for id, n := range resumeChunks {
|
||||
completedChunks[id] = n
|
||||
}
|
||||
|
||||
progressChan := make(chan int64, len(chunks)*2)
|
||||
var wg sync.WaitGroup
|
||||
var firstError error
|
||||
var errorMu sync.Mutex
|
||||
|
||||
for i := range chunks {
|
||||
chunk := &chunks[i]
|
||||
wg.Add(1)
|
||||
|
||||
go func(chunk *core.ChunkInfo) {
|
||||
defer wg.Done()
|
||||
|
||||
if resumeChunks != nil {
|
||||
if done, ok := resumeChunks[chunk.ID]; ok && done >= (chunk.End-chunk.Start+1) {
|
||||
progressChan <- done
|
||||
chunk.Status = 2
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
|
||||
err := c.downloadChunk(ctx, req, chunk, chunkPath, progressChan, username, password)
|
||||
if err != nil {
|
||||
errorMu.Lock()
|
||||
if firstError == nil {
|
||||
firstError = err
|
||||
}
|
||||
errorMu.Unlock()
|
||||
chunk.Status = 3
|
||||
chunk.Error = err
|
||||
return
|
||||
}
|
||||
// Save per-chunk progress for resume on interruption.
|
||||
completedChunksMu.Lock()
|
||||
completedChunks[chunk.ID] = chunk.End - chunk.Start + 1
|
||||
saveParallelResumeMetadata(req, totalSize, completedChunks)
|
||||
completedChunksMu.Unlock()
|
||||
chunk.Status = 2
|
||||
}(chunk)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(progressChan)
|
||||
}()
|
||||
|
||||
var totalDownloaded int64
|
||||
var cancelled bool
|
||||
aggregationLoop:
|
||||
for {
|
||||
select {
|
||||
case progress, ok := <-progressChan:
|
||||
if !ok {
|
||||
break aggregationLoop
|
||||
}
|
||||
totalDownloaded += progress
|
||||
if req.ProgressCallback != nil {
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalDownloaded) / duration.Seconds()
|
||||
req.ProgressCallback(totalDownloaded, totalSize, speed)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
cancelled = true
|
||||
break aggregationLoop
|
||||
}
|
||||
}
|
||||
|
||||
if cancelled {
|
||||
// Context was cancelled (SIGINT/SIGTERM). Return the error so
|
||||
// the caller can show "cancelled" instead of "complete", and
|
||||
// the progress bar renders the interrupted state. Partial
|
||||
// chunks are NOT merged — they remain in temp files for a
|
||||
// potential resume later.
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
if firstError != nil {
|
||||
return nil, firstError
|
||||
}
|
||||
|
||||
if req.Output != "" && req.Output != "-" {
|
||||
if err := mergeChunks(req.Output, chunks, tempDir); err != nil {
|
||||
return nil, core.NewFileError("failed to merge chunks", err)
|
||||
}
|
||||
}
|
||||
|
||||
if req.Output != "" {
|
||||
output.DeleteResumeMetadata(req.Output)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(0)
|
||||
if duration.Seconds() > 0 {
|
||||
speed = float64(totalSize) / duration.Seconds()
|
||||
}
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalSize,
|
||||
TotalSize: totalSize,
|
||||
Duration: duration,
|
||||
Speed: speed,
|
||||
Protocol: "HTTP/HTTPS",
|
||||
IPVersion: 6,
|
||||
OutputPath: req.Output,
|
||||
Parallel: true,
|
||||
ChunksCount: len(chunks),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// saveParallelResumeMetadata saves per-chunk progress to the resume metadata file.
|
||||
func saveParallelResumeMetadata(req *core.DownloadRequest, totalSize int64, chunks map[int]int64) {
|
||||
if req.Output == "" || req.Output == "-" {
|
||||
return
|
||||
}
|
||||
// Build a snapshot of the current chunk progress.
|
||||
snapshot := make(map[int]int64, len(chunks))
|
||||
for id, n := range chunks {
|
||||
snapshot[id] = n
|
||||
}
|
||||
meta := &output.ResumeMetadata{
|
||||
URL: req.URL.String(),
|
||||
Total: totalSize,
|
||||
Downloaded: totalChunkBytes(snapshot),
|
||||
LastWrite: time.Now(),
|
||||
Version: "1.0",
|
||||
Chunks: snapshot,
|
||||
}
|
||||
_ = meta.Save(req.Output)
|
||||
}
|
||||
|
||||
// totalChunkBytes sums the byte count of all completed chunks.
|
||||
func totalChunkBytes(chunks map[int]int64) int64 {
|
||||
var total int64
|
||||
for _, n := range chunks {
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func createChunks(totalSize int64, count int, cfg *core.ParallelConfig) []core.ChunkInfo {
|
||||
if count <= 1 || totalSize <= cfg.MinSize {
|
||||
return []core.ChunkInfo{
|
||||
{ID: 0, Start: 0, End: totalSize - 1},
|
||||
}
|
||||
}
|
||||
|
||||
// Divide the file into `count` equal-sized chunks. The last chunk
|
||||
// absorbs any remainder. MaxChunkSize is NOT used to further
|
||||
// subdivide — the user explicitly asked for N parallel connections
|
||||
// via --parallel, not N×M tiny range requests. Each chunk becomes
|
||||
// one HTTP Range request handled by one of the N worker goroutines.
|
||||
chunkSize := totalSize / int64(count)
|
||||
if chunkSize < cfg.MinChunkSize {
|
||||
chunkSize = cfg.MinChunkSize
|
||||
count = int((totalSize + chunkSize - 1) / chunkSize)
|
||||
}
|
||||
|
||||
chunks := make([]core.ChunkInfo, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
start := int64(i) * chunkSize
|
||||
end := start + chunkSize - 1
|
||||
if i == count-1 || end >= totalSize {
|
||||
end = totalSize - 1
|
||||
}
|
||||
chunks = append(chunks, core.ChunkInfo{
|
||||
ID: i,
|
||||
Start: start,
|
||||
End: end,
|
||||
})
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
func (c *Client) downloadChunk(ctx context.Context, req *core.DownloadRequest, chunk *core.ChunkInfo, outputPath string, progressChan chan<- int64, username, password string) error {
|
||||
chunk.Status = 1
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", req.URL.String(), nil)
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to create chunk request", err, req.URL.String())
|
||||
}
|
||||
|
||||
httpReq.Header.Set("User-Agent", c.config.UserAgent)
|
||||
httpReq.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", chunk.Start, chunk.End))
|
||||
|
||||
if req.ResumeInfo != nil {
|
||||
if req.ResumeInfo.ETag != "" {
|
||||
httpReq.Header.Set("If-Range", req.ResumeInfo.ETag)
|
||||
} else if req.ResumeInfo.LastModified != "" {
|
||||
httpReq.Header.Set("If-Range", req.ResumeInfo.LastModified)
|
||||
}
|
||||
}
|
||||
|
||||
if username != "" {
|
||||
auth.BasicAuth(httpReq, username, password)
|
||||
}
|
||||
|
||||
for key, value := range req.Headers {
|
||||
httpReq.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to fetch chunk", err, req.URL.String())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
c.updateHSTS(resp)
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return core.NewProtocolError(
|
||||
fmt.Sprintf("chunk HTTP error: %s", resp.Status),
|
||||
nil,
|
||||
req.URL.String(),
|
||||
)
|
||||
}
|
||||
|
||||
file, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return core.NewFileError("failed to create chunk file", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := io.Reader(resp.Body)
|
||||
if c.config.RateLimiter != nil {
|
||||
reader = c.config.RateLimiter.WrapReader(reader)
|
||||
}
|
||||
|
||||
buf := make([]byte, c.config.Transport.BufferSize)
|
||||
var downloaded int64
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
_, wErr := file.Write(buf[:n])
|
||||
if wErr != nil {
|
||||
return core.NewFileError("failed to write chunk", wErr)
|
||||
}
|
||||
downloaded += int64(n)
|
||||
if progressChan != nil {
|
||||
progressChan <- int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to read chunk", err, req.URL.String())
|
||||
}
|
||||
}
|
||||
|
||||
chunk.Downloaded = downloaded
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeChunks(outputPath string, chunks []core.ChunkInfo, tempDir string) error {
|
||||
outFile, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
for _, chunk := range chunks {
|
||||
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
|
||||
chunkFile, err := os.Open(chunkPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open chunk %d: %w", chunk.ID, err)
|
||||
}
|
||||
_, err = io.Copy(outFile, chunkFile)
|
||||
chunkFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy chunk %d: %w", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
format "codeberg.org/petrbalvin/goget/internal/format"
|
||||
)
|
||||
|
||||
// UploadRequest represents an upload request
|
||||
type UploadRequest struct {
|
||||
// URL to upload to
|
||||
URL string
|
||||
|
||||
// Method: PUT or POST
|
||||
Method string
|
||||
|
||||
// Path to the file for upload
|
||||
FilePath string
|
||||
|
||||
// Form data fields (for POST)
|
||||
FormData map[string]string
|
||||
|
||||
// Files for upload (for POST multipart)
|
||||
Files []UploadFile
|
||||
|
||||
// Custom headers
|
||||
Headers map[string]string
|
||||
|
||||
// Timeout
|
||||
Timeout time.Duration
|
||||
|
||||
// Callback pro progress
|
||||
ProgressCallback func(current, total int64, speed float64)
|
||||
}
|
||||
|
||||
// UploadFile represents a file for multipart upload
|
||||
type UploadFile struct {
|
||||
// Field name in the form data
|
||||
FieldName string
|
||||
|
||||
// Path to the file
|
||||
FilePath string
|
||||
|
||||
// Custom file name (optional)
|
||||
FileName string
|
||||
|
||||
// Content-Type (optional)
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// UploadResult is the result of an upload
|
||||
type UploadResult struct {
|
||||
StatusCode int
|
||||
ContentLength int64
|
||||
Duration time.Duration
|
||||
BytesUploaded int64
|
||||
Speed float64
|
||||
ResponseBody string
|
||||
}
|
||||
|
||||
// Upload uploads a file using PUT or POST
|
||||
func (c *Client) Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
if req.Method == "" {
|
||||
req.Method = "PUT"
|
||||
}
|
||||
|
||||
// Create request body
|
||||
var body io.Reader
|
||||
var contentType string
|
||||
var totalSize int64
|
||||
|
||||
if req.Method == "PUT" {
|
||||
// PUT upload - direct file
|
||||
if req.FilePath == "" {
|
||||
return nil, fmt.Errorf("put upload requires --upload-file")
|
||||
}
|
||||
|
||||
fileInfo, err := os.Stat(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
totalSize = fileInfo.Size()
|
||||
|
||||
file, err := os.Open(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
body = file
|
||||
contentType = "application/octet-stream"
|
||||
|
||||
} else if req.Method == "POST" {
|
||||
// POST upload - form data or multipart
|
||||
if len(req.Files) > 0 {
|
||||
// Multipart form data
|
||||
var bodyBuffer bytes.Buffer
|
||||
writer := multipart.NewWriter(&bodyBuffer)
|
||||
|
||||
// Add regular form fields
|
||||
for key, value := range req.FormData {
|
||||
if err := writer.WriteField(key, value); err != nil {
|
||||
return nil, fmt.Errorf("failed to write form field: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add files
|
||||
for _, uf := range req.Files {
|
||||
file, err := os.Open(uf.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file %s: %w", uf.FilePath, err)
|
||||
}
|
||||
|
||||
fileName := uf.FileName
|
||||
if fileName == "" {
|
||||
fileName = filepath.Base(uf.FilePath)
|
||||
}
|
||||
|
||||
contentType := uf.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set("Content-Disposition",
|
||||
fmt.Sprintf(`form-data; name="%s"; filename="%s"`, uf.FieldName, fileName))
|
||||
h.Set("Content-Type", contentType)
|
||||
|
||||
part, err := writer.CreatePart(h)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, fmt.Errorf("failed to create form part: %w", err)
|
||||
}
|
||||
|
||||
fileInfo, _ := os.Stat(uf.FilePath)
|
||||
totalSize += fileInfo.Size()
|
||||
|
||||
_, err = io.Copy(part, file)
|
||||
file.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write file to form: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
body = &bodyBuffer
|
||||
contentType = writer.FormDataContentType()
|
||||
|
||||
} else if len(req.FormData) > 0 {
|
||||
// URL-encoded form data
|
||||
form := url.Values{}
|
||||
for key, value := range req.FormData {
|
||||
form.Set(key, value)
|
||||
}
|
||||
|
||||
body = strings.NewReader(form.Encode())
|
||||
contentType = "application/x-www-form-urlencoded"
|
||||
totalSize = int64(len(form.Encode()))
|
||||
|
||||
} else if req.FilePath != "" {
|
||||
// POST with raw file body
|
||||
fileInfo, err := os.Stat(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
totalSize = fileInfo.Size()
|
||||
|
||||
file, err := os.Open(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
body = file
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
if body == nil {
|
||||
return nil, fmt.Errorf("no upload data provided")
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, req.Method, req.URL, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
httpReq.Header.Set("Content-Type", contentType)
|
||||
httpReq.Header.Set("User-Agent", c.config.UserAgent)
|
||||
|
||||
for key, value := range req.Headers {
|
||||
httpReq.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Apply authentication
|
||||
if c.config.Auth.Username != "" {
|
||||
authType := c.config.Auth.AuthType
|
||||
if authType == "" {
|
||||
authType = "auto"
|
||||
}
|
||||
|
||||
if authType == "auto" || authType == "basic" {
|
||||
// Try basic auth first
|
||||
applyBasicAuth(httpReq, c.config.Auth.Username, c.config.Auth.Password)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply OAuth token if configured
|
||||
if c.config.Auth.OAuth != nil && c.config.Auth.OAuth.AccessToken != "" {
|
||||
oauthClient := auth.NewOAuthClient(c.config.Auth.OAuth)
|
||||
if oauthClient.IsTokenValid() {
|
||||
oauthClient.ApplyToken(httpReq)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute request
|
||||
|
||||
if c.config.RateLimiter != nil {
|
||||
// Wrap body with rate limiter for upload
|
||||
body = c.config.RateLimiter.WrapReader(body)
|
||||
httpReq.Body = io.NopCloser(body)
|
||||
}
|
||||
|
||||
// Track progress
|
||||
if req.ProgressCallback != nil && totalSize > 0 {
|
||||
body = &progressReader{
|
||||
r: body,
|
||||
callback: req.ProgressCallback,
|
||||
total: totalSize,
|
||||
start: startTime,
|
||||
offset: 0,
|
||||
}
|
||||
httpReq.Body = io.NopCloser(body)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upload request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
return &UploadResult{
|
||||
StatusCode: resp.StatusCode,
|
||||
ContentLength: resp.ContentLength,
|
||||
Duration: duration,
|
||||
BytesUploaded: totalSize,
|
||||
Speed: float64(totalSize) / duration.Seconds(),
|
||||
ResponseBody: string(respBody),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// UploadSimple performs a simple file upload with progress tracking
|
||||
func (c *Client) UploadSimple(ctx context.Context, method, targetURL, filePath string, verbose bool) (*UploadResult, error) {
|
||||
fileInfo, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
|
||||
totalSize := fileInfo.Size()
|
||||
_ = totalSize // Used in progress callback
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var progressCallback func(int64, int64, float64)
|
||||
if verbose {
|
||||
progressCallback = func(current, total int64, speed float64) {
|
||||
percent := float64(current) / float64(total) * 100
|
||||
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
|
||||
format.Bytes(current),
|
||||
format.Bytes(total),
|
||||
percent,
|
||||
format.Speed(speed))
|
||||
}
|
||||
}
|
||||
|
||||
req := &UploadRequest{
|
||||
URL: targetURL,
|
||||
Method: method,
|
||||
FilePath: filePath,
|
||||
Timeout: c.config.Transport.Timeout,
|
||||
ProgressCallback: progressCallback,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Copy headers from client config
|
||||
for k, v := range c.config.Headers {
|
||||
req.Headers[k] = v
|
||||
}
|
||||
|
||||
result, err := c.Upload(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// UploadMultipart performs a multipart POST upload with files
|
||||
func (c *Client) UploadMultipart(ctx context.Context, targetURL string, files []UploadFile, formData map[string]string, verbose bool) (*UploadResult, error) {
|
||||
var progressCallback func(int64, int64, float64)
|
||||
if verbose {
|
||||
progressCallback = func(current, total int64, speed float64) {
|
||||
percent := float64(current) / float64(total) * 100
|
||||
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
|
||||
format.Bytes(current),
|
||||
format.Bytes(total),
|
||||
percent,
|
||||
format.Speed(speed))
|
||||
}
|
||||
}
|
||||
|
||||
req := &UploadRequest{
|
||||
URL: targetURL,
|
||||
Method: "POST",
|
||||
Files: files,
|
||||
FormData: formData,
|
||||
Timeout: c.config.Transport.Timeout,
|
||||
ProgressCallback: progressCallback,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
for k, v := range c.config.Headers {
|
||||
req.Headers[k] = v
|
||||
}
|
||||
|
||||
result, err := c.Upload(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// applyBasicAuth applies HTTP Basic Authentication to request
|
||||
func applyBasicAuth(req *http.Request, username, password string) {
|
||||
req.SetBasicAuth(username, password)
|
||||
}
|
||||
Reference in New Issue
Block a user