feat: initial goget release — modern IPv6-first download utility

This commit is contained in:
2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
+105
View File
@@ -0,0 +1,105 @@
//go:build linux || freebsd
// +build linux freebsd
package dataurl
import (
"context"
"encoding/base64"
"io"
"net/url"
"os"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Protocol handles data: URLs (RFC 2397).
type Protocol struct {
*protocol.BaseProtocol
}
// NewProtocol creates a new data: URL protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "DATA",
Scheme: "data",
DefaultPort: 0,
Features: []string{},
}),
}
}
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
startTime := time.Now()
// data:[<mediatype>][;base64],<data>
data, err := Decode(req.URL.String())
if err != nil {
return nil, core.NewProtocolError("failed to decode data URL", err, core.SafeURL(req.URL))
}
var writer io.Writer
outputPath := req.Output
if req.Writer != nil {
writer = req.Writer
} else if outputPath == "" || outputPath == "-" {
writer = os.Stdout
outputPath = "-"
} else {
f, err := os.Create(outputPath)
if err != nil {
return nil, core.NewFileError("failed to create output file", err)
}
defer f.Close()
writer = f
}
n, err := writer.Write(data)
if err != nil {
return nil, core.NewFileError("failed to write data URL content", err)
}
duration := time.Since(startTime)
return &core.DownloadResult{
BytesDownloaded: int64(n),
TotalSize: int64(len(data)),
Duration: duration,
Protocol: "DATA",
IPVersion: 0,
Speed: float64(n) / duration.Seconds(),
OutputPath: outputPath,
}, nil
}
// Decode parses and decodes a data: URL.
func Decode(rawURL string) ([]byte, error) {
if !strings.HasPrefix(rawURL, "data:") {
return nil, nil
}
content := strings.TrimPrefix(rawURL, "data:")
isBase64 := false
if idx := strings.Index(content, ";base64,"); idx >= 0 {
isBase64 = true
content = content[idx+8:]
} else if idx := strings.Index(content, ","); idx >= 0 {
content = content[idx+1:]
}
decoded, err := url.PathUnescape(content)
if err != nil {
decoded = content
}
if isBase64 {
return base64.StdEncoding.DecodeString(decoded)
}
return []byte(decoded), nil
}
+93
View File
@@ -0,0 +1,93 @@
//go:build linux || freebsd
// +build linux freebsd
package dataurl
import (
"bytes"
"context"
"net/url"
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
)
func TestDecodePlainText(t *testing.T) {
data, err := Decode("data:text/plain,hello%20world")
if err != nil {
t.Fatal(err)
}
if string(data) != "hello world" {
t.Errorf("got %q, want %q", data, "hello world")
}
}
func TestDecodeBase64(t *testing.T) {
data, err := Decode("data:text/plain;base64,SGVsbG8=")
if err != nil {
t.Fatal(err)
}
if string(data) != "Hello" {
t.Errorf("got %q, want %q", data, "Hello")
}
}
func TestDecodeEmpty(t *testing.T) {
data, err := Decode("data:,")
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Errorf("expected empty, got %d bytes", len(data))
}
}
func TestDecodeNotDataURL(t *testing.T) {
data, err := Decode("https://example.com")
if err != nil {
t.Fatal(err)
}
if data != nil {
t.Error("expected nil for non-data URL")
}
}
func TestDownloadPlainText(t *testing.T) {
var buf bytes.Buffer
proto := NewProtocol()
u, _ := url.Parse("data:text/plain,test123")
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Writer: &buf,
})
if err != nil {
t.Fatal(err)
}
if result.BytesDownloaded != 7 {
t.Errorf("bytes = %d, want 7", result.BytesDownloaded)
}
if buf.String() != "test123" {
t.Errorf("got %q, want %q", buf.String(), "test123")
}
}
func TestDownloadToFile(t *testing.T) {
tmp := t.TempDir()
dst := filepath.Join(tmp, "out.txt")
proto := NewProtocol()
u, _ := url.Parse("data:text/plain,hello")
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
})
if err != nil {
t.Fatal(err)
}
data, _ := os.ReadFile(dst)
if string(data) != "hello" {
t.Errorf("got %q, want %q", data, "hello")
}
}
+211
View File
@@ -0,0 +1,211 @@
//go:build linux || freebsd
// +build linux freebsd
package file
import (
"context"
"io"
"os"
"path/filepath"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Protocol implements file:// downloads.
type Protocol struct {
*protocol.BaseProtocol
}
// NewProtocol creates a new file protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "FILE",
Scheme: "file",
DefaultPort: 0,
Features: []string{"resume", "recursive"},
}),
}
}
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
startTime := time.Now()
filePath := req.URL.Path
if req.URL.Host != "" {
filePath = filepath.Join("/", req.URL.Host, filePath)
}
filePath = filepath.Clean(filePath)
info, err := os.Stat(filePath)
if err != nil {
return nil, core.NewFileError("failed to stat source path", err)
}
if info.IsDir() {
return p.downloadDirectory(ctx, req, filePath, startTime)
}
src, err := os.Open(filePath)
if err != nil {
return nil, core.NewFileError("failed to open source file", err)
}
defer src.Close()
totalSize := info.Size()
var writer io.Writer
var outputPath string
if req.Writer != nil {
writer = req.Writer
outputPath = req.Output
} else if req.Output == "" || req.Output == "-" {
writer = os.Stdout
outputPath = "-"
} else {
dst, err := os.Create(req.Output)
if err != nil {
return nil, core.NewFileError("failed to create output file", err)
}
defer dst.Close()
writer = dst
outputPath = req.Output
}
var written int64
if req.Resume && outputPath != "" && outputPath != "-" {
if fi, err := os.Stat(outputPath); err == nil {
written = fi.Size()
if written >= totalSize {
return &core.DownloadResult{
BytesDownloaded: totalSize,
TotalSize: totalSize,
Duration: time.Since(startTime),
Protocol: "FILE",
OutputPath: outputPath,
Resumed: true,
}, nil
}
if _, err := src.Seek(written, io.SeekStart); err != nil {
return nil, core.NewFileError("failed to seek for resume", err)
}
}
}
buf := make([]byte, 32*1024)
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
n, err := src.Read(buf)
if n > 0 {
if _, werr := writer.Write(buf[:n]); werr != nil {
return nil, core.NewFileError("failed to write data", werr)
}
written += int64(n)
if req.ProgressCallback != nil {
speed := float64(written) / time.Since(startTime).Seconds()
req.ProgressCallback(written, totalSize, speed)
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, core.NewFileError("failed to read source file", err)
}
}
duration := time.Since(startTime)
speed := float64(written) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: written,
TotalSize: totalSize,
Duration: duration,
Protocol: "FILE",
Speed: speed,
OutputPath: outputPath,
Resumed: written > 0 && req.Resume,
}, nil
}
// downloadDirectory recursively copies a local directory.
func (p *Protocol) downloadDirectory(ctx context.Context, req *core.DownloadRequest, dirPath string, startTime time.Time) (*core.DownloadResult, error) {
outputBase := req.Output
if outputBase == "" {
outputBase = filepath.Base(dirPath)
}
var totalBytes int64
var fileCount int
err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
// Calculate relative path
rel, err := filepath.Rel(dirPath, path)
if err != nil {
return err
}
outputPath := filepath.Join(outputBase, rel)
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
return err
}
src, err := os.Open(path)
if err != nil {
return err
}
defer src.Close()
dst, err := os.Create(outputPath)
if err != nil {
return err
}
defer dst.Close()
n, err := io.Copy(dst, src)
if err != nil {
return err
}
totalBytes += n
fileCount++
return nil
})
if err != nil {
return nil, core.NewFileError("failed to copy directory recursively", err)
}
duration := time.Since(startTime)
speed := float64(0)
if duration.Seconds() > 0 {
speed = float64(totalBytes) / duration.Seconds()
}
return &core.DownloadResult{
BytesDownloaded: totalBytes,
TotalSize: totalBytes,
Duration: duration,
Protocol: "FILE",
Speed: speed,
OutputPath: outputBase,
ChunksCount: fileCount,
}, nil
}
+129
View File
@@ -0,0 +1,129 @@
//go:build linux || freebsd
// +build linux freebsd
package file
import (
"bytes"
"context"
"net/url"
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
)
func TestDownloadFile(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
dst := filepath.Join(tmp, "dst.txt")
os.WriteFile(src, []byte("hello world"), 0644)
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
})
if err != nil {
t.Fatal(err)
}
if result.BytesDownloaded != 11 {
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
}
data, _ := os.ReadFile(dst)
if string(data) != "hello world" {
t.Errorf("content = %q, want %q", data, "hello world")
}
}
func TestDownloadFileToWriter(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
os.WriteFile(src, []byte("test"), 0644)
var buf bytes.Buffer
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: "-",
Writer: &buf,
})
if err != nil {
t.Fatal(err)
}
if buf.String() != "test" {
t.Errorf("content = %q, want %q", buf.String(), "test")
}
}
func TestDownloadFileNotFound(t *testing.T) {
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: "/nonexistent/path"}
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: "/tmp/out",
})
if err == nil {
t.Error("expected error for nonexistent file")
}
}
func TestDownloadDirectory(t *testing.T) {
tmp := t.TempDir()
srcDir := filepath.Join(tmp, "srcdir")
dstDir := filepath.Join(tmp, "dstdir")
os.MkdirAll(filepath.Join(srcDir, "sub"), 0755)
os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("a"), 0644)
os.WriteFile(filepath.Join(srcDir, "sub/b.txt"), []byte("b"), 0644)
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: srcDir}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dstDir,
})
if err != nil {
t.Fatal(err)
}
if result.ChunksCount != 2 {
t.Errorf("files = %d, want 2", result.ChunksCount)
}
if result.BytesDownloaded != 2 {
t.Errorf("bytes = %d, want 2", result.BytesDownloaded)
}
// Verify files exist
if _, err := os.Stat(filepath.Join(dstDir, "a.txt")); err != nil {
t.Error("a.txt not found")
}
if _, err := os.Stat(filepath.Join(dstDir, "sub", "b.txt")); err != nil {
t.Error("sub/b.txt not found")
}
}
func TestResumeFile(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
dst := filepath.Join(tmp, "dst.txt")
os.WriteFile(src, []byte("hello world"), 0644)
os.WriteFile(dst, []byte("hello"), 0644) // partial: 5 bytes
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
Resume: true,
})
if err != nil {
t.Fatal(err)
}
if !result.Resumed {
t.Error("expected Resumed=true")
}
if result.BytesDownloaded != 11 {
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
}
}
+817
View File
@@ -0,0 +1,817 @@
//go:build linux || freebsd
// +build linux freebsd
package ftp
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"codeberg.org/petrbalvin/goget/internal/output"
)
type Client struct {
conn net.Conn
reader *textproto.Reader
writer *bufio.Writer
host string
port int
user string
password string
url string
tlsConfig *tls.Config
useTLS bool
passive bool
timeout time.Duration
}
func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
u, err := url.Parse(rawURL)
if err != nil {
return nil, fmt.Errorf("invalid url: %w", err)
}
host := u.Hostname()
port := 21
if u.Port() != "" {
p, err := strconv.Atoi(u.Port())
if err != nil {
return nil, fmt.Errorf("invalid port: %w", err)
}
port = p
}
user := "anonymous"
password := "anonymous@"
if u.User != nil {
user = u.User.Username()
if pass, ok := u.User.Password(); ok {
password = pass
}
}
useTLS := u.Scheme == "ftps"
if !useTLS && (user != "anonymous" || (u.User != nil && u.User.Username() != "anonymous")) {
fmt.Fprintf(os.Stderr, "Warning: FTP credentials sent in plaintext. Use ftps:// for encrypted transfers.\n")
}
c := &Client{
host: host,
port: port,
user: user,
password: password,
url: rawURL,
useTLS: useTLS,
passive: true, // Default to passive mode
timeout: timeout,
tlsConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: host,
InsecureSkipVerify: false,
},
}
return c, nil
}
func (c *Client) Connect() error {
addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port))
conn, err := net.DialTimeout("tcp", addr, c.timeout)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
c.conn = conn
c.reader = textproto.NewReader(bufio.NewReader(conn))
c.writer = bufio.NewWriter(conn)
// Read greeting
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("failed to read greeting: %w", err)
}
if code != 220 {
return fmt.Errorf("unexpected greeting code: %d", code)
}
// Upgrade to TLS if FTPS (explicit)
if c.useTLS {
if err := c.sendCommand("AUTH TLS"); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("auth tls failed: %w", err)
}
if code != 234 && code != 334 {
// Try AUTH SSL as fallback
if err := c.sendCommand("AUTH SSL"); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("auth ssl failed: %w", err)
}
if code != 234 && code != 334 {
return fmt.Errorf("server does not support tls/ssl")
}
}
// Wrap connection with TLS
tlsConn := tls.Client(c.conn, c.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
return fmt.Errorf("tls handshake failed: %w", err)
}
c.conn = tlsConn
c.reader = textproto.NewReader(bufio.NewReader(tlsConn))
c.writer = bufio.NewWriter(tlsConn)
}
// Send USER
if err := c.sendCommand("USER %s", c.user); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("user command failed: %w", err)
}
// 331 means password required, 230 means already logged in
if code == 331 {
// Send PASS
if err := c.sendCommand("PASS %s", c.password); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("pass command failed: %w", err)
}
}
if code != 230 && code != 200 {
return fmt.Errorf("login failed with code: %d", code)
}
// Set binary mode
if err := c.sendCommand("TYPE I"); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("type command failed: %w", err)
}
if code != 200 {
return fmt.Errorf("failed to set binary mode")
}
return nil
}
func (c *Client) sendCommand(format string, args ...interface{}) error {
cmd := fmt.Sprintf(format, args...)
if _, err := c.writer.WriteString(cmd); err != nil {
return fmt.Errorf("failed to send command: %w", err)
}
if _, err := c.writer.WriteString("\r\n"); err != nil {
return fmt.Errorf("failed to send command terminator: %w", err)
}
if err := c.writer.Flush(); err != nil {
return fmt.Errorf("failed to flush command: %w", err)
}
return nil
}
func (c *Client) readResponse() (int, string, error) {
code, msg, err := c.reader.ReadResponse(0)
if err != nil {
return 0, "", err
}
return code, msg, nil
}
func (c *Client) getFileSize(path string) (int64, error) {
if err := c.sendCommand("SIZE %s", path); err != nil {
return 0, err
}
code, msg, err := c.readResponse()
if err != nil {
return 0, fmt.Errorf("size command failed: %w", err)
}
if code != 213 {
return 0, fmt.Errorf("size command failed with code: %d", code)
}
size, err := strconv.ParseInt(strings.TrimSpace(msg), 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse size: %w", err)
}
return size, nil
}
func (c *Client) establishDataConnection() (net.Conn, error) {
if c.passive {
return c.enterPassiveMode()
}
return c.enterActiveMode()
}
func (c *Client) enterPassiveMode() (net.Conn, error) {
if err := c.sendCommand("PASV"); err != nil {
return nil, err
}
code, msg, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("pasv command failed: %w", err)
}
if code != 227 {
return nil, fmt.Errorf("pasv command failed with code: %d", code)
}
// Parse PASV response: 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2)
start := strings.Index(msg, "(")
end := strings.Index(msg, ")")
if start == -1 || end == -1 {
return nil, fmt.Errorf("invalid pasv response")
}
parts := strings.Split(msg[start+1:end], ",")
if len(parts) != 6 {
return nil, fmt.Errorf("invalid pasv response format")
}
h1, _ := strconv.Atoi(parts[0])
h2, _ := strconv.Atoi(parts[1])
h3, _ := strconv.Atoi(parts[2])
h4, _ := strconv.Atoi(parts[3])
p1, _ := strconv.Atoi(parts[4])
p2, _ := strconv.Atoi(parts[5])
dataHost := fmt.Sprintf("%d.%d.%d.%d", h1, h2, h3, h4)
dataPort := p1*256 + p2
// Use original host for TLS connections to match certificate
if c.useTLS {
dataHost = c.host
}
conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", dataHost, dataPort), c.timeout)
if err != nil {
return nil, fmt.Errorf("failed to connect to data port: %w", err)
}
if c.useTLS {
tlsConn := tls.Client(conn, c.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
conn.Close()
return nil, fmt.Errorf("tls handshake on data connection failed: %w", err)
}
return tlsConn, nil
}
return conn, nil
}
func (c *Client) enterActiveMode() (net.Conn, error) {
// Get local IP from the control connection to bind only to that interface
ctrlAddr, ok := c.conn.LocalAddr().(*net.TCPAddr)
if !ok || ctrlAddr.IP == nil {
return nil, fmt.Errorf("failed to get local address for active mode")
}
ctrlIP := ctrlAddr.IP.To4()
if ctrlIP == nil {
return nil, fmt.Errorf("no ipv4 address available for active mode")
}
// Bind listener only to the interface of the control connection, not 0.0.0.0
listener, err := net.Listen("tcp", net.JoinHostPort(ctrlIP.String(), "0"))
if err != nil {
return nil, fmt.Errorf("failed to create listener: %w", err)
}
defer listener.Close()
// Set a deadline so Accept respects the timeout
if tcpListener, ok := listener.(*net.TCPListener); ok {
tcpListener.SetDeadline(time.Now().Add(c.timeout))
}
listenAddr := listener.Addr().(*net.TCPAddr)
p1 := listenAddr.Port / 256
p2 := listenAddr.Port % 256
portCmd := fmt.Sprintf("PORT %d,%d,%d,%d,%d,%d",
ctrlIP[0], ctrlIP[1], ctrlIP[2], ctrlIP[3], p1, p2)
if err := c.sendCommand("%s", portCmd); err != nil {
return nil, err
}
code, _, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("port command failed: %w", err)
}
if code != 200 {
return nil, fmt.Errorf("port command failed with code: %d", code)
}
// Accept incoming connection
connChan := make(chan net.Conn, 1)
errChan := make(chan error, 1)
go func() {
conn, err := listener.Accept()
if err != nil {
errChan <- err
return
}
connChan <- conn
}()
select {
case conn := <-connChan:
return conn, nil
case err := <-errChan:
return nil, fmt.Errorf("failed to accept data connection: %w", err)
case <-time.After(c.timeout):
return nil, fmt.Errorf("timeout waiting for data connection")
}
}
func (c *Client) DownloadFile(ctx context.Context, remotePath, localPath string, resumeOffset int64, progressFunc func(current, total int64)) error {
// Get file size
totalSize, err := c.getFileSize(remotePath)
if err != nil {
return fmt.Errorf("failed to get file size: %w", err)
}
// Check if we need to resume
var startOffset int64 = 0
if resumeOffset > 0 && resumeOffset < totalSize {
startOffset = resumeOffset
if err := c.sendCommand("REST %d", startOffset); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("rest command failed: %w", err)
}
if code != 350 {
return fmt.Errorf("rest command failed with code: %d", code)
}
} else if resumeOffset >= totalSize {
// File already complete
return nil
}
// Open local file for writing
mode := os.O_CREATE | os.O_WRONLY
if startOffset > 0 {
mode |= os.O_APPEND
} else {
mode |= os.O_TRUNC
}
localFile, err := os.OpenFile(localPath, mode, 0644)
if err != nil {
return fmt.Errorf("failed to open local file: %w", err)
}
defer localFile.Close()
// Seek to start position if resuming
if startOffset > 0 {
if _, err := localFile.Seek(startOffset, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek: %w", err)
}
}
// Establish data connection FIRST (before RETR command)
dataConn, err := c.establishDataConnection()
if err != nil {
return fmt.Errorf("failed to establish data connection: %w", err)
}
defer dataConn.Close()
// Send RETR command AFTER data connection is established
if err := c.sendCommand("RETR %s", remotePath); err != nil {
return err
}
// Read response (should be 150 or 125)
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("retr command failed: %w", err)
}
if code != 150 && code != 125 {
// Try waiting a bit for the server to respond
time.Sleep(100 * time.Millisecond)
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("retr command failed with code: %d", code)
}
if code != 150 && code != 125 {
return fmt.Errorf("retr command failed with code: %d", code)
}
}
// Copy data with progress. Polls ctx between reads so a SIGINT
// arriving mid-transfer is detected on the next iteration and the
// partial progress is saved to ResumeMetadata before returning.
var copied int64 = startOffset
buf := make([]byte, 32*1024)
for {
if ctx.Err() != nil {
if err := saveFTPResume(c.url, localPath, copied, totalSize); err != nil {
return fmt.Errorf("failed to save resume metadata: %w", err)
}
return ctx.Err()
}
n, err := dataConn.Read(buf)
if n > 0 {
written, werr := localFile.Write(buf[:n])
copied += int64(written)
if werr != nil {
return fmt.Errorf("failed to write: %w", err)
}
if progressFunc != nil {
progressFunc(copied, totalSize)
}
}
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("failed to read from data connection: %w", err)
}
}
// Read final response
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("failed to read final response: %w", err)
}
if code != 226 && code != 250 {
return fmt.Errorf("transfer failed with code: %d", code)
}
// Successful transfer — drop any stale resume sidecar.
_ = output.DeleteResumeMetadata(localPath)
return nil
}
func (c *Client) ListRemoteDir(path string) ([]string, error) {
// Establish data connection
dataConn, err := c.establishDataConnection()
if err != nil {
return nil, fmt.Errorf("failed to establish data connection: %w", err)
}
defer dataConn.Close()
// Send LIST command
cmd := "LIST"
if path != "" {
cmd = fmt.Sprintf("LIST %s", path)
}
if err := c.sendCommand("%s", cmd); err != nil {
return nil, err
}
// Read response (should be 150 or 125)
code, _, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("list command failed: %w", err)
}
if code != 150 && code != 125 {
return nil, fmt.Errorf("list command failed with code: %d", code)
}
// Read directory listing
var lines []string
scanner := bufio.NewScanner(dataConn)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to read directory listing: %w", err)
}
// Read final response
code, _, err = c.readResponse()
if err != nil {
return nil, fmt.Errorf("failed to read final response: %w", err)
}
if code != 226 && code != 250 {
return nil, fmt.Errorf("list command failed with code: %d", code)
}
return lines, nil
}
func (c *Client) ChangeDir(path string) error {
if err := c.sendCommand("CWD %s", path); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("cwd command failed: %w", err)
}
if code != 250 {
return fmt.Errorf("cwd command failed with code: %d", code)
}
return nil
}
func (c *Client) GetCurrentDir() (string, error) {
if err := c.sendCommand("PWD"); err != nil {
return "", err
}
code, msg, err := c.readResponse()
if err != nil {
return "", fmt.Errorf("pwd command failed: %w", err)
}
if code != 257 {
return "", fmt.Errorf("pwd command failed with code: %d", code)
}
// Extract path from response (usually quoted)
start := strings.Index(msg, "\"")
end := strings.LastIndex(msg, "\"")
if start != -1 && end > start {
return msg[start+1 : end], nil
}
return strings.TrimSpace(msg), nil
}
func (c *Client) Close() error {
if err := c.sendCommand("QUIT"); err != nil {
return err
}
_, _, _ = c.readResponse() // Ignore errors on close
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// Helper function to download entire directory recursively
func (c *Client) DownloadDirectory(ctx context.Context, remotePath, localPath string, progressFunc func(file string, current, total int64)) error {
// Create local directory
if err := os.MkdirAll(localPath, 0755); err != nil {
return fmt.Errorf("failed to create local directory: %w", err)
}
// Save current directory
origDir, err := c.GetCurrentDir()
if err != nil {
return err
}
defer func() {
c.ChangeDir(origDir)
}()
// Change to remote directory
if err := c.ChangeDir(remotePath); err != nil {
return err
}
// List directory contents
entries, err := c.ListRemoteDir("")
if err != nil {
return err
}
// Parse entries and download files
for _, entry := range entries {
fields := strings.Fields(entry)
if len(fields) < 9 {
continue
}
name := fields[len(fields)-1]
if name == "." || name == ".." {
continue
}
isDir := fields[0][0] == 'd'
remoteFilePath := filepath.Join(remotePath, name)
localFilePath := filepath.Join(localPath, name)
if isDir {
if err := c.DownloadDirectory(ctx, remoteFilePath, localFilePath, progressFunc); err != nil {
return err
}
} else {
if progressFunc != nil {
progressFunc(name, 0, 0)
}
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, nil); err != nil {
return err
}
}
}
return nil
}
// saveFTPResume persists partial FTP download progress to the standard
// `.goget.meta` sidecar so a subsequent `goget --resume` can continue
// from `downloaded` bytes. Errors are wrapped to surface failures
// without aborting the cancellation handling.
func saveFTPResume(url, localPath string, downloaded, total int64) error {
if localPath == "" || localPath == "-" || downloaded <= 0 {
return nil
}
meta := output.NewResumeMetadata(url, "", "", downloaded, total)
return meta.Save(localPath)
}
// clientPool holds N pre-connected FTP clients. A single shared pool
// removes the need to open/close connections for every file in a
// recursive download, and is required for the parallel recursive
// implementation because each *Client owns its own TCP control
// connection and is not safe for concurrent use.
type clientPool struct {
available chan *Client
all []*Client
}
// newClientPool opens `size` independent FTP connections to the same
// server. Connections are kept open until close() is called.
func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) {
if size < 1 {
size = 1
}
all := make([]*Client, 0, size)
avail := make(chan *Client, size)
for i := 0; i < size; i++ {
c, err := NewClient(rawURL, timeout)
if err != nil {
for _, existing := range all {
_ = existing.Close()
}
return nil, err
}
if err := c.Connect(); err != nil {
for _, existing := range all {
_ = existing.Close()
}
return nil, err
}
all = append(all, c)
avail <- c
}
return &clientPool{available: avail, all: all}, nil
}
// acquire blocks until a client is available, then returns it. The
// caller must call release() when done.
func (p *clientPool) acquire() *Client {
return <-p.available
}
// release returns a client to the pool. It must be called exactly once
// per acquire().
func (p *clientPool) release(c *Client) {
p.available <- c
}
// size returns the number of clients in the pool.
func (p *clientPool) size() int {
return len(p.all)
}
// close shuts down every client in the pool. The pool must not be used
// after this call.
func (p *clientPool) close() {
for _, c := range p.all {
_ = c.Close()
}
}
// DownloadDirectoryConcurrent downloads a remote directory to localPath
// using a pool of N FTP connections. All operations (LIST, RETR) use
// absolute paths so concurrent workers never share or mutate CWD state
// on the server side. The returned error is the first error encountered
// by any worker; other workers continue in the background until they
// hit ctx cancellation or finish their tasks.
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error {
if parallel < 1 {
parallel = 1
}
pool, err := newClientPool(rawURL, timeout, parallel)
if err != nil {
return err
}
defer pool.close()
sem := make(chan struct{}, parallel)
var wg sync.WaitGroup
var firstErr error
var errMu sync.Mutex
setErr := func(err error) {
errMu.Lock()
if firstErr == nil {
firstErr = err
}
errMu.Unlock()
}
processEntry(ctx, pool, sem, &wg, setErr, remotePath, localPath, progressFunc)
wg.Wait()
return firstErr
}
// processEntry processes one remote path: lists it, then dispatches
// its child entries (file → download, subdir → recursive processEntry)
// through the shared semaphore-bounded goroutine pool. A single
// sem is threaded through the recursion so the total in-flight
// goroutines never exceed `parallel`, regardless of directory depth.
func processEntry(ctx context.Context, pool *clientPool, sem chan struct{}, wg *sync.WaitGroup, setErr func(error), remotePath, localPath string, progressFunc func(file string, current, total int64)) {
defer wg.Done()
defer func() { <-sem }()
if ctx.Err() != nil {
return
}
if err := os.MkdirAll(localPath, 0755); err != nil {
setErr(fmt.Errorf("failed to create local dir %s: %w", localPath, err))
return
}
c := pool.acquire()
entries, err := c.ListRemoteDir(remotePath)
pool.release(c)
if err != nil {
setErr(fmt.Errorf("list %s: %w", remotePath, err))
return
}
for _, entry := range entries {
fields := strings.Fields(entry)
if len(fields) < 9 {
continue
}
name := fields[len(fields)-1]
if name == "." || name == ".." {
continue
}
isDir := fields[0][0] == 'd'
remoteFilePath := filepath.Join(remotePath, name)
localFilePath := filepath.Join(localPath, name)
if isDir {
wg.Add(1)
select {
case sem <- struct{}{}:
go processEntry(ctx, pool, sem, wg, setErr, remoteFilePath, localFilePath, progressFunc)
case <-ctx.Done():
return
}
} else {
wg.Add(1)
select {
case sem <- struct{}{}:
go func(remoteFilePath, localFilePath, name string) {
defer wg.Done()
defer func() { <-sem }()
if ctx.Err() != nil {
return
}
c := pool.acquire()
defer pool.release(c)
if progressFunc != nil {
progressFunc(name, 0, 0)
}
// DownloadFile's progress callback has a different
// signature than our directory-level progressFunc, so
// we wrap it here to keep the existing per-file
// progress behaviour.
var fileProgress func(current, total int64)
if progressFunc != nil {
fileProgress = func(current, total int64) {
progressFunc(name, current, total)
}
}
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, fileProgress); err != nil {
setErr(fmt.Errorf("download %s: %w", remoteFilePath, err))
}
}(remoteFilePath, localFilePath, name)
case <-ctx.Done():
return
}
}
}
}
+382
View File
@@ -0,0 +1,382 @@
//go:build linux || freebsd
// +build linux freebsd
package ftp
import (
"errors"
"net/url"
"strings"
"testing"
"time"
)
// ---------- Protocol tests ----------
func TestNewProtocol(t *testing.T) {
p := NewProtocol()
if p == nil {
t.Fatal("NewProtocol() returned nil")
}
}
func TestScheme(t *testing.T) {
p := &Protocol{}
if got := p.Scheme(); got != "ftp" {
t.Errorf("Scheme() = %q, want %q", got, "ftp")
}
}
func TestCanHandle(t *testing.T) {
tests := []struct {
name string
raw string
want bool
}{
{"ftp scheme", "ftp://ftp.example.com/file.txt", true},
{"ftps scheme", "ftps://ftp.example.com/file.txt", true},
{"http scheme", "http://example.com/file.txt", false},
{"https scheme", "https://example.com/file.txt", false},
{"sftp scheme", "sftp://example.com/file.txt", false},
{"empty scheme", "file.txt", false},
{"nil URL", "", false},
}
p := &Protocol{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var u *url.URL
if tt.raw != "" {
var err error
u, err = url.Parse(tt.raw)
if err != nil {
t.Fatalf("failed to parse URL %q: %v", tt.raw, err)
}
}
if got := p.CanHandle(u); got != tt.want {
t.Errorf("CanHandle(%q) = %v, want %v", tt.raw, got, tt.want)
}
})
}
}
func TestCanHandleCaseInsensitive(t *testing.T) {
p := &Protocol{}
u, _ := url.Parse("FTP://example.com/file.txt")
if !p.CanHandle(u) {
t.Errorf("CanHandle should be case-insensitive for FTP")
}
u, _ = url.Parse("FTPS://example.com/file.txt")
if !p.CanHandle(u) {
t.Errorf("CanHandle should be case-insensitive for FTPS")
}
}
func TestCapabilities(t *testing.T) {
p := &Protocol{}
caps := p.Capabilities()
if caps == nil {
t.Fatal("Capabilities() returned nil")
}
expected := map[string]bool{
"download": true,
"resume": true,
"recursive": true,
"tls": true,
}
if len(caps) != len(expected) {
t.Errorf("Capabilities() length = %d, want %d", len(caps), len(expected))
}
for _, cap := range caps {
if !expected[cap] {
t.Errorf("unexpected capability: %q", cap)
}
}
}
func TestSupportsResume(t *testing.T) {
p := &Protocol{}
if !p.SupportsResume() {
t.Errorf("SupportsResume() should return true")
}
}
func TestSupportsCompression(t *testing.T) {
p := &Protocol{}
if p.SupportsCompression() {
t.Errorf("SupportsCompression() should return false")
}
}
func TestSupportsParallel(t *testing.T) {
p := &Protocol{}
if p.SupportsParallel() {
t.Errorf("SupportsParallel() should return false")
}
}
func TestSupportsRecursive(t *testing.T) {
p := &Protocol{}
if !p.SupportsRecursive() {
t.Errorf("SupportsRecursive() should return true")
}
}
func TestReadAll(t *testing.T) {
t.Run("normal content", func(t *testing.T) {
input := "Hello, FTP world!"
r := strings.NewReader(input)
data, err := readAll(r)
if err != nil {
t.Fatalf("readAll() returned error: %v", err)
}
if string(data) != input {
t.Errorf("readAll() = %q, want %q", string(data), input)
}
})
t.Run("empty reader", func(t *testing.T) {
r := strings.NewReader("")
data, err := readAll(r)
if err != nil {
t.Fatalf("readAll() returned error: %v", err)
}
if len(data) != 0 {
t.Errorf("readAll() returned %d bytes, want 0", len(data))
}
})
t.Run("large content", func(t *testing.T) {
// Create content larger than the 32KB buffer to ensure multiple reads
content := strings.Repeat("A", 100*1024)
r := strings.NewReader(content)
data, err := readAll(r)
if err != nil {
t.Fatalf("readAll() returned error: %v", err)
}
if len(data) != len(content) {
t.Errorf("readAll() returned %d bytes, want %d", len(data), len(content))
}
})
t.Run("reader error", func(t *testing.T) {
expectedErr := errors.New("read error")
r := &errorReader{err: expectedErr}
_, err := readAll(r)
if err == nil {
t.Fatal("readAll() should return error")
}
if !strings.Contains(err.Error(), expectedErr.Error()) {
t.Errorf("readAll() error = %v, want %v", err, expectedErr)
}
})
}
// errorReader implements io.Reader that always returns an error
type errorReader struct {
err error
}
func (r *errorReader) Read(p []byte) (n int, err error) {
return 0, r.err
}
// ---------- Client tests ----------
func TestNewClient(t *testing.T) {
tests := []struct {
name string
rawURL string
wantErr bool
}{
{"basic FTP URL", "ftp://ftp.example.com/pub/file.txt", false},
{"FTP URL with path", "ftp://ftp.example.com/pub/", false},
{"FTP URL root", "ftp://ftp.example.com", false},
{"FTP URL with trailing slash", "ftp://ftp.example.com/", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client, err := NewClient(tt.rawURL, 30*time.Second)
if tt.wantErr {
if err == nil {
t.Errorf("NewClient(%q) expected error", tt.rawURL)
}
return
}
if err != nil {
t.Fatalf("NewClient(%q) returned error: %v", tt.rawURL, err)
}
if client == nil {
t.Fatal("NewClient() returned nil")
}
})
}
}
func TestNewClientWithAuth(t *testing.T) {
client, err := NewClient("ftp://user:password@ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.user != "user" {
t.Errorf("client.user = %q, want %q", client.user, "user")
}
if client.password != "password" {
t.Errorf("client.password = %q, want %q", client.password, "password")
}
}
func TestNewClientWithUserOnly(t *testing.T) {
client, err := NewClient("ftp://user@ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.user != "user" {
t.Errorf("client.user = %q, want %q", client.user, "user")
}
if client.password != "anonymous@" {
t.Errorf("client.password should default to %q when only username is provided, got %q", "anonymous@", client.password)
}
}
func TestNewClientFTPS(t *testing.T) {
client, err := NewClient("ftps://ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if !client.useTLS {
t.Errorf("client.useTLS should be true for ftps:// scheme")
}
}
func TestNewClientFTPSWithAuth(t *testing.T) {
client, err := NewClient("ftps://alice:secret@ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if !client.useTLS {
t.Errorf("client.useTLS should be true for ftps:// scheme")
}
if client.user != "alice" {
t.Errorf("client.user = %q, want %q", client.user, "alice")
}
if client.password != "secret" {
t.Errorf("client.password = %q, want %q", client.password, "secret")
}
}
func TestNewClientInvalidURL(t *testing.T) {
// Only inputs where url.Parse itself fails
_, err := NewClient("://invalid", 30*time.Second)
if err == nil {
t.Errorf("NewClient(%q) expected error", "://invalid")
}
}
func TestNewClientEmptyURL(t *testing.T) {
client, err := NewClient("", 30*time.Second)
if err != nil {
t.Fatalf("NewClient(\"\") returned error: %v", err)
}
// url.Parse("") returns an empty URL struct, so host will be empty
if client.host != "" {
t.Errorf("client.host = %q, want empty", client.host)
}
if client.port != 21 {
t.Errorf("client.port = %d, want %d", client.port, 21)
}
}
func TestNewClientNoScheme(t *testing.T) {
client, err := NewClient("not-a-url", 30*time.Second)
if err != nil {
t.Fatalf("NewClient(\"not-a-url\") returned error: %v", err)
}
// url.Parse("not-a-url") parses it as a path with empty host
if client.host != "" {
t.Errorf("client.host = %q, want empty", client.host)
}
if client.port != 21 {
t.Errorf("client.port = %d, want %d", client.port, 21)
}
}
func TestNewClientDefaultPort(t *testing.T) {
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.port != 21 {
t.Errorf("client.port = %d, want %d", client.port, 21)
}
}
func TestNewClientCustomPort(t *testing.T) {
client, err := NewClient("ftp://ftp.example.com:2121/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.port != 2121 {
t.Errorf("client.port = %d, want %d", client.port, 2121)
}
}
func TestNewClientInvalidPort(t *testing.T) {
_, err := NewClient("ftp://ftp.example.com:invalid/file.txt", 30*time.Second)
if err == nil {
t.Errorf("NewClient() expected error for invalid port")
}
}
func TestNewClientDefaultCredentials(t *testing.T) {
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.user != "anonymous" {
t.Errorf("default user = %q, want %q", client.user, "anonymous")
}
if client.password != "anonymous@" {
t.Errorf("default password = %q, want %q", client.password, "anonymous@")
}
}
func TestNewClientHostParsing(t *testing.T) {
client, err := NewClient("ftp://ftp.example.com/path/to/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.host != "ftp.example.com" {
t.Errorf("client.host = %q, want %q", client.host, "ftp.example.com")
}
}
func TestNewClientPassiveModeDefault(t *testing.T) {
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if !client.passive {
t.Errorf("client.passive should default to true")
}
}
func TestNewClientTLSConfig(t *testing.T) {
client, err := NewClient("ftps://secure.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
if client.tlsConfig == nil {
t.Fatal("client.tlsConfig should not be nil")
}
if client.tlsConfig.ServerName != "secure.example.com" {
t.Errorf("tlsConfig.ServerName = %q, want %q", client.tlsConfig.ServerName, "secure.example.com")
}
if client.tlsConfig.InsecureSkipVerify {
t.Errorf("tlsConfig.InsecureSkipVerify should be false by default")
}
}
+220
View File
@@ -0,0 +1,220 @@
//go:build linux || freebsd
// +build linux freebsd
package ftp
import (
"context"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
)
// Protocol implements the core.Protocol interface for FTP/FTPS
type Protocol struct{}
// NewProtocol creates a new instance of FTP protocol
func NewProtocol() *Protocol {
return &Protocol{}
}
// Scheme returns the protocol scheme
func (p *Protocol) Scheme() string {
return "ftp"
}
// CanHandle checks whether the protocol can handle the given URL
func (p *Protocol) CanHandle(u *url.URL) bool {
if u == nil {
return false
}
scheme := strings.ToLower(u.Scheme)
return scheme == "ftp" || scheme == "ftps"
}
// Capabilities returns the list of supported features
func (p *Protocol) Capabilities() []string {
return []string{
"download",
"resume",
"recursive",
"tls",
}
}
// SupportsResume returns whether the protocol supports resume
func (p *Protocol) SupportsResume() bool {
return true
}
// SupportsCompression returns whether the protocol supports compression
func (p *Protocol) SupportsCompression() bool {
return false
}
// SupportsParallel returns whether the protocol supports parallel downloading
func (p *Protocol) SupportsParallel() bool {
return false
}
// SupportsRecursive returns whether the protocol supports recursive downloading
func (p *Protocol) SupportsRecursive() bool {
return true
}
// Download downloads a file or directory via FTP/FTPS
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
timeout := req.Timeout
if timeout <= 0 {
timeout = 30 * time.Second
}
client, err := NewClient(req.URL.String(), timeout)
if err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("failed to create FTP client: %v", err), nil, core.SafeURL(req.URL))
}
if err := client.Connect(); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("failed to connect: %v", err), nil, core.SafeURL(req.URL))
}
defer client.Close()
startTime := time.Now()
var bytesDownloaded int64 = 0
var chunksCount int
// Determine whether it is a file or directory
remotePath := req.URL.Path
if remotePath == "" {
remotePath = "/"
}
// Try to get file size - if it fails, it's probably a directory
fileSize, err := client.getFileSize(remotePath)
isFile := (err == nil)
if isFile {
// Downloading file
var outputFile string
if req.Output != "" {
outputFile = req.Output
} else {
outputFile = filepath.Base(remotePath)
if outputFile == "" || outputFile == "/" {
outputFile = "index.html"
}
}
// Kontrola resume
var resumeOffset int64 = 0
if req.Resume && req.Output != "" {
if canResume, meta, _ := output.CanResume(req.Output, req.URL.String()); canResume && meta != nil {
resumeOffset = meta.Downloaded
if req.Verbose {
fmt.Fprintf(os.Stderr, "Resuming from offset %d\n", resumeOffset)
}
}
}
// Progress callback
var progressCallback func(current, total int64)
if req.ProgressCallback != nil && fileSize > 0 {
totalSize := fileSize
startOffset := resumeOffset
progressCallback = func(current, _ int64) {
req.ProgressCallback(startOffset+current, totalSize, 0)
}
}
if err := client.DownloadFile(ctx, remotePath, outputFile, resumeOffset, progressCallback); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("download failed: %v", err), nil, core.SafeURL(req.URL))
}
// Get the actual size of the downloaded file
if info, err := os.Stat(outputFile); err == nil {
bytesDownloaded = info.Size()
}
chunksCount = 1
} else {
// Recursive download of directory
if !req.Recursive {
return nil, core.NewProtocolError("remote path is a directory, use --recursive to download", nil, core.SafeURL(req.URL))
}
outputDir := req.Output
if outputDir == "" {
outputDir = filepath.Base(remotePath)
if outputDir == "" || outputDir == "/" {
outputDir = "download"
}
}
var progressCallback func(file string, current, total int64)
if req.Verbose {
progressCallback = func(file string, current, total int64) {
fmt.Fprintf(os.Stderr, "Downloading: %s\n", file)
}
}
// Dispatch to the parallel implementation when the user
// requested worker-pool concurrency, otherwise fall back to the
// single-connection sequential implementation. The parallel path
// opens N control connections to avoid stepping on CWD state and
// to keep the existing client-side protocol serialisation.
if req.RecursiveParallel > 1 {
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, progressCallback); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
}
} else {
if err := client.DownloadDirectory(ctx, remotePath, outputDir, progressCallback); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
}
}
// Calculate total size and file count
filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
bytesDownloaded += info.Size()
chunksCount++
}
return nil
})
}
duration := time.Since(startTime)
return &core.DownloadResult{
BytesDownloaded: bytesDownloaded,
Duration: duration,
ChunksCount: chunksCount,
}, nil
}
// Helper function for reading from io.Reader
func readAll(reader io.Reader) ([]byte, error) {
var data []byte
buf := make([]byte, 32*1024)
for {
n, err := reader.Read(buf)
if n > 0 {
data = append(data, buf[:n]...)
}
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
}
return data, nil
}
+62
View File
@@ -0,0 +1,62 @@
//go:build linux || freebsd
// +build linux freebsd
package ftp
import (
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/output"
)
// TestSaveFTPResume verifies the helper persists partial FTP progress
// to a .goget.meta sidecar and that the sidecar is suitable for a
// subsequent goget --resume invocation. The function is called from
// Client.DownloadFile when ctx is cancelled mid-transfer; without a
// working sidecar the user's interrupted download cannot be continued.
func TestSaveFTPResume(t *testing.T) {
tmp := t.TempDir()
out := filepath.Join(tmp, "file.bin")
// 1. First call records partial progress.
if err := saveFTPResume("ftp://example.com/file.bin", out, 4096, 1<<20); err != nil {
t.Fatalf("saveFTPResume: %v", err)
}
meta, err := output.LoadResumeMetadata(out)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata, got nil")
}
if meta.URL != "ftp://example.com/file.bin" {
t.Errorf("URL = %q, want ftp://example.com/file.bin", meta.URL)
}
if meta.Downloaded != 4096 {
t.Errorf("Downloaded = %d, want 4096", meta.Downloaded)
}
if meta.Total != 1<<20 {
t.Errorf("Total = %d, want %d", meta.Total, 1<<20)
}
// 2. Second call overwrites the sidecar with a later snapshot.
if err := saveFTPResume("ftp://example.com/file.bin", out, 8192, 1<<20); err != nil {
t.Fatalf("saveFTPResume (2): %v", err)
}
meta2, _ := output.LoadResumeMetadata(out)
if meta2 == nil || meta2.Downloaded != 8192 {
t.Errorf("second call did not update sidecar: %+v", meta2)
}
// 3. Zero-byte save is a no-op (no point persisting a sidecar
// for an empty transfer).
out2 := filepath.Join(tmp, "empty.bin")
if err := saveFTPResume("ftp://x", out2, 0, 100); err != nil {
t.Errorf("zero-byte save returned error: %v", err)
}
if _, err := os.Stat(out2 + output.ResumeMetaFileSuffix); !os.IsNotExist(err) {
t.Errorf("expected no sidecar for zero-byte save, got one")
}
}
+203
View File
@@ -0,0 +1,203 @@
//go:build linux || freebsd
// +build linux freebsd
package gemini
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/url"
"os"
"strconv"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Protocol implements gemini:// downloads.
type Protocol struct {
*protocol.BaseProtocol
TLSInsecure bool // allow self-signed certs (testing only)
}
// maxGeminiRedirects limits redirect chain depth to prevent stack overflow.
const maxGeminiRedirects = 10
// NewProtocol creates a new Gemini protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "GEMINI",
Scheme: "gemini",
DefaultPort: 1965,
Features: []string{},
}),
}
}
// Download starts a Gemini download. It handles redirect limits internally.
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
return p.downloadWithRedirects(ctx, req, 0)
}
func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.DownloadRequest, redirectCount int) (*core.DownloadResult, error) {
startTime := time.Now()
host := req.URL.Hostname()
port := req.URL.Port()
if port == "" {
port = "1965"
}
addr := net.JoinHostPort(host, port)
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: host,
InsecureSkipVerify: p.TLSInsecure,
}
dialer := &net.Dialer{Timeout: 30 * time.Second}
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsCfg)
if err != nil {
return nil, core.NewNetworkError("failed to connect to gemini server", err, core.SafeURL(req.URL))
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(30 * time.Second))
// Send request: URL + CRLF
requestURL := req.URL.String()
if _, err := fmt.Fprintf(conn, "%s\r\n", requestURL); err != nil {
return nil, core.NewNetworkError("failed to send gemini request", err, core.SafeURL(req.URL))
}
// Read response header
reader := bufio.NewReader(conn)
header, err := reader.ReadString('\n')
if err != nil {
return nil, core.NewNetworkError("failed to read gemini response header", err, core.SafeURL(req.URL))
}
header = strings.TrimSpace(header)
// Parse status code (first two chars)
if len(header) < 2 {
return nil, core.NewProtocolError("invalid gemini response header", nil, core.SafeURL(req.URL))
}
statusCode, _ := strconv.Atoi(header[:2])
meta := ""
if len(header) > 3 {
meta = header[3:]
}
switch {
case statusCode >= 10 && statusCode < 20:
// 1x INPUT — server requests input (prompt user with meta)
return nil, core.NewProtocolError(
fmt.Sprintf("gemini input required: %s", meta), nil, core.SafeURL(req.URL))
case statusCode >= 30 && statusCode < 40:
// 3x REDIRECT — follow the redirect
if meta == "" {
return nil, core.NewProtocolError("gemini redirect with no target URL", nil, core.SafeURL(req.URL))
}
if redirectCount >= maxGeminiRedirects {
return nil, core.NewProtocolError(
fmt.Sprintf("gemini redirect chain exceeded %d", maxGeminiRedirects), nil, core.SafeURL(req.URL))
}
redirectURL, err := url.Parse(meta)
if err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("invalid gemini redirect URL: %s", meta), err, core.SafeURL(req.URL))
}
// Resolve relative redirects
resolved := req.URL.ResolveReference(redirectURL)
if req.Verbose {
fmt.Fprintf(os.Stderr, "[gemini] redirecting to: %s\n", resolved.String())
}
// Follow redirect with incremented depth
redirectReq := &core.DownloadRequest{
URL: resolved,
Output: req.Output,
Verbose: req.Verbose,
Writer: req.Writer,
Ctx: ctx,
}
return p.downloadWithRedirects(ctx, redirectReq, redirectCount+1)
case statusCode >= 40 && statusCode < 50:
// 4x TEMPORARY FAILURE
return nil, core.NewNetworkError(
fmt.Sprintf("gemini temporary failure: %d %s", statusCode, meta),
nil, core.SafeURL(req.URL))
case statusCode >= 50 && statusCode < 60:
// 5x PERMANENT FAILURE
return nil, core.NewProtocolError(
fmt.Sprintf("gemini permanent failure: %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
case statusCode >= 60 && statusCode < 70:
// 6x CLIENT CERTIFICATE REQUIRED
return nil, core.NewProtocolError(
fmt.Sprintf("gemini client certificate required: %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
case statusCode < 20 || statusCode >= 30:
return nil, core.NewProtocolError(
fmt.Sprintf("gemini server returned %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
}
// Category 2x: success — continue to read body
var totalRead int64
var writer io.Writer
if req.Writer != nil {
writer = req.Writer
} else {
writer = io.Discard
}
buf := make([]byte, 32*1024)
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
n, err := reader.Read(buf)
if n > 0 {
if _, werr := writer.Write(buf[:n]); werr != nil {
return nil, core.NewFileError("failed to write gemini data", werr)
}
totalRead += int64(n)
if req.ProgressCallback != nil {
speed := float64(totalRead) / time.Since(startTime).Seconds()
req.ProgressCallback(totalRead, -1, speed)
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, core.NewNetworkError("failed to read gemini response body", err, core.SafeURL(req.URL))
}
}
duration := time.Since(startTime)
speed := float64(totalRead) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: totalRead,
TotalSize: totalRead,
Duration: duration,
Protocol: "GEMINI",
IPVersion: 4,
Speed: speed,
OutputPath: req.Output,
}, nil
}
+134
View File
@@ -0,0 +1,134 @@
//go:build linux || freebsd
// +build linux freebsd
package gemini
import (
"bufio"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"math/big"
"net"
"net/url"
"strconv"
"strings"
"testing"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
)
type stringWriter struct{ sb *strings.Builder }
func (w *stringWriter) Write(p []byte) (int, error) { return w.sb.Write(p) }
func testTLSConfig() *tls.Config {
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "localhost"},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour),
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
}
certDER, _ := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
return &tls.Config{
Certificates: []tls.Certificate{{
Certificate: [][]byte{certDER},
PrivateKey: key,
}},
}
}
func TestGeminiSuccess(t *testing.T) {
cfg := testTLSConfig()
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("20 text/gemini\r\n# Hello\n\nContent.\n"))
}()
proto := NewProtocol()
proto.TLSInsecure = true
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
var buf strings.Builder
_, err = proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Writer: &stringWriter{&buf},
})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(buf.String(), "Hello") {
t.Errorf("got %q", buf.String())
}
}
func TestGeminiTemporaryFailure(t *testing.T) {
cfg := testTLSConfig()
listener, _ := tls.Listen("tcp", "127.0.0.1:0", cfg)
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("40 overloaded\r\n"))
}()
proto := NewProtocol()
proto.TLSInsecure = true
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
_, err := proto.Download(context.Background(), &core.DownloadRequest{URL: u})
if err == nil {
t.Error("expected temporary failure error")
}
}
func TestGeminiPermanentFailure(t *testing.T) {
cfg := testTLSConfig()
listener, _ := tls.Listen("tcp", "127.0.0.1:0", cfg)
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("51 not found\r\n"))
}()
proto := NewProtocol()
proto.TLSInsecure = true
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
_, err := proto.Download(context.Background(), &core.DownloadRequest{URL: u})
if err == nil {
t.Error("expected permanent failure error")
}
}
+384
View File
@@ -0,0 +1,384 @@
//go:build linux || freebsd
// +build linux freebsd
package gopher
import (
"bufio"
"context"
"fmt"
"io"
"net"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Gopher item types (RFC 1436).
const (
typeTextFile = '0'
typeDirectory = '1'
typeError = '3'
typeBinaryFile = '9'
typeGIF = 'g'
typeImage = 'I'
typeHTML = 'h'
typeInformation = 'i'
typeSound = 's'
typeTelnet = '8'
typeSearch = '7' // index search
)
// Protocol implements gopher:// downloads (RFC 1436).
type Protocol struct {
*protocol.BaseProtocol
}
// NewProtocol creates a new Gopher protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "GOPHER",
Scheme: "gopher",
DefaultPort: 70,
Features: []string{},
}),
}
}
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
startTime := time.Now()
host := req.URL.Hostname()
port := req.URL.Port()
if port == "" {
port = "70"
}
selector := req.URL.Path
if req.URL.RawQuery != "" {
selector += "?" + req.URL.RawQuery
}
if selector == "" {
selector = "/"
}
addr := net.JoinHostPort(host, port)
conn, err := net.DialTimeout("tcp", addr, 30*time.Second)
if err != nil {
return nil, core.NewNetworkError("failed to connect to gopher server", err, core.SafeURL(req.URL))
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(30 * time.Second))
// Send selector + CRLF
if _, err := fmt.Fprintf(conn, "%s\r\n", selector); err != nil {
return nil, core.NewNetworkError("failed to send gopher selector", err, core.SafeURL(req.URL))
}
// Peek at first byte to determine response type
reader := bufio.NewReader(conn)
firstByte, err := reader.Peek(1)
if err != nil {
return nil, core.NewNetworkError("failed to read gopher response", err, core.SafeURL(req.URL))
}
itemType := firstByte[0]
switch itemType {
case typeError:
// Type 3: error — read the error message
line, err := reader.ReadString('\n')
if err != nil && err != io.EOF {
return nil, core.NewNetworkError("failed to read gopher error", err, core.SafeURL(req.URL))
}
msg := strings.TrimSpace(line)
if len(msg) > 2 {
msg = msg[2:] // strip type + first char
}
return nil, core.NewProtocolError("gopher server error: "+msg, nil, core.SafeURL(req.URL))
case typeDirectory:
// Type 1: directory listing — format as readable text
return p.downloadDirectory(reader, req, startTime)
case typeTextFile, typeHTML, typeInformation:
// Text-based types — strip item-type prefix, handle .\r\n terminator
return p.downloadText(ctx, reader, req, startTime, itemType)
case typeSearch:
// Type 7: index search — server expects a query
return nil, core.NewProtocolError(
"gopher search server requires a query (append ?query to URL)", nil, core.SafeURL(req.URL))
case typeTelnet:
// Type 8: telnet session — interactive, cannot download
return nil, core.NewProtocolError(
"gopher telnet session type cannot be downloaded", nil, core.SafeURL(req.URL))
default:
// Binary types (9, g, I, s, etc.) — raw data with .\r\n termination
// But first check which ones have the type-prefix-per-line format
if isTextGopherType(itemType) {
return p.downloadText(ctx, reader, req, startTime, itemType)
}
return p.downloadBinary(ctx, reader, req, startTime)
}
}
// isTextGopherType returns true for types that use line-prefix format.
func isTextGopherType(t byte) bool {
switch t {
case typeTextFile, typeDirectory, typeError, typeInformation, typeHTML:
return true
default:
return false
}
}
// downloadText handles text responses where each line has a type prefix.
func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time, firstType byte) (*core.DownloadResult, error) {
var writer io.Writer
if req.Writer != nil {
writer = req.Writer
} else {
writer = io.Discard
}
var totalRead int64
lineCount := 0
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
line, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return nil, core.NewNetworkError("failed to read gopher line", err, core.SafeURL(req.URL))
}
// End-of-transmission: line containing only ".\r\n"
if line == ".\r\n" || line == ".\n" {
break
}
// Strip item-type prefix if present (first char = type, second char is usually tab/space)
displayLine := line
if len(line) > 1 && line[1] != '.' {
// Full menu line: type + display string + tab + selector + tab + host + tab + port
if true {
// For text downloads, strip the type and metadata, show only display string + \n
clean := stripGopherMeta(line)
displayLine = clean + "\n"
}
}
// For informational lines (type i), don't strip anything — just pass through
if firstType == typeInformation {
displayLine = line
}
n, err := writer.Write([]byte(displayLine))
if err != nil {
return nil, core.NewFileError("failed to write gopher text", err)
}
totalRead += int64(n)
lineCount++
if req.ProgressCallback != nil {
speed := float64(totalRead) / time.Since(startTime).Seconds()
req.ProgressCallback(totalRead, -1, speed)
}
}
duration := time.Since(startTime)
speed := float64(totalRead) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: totalRead,
TotalSize: totalRead,
Duration: duration,
Protocol: "GOPHER",
IPVersion: 4,
Speed: speed,
OutputPath: req.Output,
}, nil
}
// downloadDirectory formats a directory listing as readable text.
func (p *Protocol) downloadDirectory(reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
var writer io.Writer
if req.Writer != nil {
writer = req.Writer
} else {
writer = io.Discard
}
var totalRead int64
var sb strings.Builder
for {
line, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return nil, core.NewNetworkError("failed to read gopher directory", err, core.SafeURL(req.URL))
}
if line == ".\r\n" || line == ".\n" {
break
}
// Parse gopher menu line: type + display string + tab + selector + tab + host + tab + port
formatted := formatGopherMenu(line)
sb.WriteString(formatted)
sb.WriteByte('\n')
}
output := sb.String()
n, err := writer.Write([]byte(output))
if err != nil {
return nil, core.NewFileError("failed to write gopher directory", err)
}
totalRead = int64(n)
duration := time.Since(startTime)
speed := float64(totalRead) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: totalRead,
TotalSize: totalRead,
Duration: duration,
Protocol: "GOPHER",
IPVersion: 4,
Speed: speed,
OutputPath: req.Output,
}, nil
}
// downloadBinary handles binary responses.
func (p *Protocol) downloadBinary(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
var writer io.Writer
if req.Writer != nil {
writer = req.Writer
} else {
writer = io.Discard
}
var totalRead int64
buf := make([]byte, 32*1024)
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
n, err := reader.Read(buf)
if n > 0 {
if _, werr := writer.Write(buf[:n]); werr != nil {
return nil, core.NewFileError("failed to write gopher binary data", werr)
}
totalRead += int64(n)
if req.ProgressCallback != nil {
speed := float64(totalRead) / time.Since(startTime).Seconds()
req.ProgressCallback(totalRead, -1, speed)
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, core.NewNetworkError("failed to read gopher binary", err, core.SafeURL(req.URL))
}
}
duration := time.Since(startTime)
speed := float64(totalRead) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: totalRead,
TotalSize: totalRead,
Duration: duration,
Protocol: "GOPHER",
IPVersion: 4,
Speed: speed,
OutputPath: req.Output,
}, nil
}
// stripGopherMeta removes the gopher metadata (tabs + selector + host + port) from a menu line.
func stripGopherMeta(line string) string {
line = strings.TrimRight(line, "\r\n")
if len(line) < 2 {
return line
}
// Format: Xdisplay string\tselector\thost\tport
// Everything before the first tab is the type + display string
tabIdx := strings.IndexByte(line, '\t')
if tabIdx <= 1 {
return line // no metadata
}
return line[1:tabIdx] // strip type char, keep display text
}
// formatGopherMenu formats a gopher menu line for human display.
func formatGopherMenu(line string) string {
line = strings.TrimRight(line, "\r\n")
if len(line) < 2 {
return line
}
itemType := line[0]
rest := line[1:]
// Split by tabs: display\0selector\0host\0port
parts := strings.Split(rest, "\t")
display := parts[0]
selector := ""
if len(parts) > 1 {
selector = parts[1]
}
icon := gopherIcon(itemType)
if selector != "" && selector != "(null)" {
return fmt.Sprintf("%s %s [%s]", icon, display, selector)
}
return fmt.Sprintf("%s %s", icon, display)
}
// gopherIcon returns an icon for a gopher item type.
func gopherIcon(t byte) string {
switch t {
case typeDirectory:
return "📁"
case typeTextFile:
return "📄"
case typeBinaryFile:
return "💾"
case typeGIF, typeImage:
return "🖼"
case typeHTML:
return "🌐"
case typeSound:
return "🔊"
case typeSearch:
return "🔍"
case typeTelnet:
return "🖥"
case typeInformation:
return "️"
default:
return "❓"
}
}
+110
View File
@@ -0,0 +1,110 @@
//go:build linux || freebsd
// +build linux freebsd
package gopher
import (
"bufio"
"context"
"net"
"net/url"
"strconv"
"strings"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
)
func TestFormatGopherMenu(t *testing.T) {
tests := []struct {
line string
expected string
}{
{"0About\t/about\tgopher.floodgap.com\t70\r\n", "📄 About [/about]"},
{"1Fun\t/fun\tgopher.floodgap.com\t70\r\n", "📁 Fun [/fun]"},
{"iWelcome to Gopher\t(null)\t(null)\t0\r\n", "️ Welcome to Gopher"},
{"hNews\t/news\tgopher.floodgap.com\t70\r\n", "🌐 News [/news]"},
}
for _, tc := range tests {
result := formatGopherMenu(tc.line)
if result != tc.expected {
t.Errorf("formatGopherMenu(%q) = %q, want %q", tc.line, result, tc.expected)
}
}
}
func TestStripGopherMeta(t *testing.T) {
result := stripGopherMeta("0About\t/about\thost\t70")
if result != "About" {
t.Errorf("got %q, want %q", result, "About")
}
}
func TestGopherIcon(t *testing.T) {
if i := gopherIcon('0'); i != "📄" {
t.Errorf("got %q", i)
}
if i := gopherIcon('1'); i != "📁" {
t.Errorf("got %q", i)
}
if i := gopherIcon('9'); i != "💾" {
t.Errorf("got %q", i)
}
if i := gopherIcon('h'); i != "🌐" {
t.Errorf("got %q", i)
}
if i := gopherIcon('7'); i != "🔍" {
t.Errorf("got %q", i)
}
}
func TestIsTextGopherType(t *testing.T) {
if !isTextGopherType('0') {
t.Error("0 should be text")
}
if isTextGopherType('9') {
t.Error("9 should not be text")
}
}
type stringWriter struct{ sb *strings.Builder }
func (w *stringWriter) Write(p []byte) (int, error) { return w.sb.Write(p) }
func TestGopherDownloadMock(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
reader := bufio.NewReader(conn)
reader.ReadString('\n') // selector
conn.Write([]byte("0Hello from gopher\t(null)\t(null)\t0\r\n"))
conn.Write([]byte(".\r\n"))
}()
proto := NewProtocol()
u, _ := url.Parse("gopher://127.0.0.1:" + strconv.Itoa(port) + "/")
var buf strings.Builder
_, err = proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Writer: &stringWriter{&buf},
})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(buf.String(), "Hello from gopher") {
t.Errorf("output missing text: %q", buf.String())
}
}
File diff suppressed because it is too large Load Diff
+107
View File
@@ -0,0 +1,107 @@
//go:build linux || freebsd
// +build linux freebsd
package http
import (
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
"codeberg.org/petrbalvin/goget/internal/hsts"
"codeberg.org/petrbalvin/goget/internal/transport"
)
// TransportConfig groups network, TLS, proxy, and connection settings.
type TransportConfig struct {
// Connection
Timeout time.Duration
ConnectTimeout time.Duration
MaxTime time.Duration
MaxFileSize int64
BufferSize int
FollowRedirects bool
MaxRedirects int
NoClobber bool
// TLS and certificates
TLS *transport.TLSConfig
CACertFile string
HSTSCache *hsts.Cache
// Proxy and dialer
Proxy *transport.ProxyConfig
Dialer *transport.DialerConfig
BindInterface string
// DNS and IP filtering
DNSServers []string
BlockPrivateIPs bool
// Retry
Retry *transport.RetryConfig
MaxRetries int
RetryDelay time.Duration
RetryHTTPCodes []int
RetryAllErrors bool
}
// AuthConfig groups authentication settings.
type AuthConfig struct {
Username string
Password string
AuthType string // "basic", "digest", "auto"
OAuth *auth.OAuthConfig
}
// RecursiveConfig groups recursive download and mirroring settings.
type RecursiveConfig struct {
// Core recursion
Enabled bool
MaxDepth int
FollowExternal bool
ExcludePatterns []string
Accept string
Reject []string
// Parallel worker count for recursive downloads. 0 falls back to
// Parallel.Connections (or the default). Applies to all recursive
// protocols that support parallel file processing.
Parallel int
// Scope control
NoParent bool
SpanHosts bool
PageRequisites bool
Domains []string
// Rate limiting for recursion
Wait time.Duration
RandomWait bool
// Output control
CutDirs int
AdjustExt bool
DryRun bool
// Mirror mode
Mirror bool
MirrorAssets bool
ConvertLinks bool
RespectRobots bool
}
// OutputConfig groups output, progress, and display settings.
type OutputConfig struct {
OutputDir string
Verbose bool
JSONOutput bool
ShowHeaders bool
ContentDisposition bool
KeepTimestamps bool
WriteOut string
TraceTime bool
OnComplete string
WarcFile string
AutoExtract bool
ExtractDir string
}
+179
View File
@@ -0,0 +1,179 @@
//go:build linux || freebsd
// +build linux freebsd
package http
import (
"fmt"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
)
// URLInfo contains extracted information from an HTTP URL
type URLInfo struct {
Scheme string
Host string
Port string
Path string
Query string
Fragment string
User *url.Userinfo
}
// Handler contains HTTP-specific logic
type Handler struct {
client *Client
}
// NewHandler creates new handler
func NewHandler(client *Client) *Handler {
return &Handler{client: client}
}
// ParseHTTPURL parses an HTTP URL and extracts information
func ParseHTTPURL(u *url.URL) (*URLInfo, error) {
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("invalid http url scheme: %s", u.Scheme)
}
host := u.Hostname()
port := u.Port()
if port == "" {
if u.Scheme == "https" {
port = "443"
} else {
port = "80"
}
}
return &URLInfo{
Scheme: u.Scheme,
Host: host,
Port: port,
Path: u.Path,
Query: u.RawQuery,
Fragment: u.Fragment,
User: u.User,
}, nil
}
// GetContentType extracts Content-Type from the response
func GetContentType(resp *http.Response) string {
return resp.Header.Get("Content-Type")
}
// GetContentLength extracts Content-Length from the response
func GetContentLength(resp *http.Response) int64 {
return resp.ContentLength
}
// GetContentEncoding extracts Content-Encoding from the response
func GetContentEncoding(resp *http.Response) string {
return resp.Header.Get("Content-Encoding")
}
// GetETag extracts ETag from the response
func GetETag(resp *http.Response) string {
return resp.Header.Get("ETag")
}
// GetLastModified extracts Last-Modified from the response
func GetLastModified(resp *http.Response) string {
return resp.Header.Get("Last-Modified")
}
// SupportsRangeCheck checks whether the server supports range requests
func SupportsRangeCheck(resp *http.Response) bool {
acceptRanges := resp.Header.Get("Accept-Ranges")
return strings.ToLower(acceptRanges) == "bytes"
}
// IsRedirect checks whether the response is a redirect
func IsRedirect(statusCode int) bool {
return statusCode >= 300 && statusCode < 400
}
// GetRedirectLocation gets the Location header from a redirect response
func GetRedirectLocation(resp *http.Response) string {
return resp.Header.Get("Location")
}
// BuildURLWithQuery creates a URL with query parameters
func BuildURLWithQuery(base *url.URL, params map[string]string) (*url.URL, error) {
if base == nil {
return nil, fmt.Errorf("base url cannot be nil")
}
result := *base
query := result.Query()
for key, value := range params {
query.Set(key, value)
}
result.RawQuery = query.Encode()
return &result, nil
}
// GetUserinfo extracts authentication credentials from the URL
func GetUserinfo(u *url.URL) (username, password string, ok bool) {
if u.User == nil {
return "", "", false
}
username = u.User.Username()
password, ok = u.User.Password()
return username, password, true
}
// IsSecure checks whether the URL uses HTTPS
func IsSecure(u *url.URL) bool {
return u.Scheme == "https"
}
// GetDefaultPortForScheme returns the default port for the scheme
func GetDefaultPortForScheme(scheme string) string {
switch scheme {
case "https":
return "443"
case "http":
return "80"
default:
return ""
}
}
// ParseContentDisposition extracts filename from Content-Disposition header
func ParseContentDisposition(headerValue string) string {
if headerValue == "" {
return ""
}
_, params, err := mime.ParseMediaType(headerValue)
if err != nil {
return ""
}
// Try filename* first (RFC 5987, extended filename)
if filename, ok := params["filename*"]; ok {
return filename
}
// Fall back to filename
if filename, ok := params["filename"]; ok {
return filename
}
return ""
}
// SanitizeFilename removes path components from a filename for security
func SanitizeFilename(name string) string {
name = filepath.Base(name)
if name == "" || name == "." || name == ".." {
return ""
}
return name
}
File diff suppressed because it is too large Load Diff
+559
View File
@@ -0,0 +1,559 @@
//go:build linux || freebsd
// +build linux freebsd
package http
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/transport"
)
// testWriter implements io.Writer for testing
type testWriter struct {
buf strings.Builder
}
func (w *testWriter) Write(p []byte) (int, error) {
return w.buf.Write(p)
}
func TestDownloadSequentialIntegration(t *testing.T) {
// Create a test server that returns known content
testContent := "Hello, this is test content for download integration testing!"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
w.WriteHeader(http.StatusOK)
w.Write([]byte(testContent))
}))
defer server.Close()
// Parse the server URL
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("Failed to parse server URL: %v", err)
}
// Create the HTTP client
client, err := NewClient(DefaultConfig())
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
// Create download request
req := &core.DownloadRequest{
URL: parsedURL,
Output: "",
Writer: &testWriter{},
Verbose: false,
Headers: make(map[string]string),
}
// Execute download
result, err := client.Download(context.Background(), req)
if err != nil {
t.Fatalf("Download failed: %v", err)
}
if result.BytesDownloaded <= 0 {
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
}
if result.BytesDownloaded != int64(len(testContent)) {
t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded)
}
}
func TestDownloadParallelIntegration(t *testing.T) {
// Create test content for parallel download
testContent := strings.Repeat("B", 1024*1024) // 1MB
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Support range requests
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Type", "application/octet-stream")
rangeHeader := r.Header.Get("Range")
if rangeHeader != "" {
var start, end int64
n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
if n >= 1 {
if end == 0 || end >= int64(len(testContent)) {
end = int64(len(testContent)) - 1
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent)))
w.WriteHeader(http.StatusPartialContent)
w.Write([]byte(testContent[start : end+1]))
return
}
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
w.WriteHeader(http.StatusOK)
w.Write([]byte(testContent))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("Failed to parse server URL: %v", err)
}
cfg := DefaultConfig()
cfg.Parallel = &core.ParallelConfig{
Connections: 4,
MinSize: 100, // Force parallel for small files
MinChunkSize: 100,
MaxChunkSize: 1024 * 1024,
}
client, err := NewClient(cfg)
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
req := &core.DownloadRequest{
URL: parsedURL,
Output: "",
Writer: &testWriter{},
Verbose: false,
Headers: make(map[string]string),
Parallel: cfg.Parallel,
}
result, err := client.Download(context.Background(), req)
if err != nil {
t.Fatalf("Download failed: %v", err)
}
if result.BytesDownloaded <= 0 {
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
}
if !result.Parallel {
t.Log("Download was not parallel (may be expected depending on server negotiation)")
}
}
func TestDownloadResumeIntegration(t *testing.T) {
testContent := "This is content for resume testing!"
var requestCount int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestCount++
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("ETag", "\"test-etag-123\"")
rangeHeader := r.Header.Get("Range")
if rangeHeader != "" {
var start int64
fmt.Sscanf(rangeHeader, "bytes=%d-", &start)
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent)))
w.WriteHeader(http.StatusPartialContent)
w.Write([]byte(testContent[start:]))
return
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
w.WriteHeader(http.StatusOK)
w.Write([]byte(testContent))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("Failed to parse server URL: %v", err)
}
client, err := NewClient(DefaultConfig())
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
// Create resume info simulating a partially downloaded file
resumeInfo := &core.ResumeInfo{
DownloadedBytes: 10,
ETag: "\"test-etag-123\"",
URL: server.URL,
}
req := &core.DownloadRequest{
URL: parsedURL,
Output: "",
Writer: &testWriter{},
Resume: true,
ResumeInfo: resumeInfo,
Verbose: false,
Headers: make(map[string]string),
}
result, err := client.Download(context.Background(), req)
if err != nil {
t.Fatalf("Download failed: %v", err)
}
if result.BytesDownloaded <= 0 {
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
}
if result.Resumed {
t.Log("Download was resumed successfully")
}
}
func TestUploadIntegration(t *testing.T) {
var receivedBody string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Failed to read body", http.StatusInternalServerError)
return
}
receivedBody = string(body)
w.WriteHeader(http.StatusOK)
w.Write([]byte("OK"))
}))
defer server.Close()
client, err := NewClient(DefaultConfig())
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
// Test POST upload with form data
formReq := &UploadRequest{
URL: server.URL,
Method: "POST",
FormData: map[string]string{"key": "value"},
Timeout: 0,
}
result, err := client.Upload(context.Background(), formReq)
if err != nil {
t.Fatalf("Upload failed: %v", err)
}
if result.StatusCode != http.StatusOK {
t.Errorf("Expected status 200, got %d", result.StatusCode)
}
if result.BytesUploaded <= 0 {
t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded)
}
if !strings.Contains(receivedBody, "key=value") {
t.Errorf("Expected body to contain form data, got: %s", receivedBody)
}
}
func TestDownloadFollowsRedirectIntegration(t *testing.T) {
const finalBody = "redirected content"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/redirect" {
http.Redirect(w, r, "/target", http.StatusFound)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(finalBody))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL + "/redirect")
if err != nil {
t.Fatalf("parse url: %v", err)
}
client, err := NewClient(DefaultConfig())
if err != nil {
t.Fatalf("create client: %v", err)
}
result, err := client.Download(context.Background(), &core.DownloadRequest{
URL: parsedURL,
Writer: &testWriter{},
Headers: map[string]string{},
})
if err != nil {
t.Fatalf("download: %v", err)
}
if result.BytesDownloaded != int64(len(finalBody)) {
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
}
}
// TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a
// server that always answers 500. It verifies the loop actually attempts the
// configured number of retries and that each attempt's response body is
// closed — guarding against the regression where http.Client.Do returned a
// non-nil resp alongside an error and the body was never closed, exhausting
// the connection pool.
func TestRetryAllErrorsClosesBody(t *testing.T) {
var requestCount int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&requestCount, 1)
w.WriteHeader(http.StatusInternalServerError)
// Write a non-empty body so a leaking response body would matter
// (an empty body is closed trivially by net/http's chunked reader).
_, _ = w.Write([]byte("server error"))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("parse url: %v", err)
}
cfg := DefaultConfig()
cfg.Transport.RetryAllErrors = true
cfg.Transport.MaxRetries = 3
cfg.Transport.Retry = &transport.RetryConfig{
MaxRetries: 3,
InitialDelay: time.Millisecond,
}
client, err := NewClient(cfg)
if err != nil {
t.Fatalf("create client: %v", err)
}
_, err = client.Download(context.Background(), &core.DownloadRequest{
URL: parsedURL,
Writer: &testWriter{},
Headers: map[string]string{},
})
if err == nil {
t.Fatal("expected error after exhausting retries on 500 responses")
}
// We expect at least 1 (initial transport.Retry attempt) + MaxRetries
// (RetryAllErrors loop). The exact number may be higher because Go's
// HTTP transport may also retry once on its own when an attempt fails
// (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert
// the lower bound only — the goal is to confirm the retry loop runs and
// that every failed response body is closed, not to pin a Go-internal
// detail.
minRequests := int32(1 + cfg.Transport.MaxRetries)
if got := atomic.LoadInt32(&requestCount); got < minRequests {
t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries)
}
}
// TestUploadRateLimiterAndProgressAreChained is a regression guard for the
// BACKLOG entry "Rate limiter silently dropped when progress wrapper
// enabled". The two wrappers must compose — httpReq.Body must end up
// reading through the rate-limited reader, with progress reported on top.
func TestUploadRateLimiterAndProgressAreChained(t *testing.T) {
const payload = "the quick brown fox jumps over the lazy dog"
var receivedBody string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "read error", http.StatusInternalServerError)
return
}
receivedBody = string(body)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
// Write payload to a temp file (Upload's PUT path opens the file).
tmpDir := t.TempDir()
tmpFile := filepath.Join(tmpDir, "payload.txt")
if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil {
t.Fatalf("write tmp: %v", err)
}
cfg := DefaultConfig()
// Generous rate limit so the test isn't dominated by the limiter.
// limitedReader.Read asks for len(p) tokens on every call, and the
// HTTP transport uses a 32 KB buffer, so burst must comfortably exceed
// that or Allow() blocks forever.
cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024)
cfg.Transport.MaxRetries = 0
cfg.Transport.Retry = &transport.RetryConfig{
MaxRetries: 0,
InitialDelay: time.Millisecond,
}
client, err := NewClient(cfg)
if err != nil {
t.Fatalf("create client: %v", err)
}
progressCalls := 0
var lastCurrent int64
progressCb := func(current, total int64, _ float64) {
progressCalls++
if current < lastCurrent {
t.Errorf("progress went backwards: %d -> %d", lastCurrent, current)
}
lastCurrent = current
}
_, err = client.Upload(context.Background(), &UploadRequest{
URL: server.URL,
Method: "PUT",
FilePath: tmpFile,
ProgressCallback: progressCb,
})
if err != nil {
t.Fatalf("upload: %v", err)
}
// The server must receive the full, unmodified payload. If the rate
// limiter wrapper was silently dropped (e.g. httpReq.Body not updated
// after body was reassigned), the server would still get the data via
// the original reader — so this check alone doesn't expose the bug.
if receivedBody != payload {
t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload)
}
// Progress callback must fire at least once and reach the full payload
// length. The exact call count depends on the read-buffer behaviour of
// net/http (a 43-byte payload may be read in a single Read), but a count
// of zero would prove the progress reader was bypassed — which is the
// exact regression the BACKLOG entry warns about.
if progressCalls < 1 {
t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls)
}
if lastCurrent != int64(len(payload)) {
t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload))
}
}
// TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few
// 500s short-circuits the retry loop and produces a successful download —
// another code path that must close every preceding failed response body
// before jumping to okStatus.
func TestRetryAllErrorsRecoversOn200(t *testing.T) {
const finalBody = "ok after retries"
var requestCount int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&requestCount, 1)
if n < 3 {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte("server error"))
return
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody)))
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(finalBody))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("parse url: %v", err)
}
cfg := DefaultConfig()
cfg.Transport.RetryAllErrors = true
cfg.Transport.MaxRetries = 5
cfg.Transport.Retry = &transport.RetryConfig{
MaxRetries: 5,
InitialDelay: time.Millisecond,
}
client, err := NewClient(cfg)
if err != nil {
t.Fatalf("create client: %v", err)
}
result, err := client.Download(context.Background(), &core.DownloadRequest{
URL: parsedURL,
Writer: &testWriter{},
Headers: map[string]string{},
})
if err != nil {
t.Fatalf("download: %v", err)
}
if result.BytesDownloaded != int64(len(finalBody)) {
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
}
if got := atomic.LoadInt32(&requestCount); got != 3 {
t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got)
}
}
// TestTraceTimeTTFBOnReusedConnection is a regression guard for the
// BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on
// reused connections". The previous implementation measured TTFB as
// time.Since(dnsStart); on a pooled HTTP connection the DNSStart
// callback is silent, dnsStart stayed zero, and the result was ~55 years
// instead of a plausible request-time duration. The fix anchors TTFB to
// a requestStart timestamp captured before the trace is attached, so
// the value is meaningful whether or not the connection is reused.
func TestTraceTimeTTFBOnReusedConnection(t *testing.T) {
var connectionCount int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&connectionCount, 1)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("hello"))
}))
defer server.Close()
parsedURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("Failed to parse server URL: %v", err)
}
cfg := DefaultConfig()
cfg.Output.TraceTime = true
client, err := NewClient(cfg)
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
reqTemplate := func() *core.DownloadRequest {
return &core.DownloadRequest{
URL: parsedURL,
Writer: &testWriter{},
Headers: make(map[string]string),
}
}
// First request opens the connection — DNSStart/ConnectStart/
// TLSHandshakeStart all fire, populating the trace timestamps.
if _, err := client.Download(context.Background(), reqTemplate()); err != nil {
t.Fatalf("first download failed: %v", err)
}
// Second request should be served from the pool. Whether or not
// keep-alive actually reuses the connection in this test depends
// on the transport's MaxIdleConnsPerHost, but the assertion is the
// same: TTFB must be a plausible duration in either case.
result, err := client.Download(context.Background(), reqTemplate())
if err != nil {
t.Fatalf("second download failed: %v", err)
}
if result.TraceTimings == nil {
t.Fatal("TraceTimings is nil despite TraceTime=true")
}
// 1 hour is a generous upper bound that flags the original ~55-year
// regression (time.Since(time.Time{}) on a fresh time.Time) without
// being flaky on a busy CI runner.
if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour {
t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb)
}
if ttfb := result.TraceTimings.TTFB; ttfb < 0 {
t.Errorf("TTFB = %v; want non-negative", ttfb)
}
}
+323
View File
@@ -0,0 +1,323 @@
//go:build linux || freebsd
// +build linux freebsd
package http
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
)
func (c *Client) downloadParallel(ctx context.Context, req *core.DownloadRequest, totalSize int64, connections int, resumeChunks map[int]int64, username, password string) (*core.DownloadResult, error) {
startTime := time.Now()
chunks := createChunks(totalSize, connections, c.config.Parallel)
if c.config.Output.Verbose {
fmt.Fprintf(os.Stderr, "[parallel] Split into %d chunks\n", len(chunks))
}
tempDir, err := os.MkdirTemp("", "goget-chunks-*")
if err != nil {
return nil, core.NewFileError("failed to create temp dir", err)
}
defer os.RemoveAll(tempDir)
// completedChunks tracks per-chunk downloaded bytes for resume metadata.
completedChunks := make(map[int]int64)
var completedChunksMu sync.Mutex
for id, n := range resumeChunks {
completedChunks[id] = n
}
progressChan := make(chan int64, len(chunks)*2)
var wg sync.WaitGroup
var firstError error
var errorMu sync.Mutex
for i := range chunks {
chunk := &chunks[i]
wg.Add(1)
go func(chunk *core.ChunkInfo) {
defer wg.Done()
if resumeChunks != nil {
if done, ok := resumeChunks[chunk.ID]; ok && done >= (chunk.End-chunk.Start+1) {
progressChan <- done
chunk.Status = 2
return
}
}
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
err := c.downloadChunk(ctx, req, chunk, chunkPath, progressChan, username, password)
if err != nil {
errorMu.Lock()
if firstError == nil {
firstError = err
}
errorMu.Unlock()
chunk.Status = 3
chunk.Error = err
return
}
// Save per-chunk progress for resume on interruption.
completedChunksMu.Lock()
completedChunks[chunk.ID] = chunk.End - chunk.Start + 1
saveParallelResumeMetadata(req, totalSize, completedChunks)
completedChunksMu.Unlock()
chunk.Status = 2
}(chunk)
}
go func() {
wg.Wait()
close(progressChan)
}()
var totalDownloaded int64
var cancelled bool
aggregationLoop:
for {
select {
case progress, ok := <-progressChan:
if !ok {
break aggregationLoop
}
totalDownloaded += progress
if req.ProgressCallback != nil {
duration := time.Since(startTime)
speed := float64(totalDownloaded) / duration.Seconds()
req.ProgressCallback(totalDownloaded, totalSize, speed)
}
case <-ctx.Done():
cancelled = true
break aggregationLoop
}
}
if cancelled {
// Context was cancelled (SIGINT/SIGTERM). Return the error so
// the caller can show "cancelled" instead of "complete", and
// the progress bar renders the interrupted state. Partial
// chunks are NOT merged — they remain in temp files for a
// potential resume later.
return nil, ctx.Err()
}
if firstError != nil {
return nil, firstError
}
if req.Output != "" && req.Output != "-" {
if err := mergeChunks(req.Output, chunks, tempDir); err != nil {
return nil, core.NewFileError("failed to merge chunks", err)
}
}
if req.Output != "" {
output.DeleteResumeMetadata(req.Output)
}
duration := time.Since(startTime)
speed := float64(0)
if duration.Seconds() > 0 {
speed = float64(totalSize) / duration.Seconds()
}
return &core.DownloadResult{
BytesDownloaded: totalSize,
TotalSize: totalSize,
Duration: duration,
Speed: speed,
Protocol: "HTTP/HTTPS",
IPVersion: 6,
OutputPath: req.Output,
Parallel: true,
ChunksCount: len(chunks),
}, nil
}
// saveParallelResumeMetadata saves per-chunk progress to the resume metadata file.
func saveParallelResumeMetadata(req *core.DownloadRequest, totalSize int64, chunks map[int]int64) {
if req.Output == "" || req.Output == "-" {
return
}
// Build a snapshot of the current chunk progress.
snapshot := make(map[int]int64, len(chunks))
for id, n := range chunks {
snapshot[id] = n
}
meta := &output.ResumeMetadata{
URL: req.URL.String(),
Total: totalSize,
Downloaded: totalChunkBytes(snapshot),
LastWrite: time.Now(),
Version: "1.0",
Chunks: snapshot,
}
_ = meta.Save(req.Output)
}
// totalChunkBytes sums the byte count of all completed chunks.
func totalChunkBytes(chunks map[int]int64) int64 {
var total int64
for _, n := range chunks {
total += n
}
return total
}
func createChunks(totalSize int64, count int, cfg *core.ParallelConfig) []core.ChunkInfo {
if count <= 1 || totalSize <= cfg.MinSize {
return []core.ChunkInfo{
{ID: 0, Start: 0, End: totalSize - 1},
}
}
// Divide the file into `count` equal-sized chunks. The last chunk
// absorbs any remainder. MaxChunkSize is NOT used to further
// subdivide — the user explicitly asked for N parallel connections
// via --parallel, not N×M tiny range requests. Each chunk becomes
// one HTTP Range request handled by one of the N worker goroutines.
chunkSize := totalSize / int64(count)
if chunkSize < cfg.MinChunkSize {
chunkSize = cfg.MinChunkSize
count = int((totalSize + chunkSize - 1) / chunkSize)
}
chunks := make([]core.ChunkInfo, 0, count)
for i := 0; i < count; i++ {
start := int64(i) * chunkSize
end := start + chunkSize - 1
if i == count-1 || end >= totalSize {
end = totalSize - 1
}
chunks = append(chunks, core.ChunkInfo{
ID: i,
Start: start,
End: end,
})
}
return chunks
}
func (c *Client) downloadChunk(ctx context.Context, req *core.DownloadRequest, chunk *core.ChunkInfo, outputPath string, progressChan chan<- int64, username, password string) error {
chunk.Status = 1
httpReq, err := http.NewRequestWithContext(ctx, "GET", req.URL.String(), nil)
if err != nil {
return core.NewNetworkError("failed to create chunk request", err, req.URL.String())
}
httpReq.Header.Set("User-Agent", c.config.UserAgent)
httpReq.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", chunk.Start, chunk.End))
if req.ResumeInfo != nil {
if req.ResumeInfo.ETag != "" {
httpReq.Header.Set("If-Range", req.ResumeInfo.ETag)
} else if req.ResumeInfo.LastModified != "" {
httpReq.Header.Set("If-Range", req.ResumeInfo.LastModified)
}
}
if username != "" {
auth.BasicAuth(httpReq, username, password)
}
for key, value := range req.Headers {
httpReq.Header.Set(key, value)
}
resp, err := c.httpClient.Do(httpReq)
if err != nil {
return core.NewNetworkError("failed to fetch chunk", err, req.URL.String())
}
defer resp.Body.Close()
c.updateHSTS(resp)
if resp.StatusCode != http.StatusPartialContent {
return core.NewProtocolError(
fmt.Sprintf("chunk HTTP error: %s", resp.Status),
nil,
req.URL.String(),
)
}
file, err := os.Create(outputPath)
if err != nil {
return core.NewFileError("failed to create chunk file", err)
}
defer file.Close()
reader := io.Reader(resp.Body)
if c.config.RateLimiter != nil {
reader = c.config.RateLimiter.WrapReader(reader)
}
buf := make([]byte, c.config.Transport.BufferSize)
var downloaded int64
for {
if ctx.Err() != nil {
return ctx.Err()
}
n, err := reader.Read(buf)
if n > 0 {
_, wErr := file.Write(buf[:n])
if wErr != nil {
return core.NewFileError("failed to write chunk", wErr)
}
downloaded += int64(n)
if progressChan != nil {
progressChan <- int64(n)
}
}
if err == io.EOF {
break
}
if err != nil {
return core.NewNetworkError("failed to read chunk", err, req.URL.String())
}
}
chunk.Downloaded = downloaded
return nil
}
func mergeChunks(outputPath string, chunks []core.ChunkInfo, tempDir string) error {
outFile, err := os.Create(outputPath)
if err != nil {
return err
}
defer outFile.Close()
for _, chunk := range chunks {
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
chunkFile, err := os.Open(chunkPath)
if err != nil {
return fmt.Errorf("failed to open chunk %d: %w", chunk.ID, err)
}
_, err = io.Copy(outFile, chunkFile)
chunkFile.Close()
if err != nil {
return fmt.Errorf("failed to copy chunk %d: %w", chunk.ID, err)
}
}
return nil
}
+379
View File
@@ -0,0 +1,379 @@
//go:build linux || freebsd
// +build linux freebsd
package http
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
format "codeberg.org/petrbalvin/goget/internal/format"
)
// UploadRequest represents an upload request
type UploadRequest struct {
// URL to upload to
URL string
// Method: PUT or POST
Method string
// Path to the file for upload
FilePath string
// Form data fields (for POST)
FormData map[string]string
// Files for upload (for POST multipart)
Files []UploadFile
// Custom headers
Headers map[string]string
// Timeout
Timeout time.Duration
// Callback pro progress
ProgressCallback func(current, total int64, speed float64)
}
// UploadFile represents a file for multipart upload
type UploadFile struct {
// Field name in the form data
FieldName string
// Path to the file
FilePath string
// Custom file name (optional)
FileName string
// Content-Type (optional)
ContentType string
}
// UploadResult is the result of an upload
type UploadResult struct {
StatusCode int
ContentLength int64
Duration time.Duration
BytesUploaded int64
Speed float64
ResponseBody string
}
// Upload uploads a file using PUT or POST
func (c *Client) Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error) {
startTime := time.Now()
if req.Method == "" {
req.Method = "PUT"
}
// Create request body
var body io.Reader
var contentType string
var totalSize int64
if req.Method == "PUT" {
// PUT upload - direct file
if req.FilePath == "" {
return nil, fmt.Errorf("put upload requires --upload-file")
}
fileInfo, err := os.Stat(req.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to stat file: %w", err)
}
totalSize = fileInfo.Size()
file, err := os.Open(req.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
body = file
contentType = "application/octet-stream"
} else if req.Method == "POST" {
// POST upload - form data or multipart
if len(req.Files) > 0 {
// Multipart form data
var bodyBuffer bytes.Buffer
writer := multipart.NewWriter(&bodyBuffer)
// Add regular form fields
for key, value := range req.FormData {
if err := writer.WriteField(key, value); err != nil {
return nil, fmt.Errorf("failed to write form field: %w", err)
}
}
// Add files
for _, uf := range req.Files {
file, err := os.Open(uf.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to open file %s: %w", uf.FilePath, err)
}
fileName := uf.FileName
if fileName == "" {
fileName = filepath.Base(uf.FilePath)
}
contentType := uf.ContentType
if contentType == "" {
contentType = "application/octet-stream"
}
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"; filename="%s"`, uf.FieldName, fileName))
h.Set("Content-Type", contentType)
part, err := writer.CreatePart(h)
if err != nil {
file.Close()
return nil, fmt.Errorf("failed to create form part: %w", err)
}
fileInfo, _ := os.Stat(uf.FilePath)
totalSize += fileInfo.Size()
_, err = io.Copy(part, file)
file.Close()
if err != nil {
return nil, fmt.Errorf("failed to write file to form: %w", err)
}
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
}
body = &bodyBuffer
contentType = writer.FormDataContentType()
} else if len(req.FormData) > 0 {
// URL-encoded form data
form := url.Values{}
for key, value := range req.FormData {
form.Set(key, value)
}
body = strings.NewReader(form.Encode())
contentType = "application/x-www-form-urlencoded"
totalSize = int64(len(form.Encode()))
} else if req.FilePath != "" {
// POST with raw file body
fileInfo, err := os.Stat(req.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to stat file: %w", err)
}
totalSize = fileInfo.Size()
file, err := os.Open(req.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
body = file
contentType = "application/octet-stream"
}
}
if body == nil {
return nil, fmt.Errorf("no upload data provided")
}
// Create HTTP request
httpReq, err := http.NewRequestWithContext(ctx, req.Method, req.URL, body)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
// Set headers
httpReq.Header.Set("Content-Type", contentType)
httpReq.Header.Set("User-Agent", c.config.UserAgent)
for key, value := range req.Headers {
httpReq.Header.Set(key, value)
}
// Apply authentication
if c.config.Auth.Username != "" {
authType := c.config.Auth.AuthType
if authType == "" {
authType = "auto"
}
if authType == "auto" || authType == "basic" {
// Try basic auth first
applyBasicAuth(httpReq, c.config.Auth.Username, c.config.Auth.Password)
}
}
// Apply OAuth token if configured
if c.config.Auth.OAuth != nil && c.config.Auth.OAuth.AccessToken != "" {
oauthClient := auth.NewOAuthClient(c.config.Auth.OAuth)
if oauthClient.IsTokenValid() {
oauthClient.ApplyToken(httpReq)
}
}
// Execute request
if c.config.RateLimiter != nil {
// Wrap body with rate limiter for upload
body = c.config.RateLimiter.WrapReader(body)
httpReq.Body = io.NopCloser(body)
}
// Track progress
if req.ProgressCallback != nil && totalSize > 0 {
body = &progressReader{
r: body,
callback: req.ProgressCallback,
total: totalSize,
start: startTime,
offset: 0,
}
httpReq.Body = io.NopCloser(body)
}
resp, err := c.httpClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("upload request failed: %w", err)
}
defer resp.Body.Close()
// Read response body
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
duration := time.Since(startTime)
return &UploadResult{
StatusCode: resp.StatusCode,
ContentLength: resp.ContentLength,
Duration: duration,
BytesUploaded: totalSize,
Speed: float64(totalSize) / duration.Seconds(),
ResponseBody: string(respBody),
}, nil
}
// UploadSimple performs a simple file upload with progress tracking
func (c *Client) UploadSimple(ctx context.Context, method, targetURL, filePath string, verbose bool) (*UploadResult, error) {
fileInfo, err := os.Stat(filePath)
if err != nil {
return nil, fmt.Errorf("failed to stat file: %w", err)
}
totalSize := fileInfo.Size()
_ = totalSize // Used in progress callback
file, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
var progressCallback func(int64, int64, float64)
if verbose {
progressCallback = func(current, total int64, speed float64) {
percent := float64(current) / float64(total) * 100
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
format.Bytes(current),
format.Bytes(total),
percent,
format.Speed(speed))
}
}
req := &UploadRequest{
URL: targetURL,
Method: method,
FilePath: filePath,
Timeout: c.config.Transport.Timeout,
ProgressCallback: progressCallback,
Headers: make(map[string]string),
}
// Copy headers from client config
for k, v := range c.config.Headers {
req.Headers[k] = v
}
result, err := c.Upload(ctx, req)
if err != nil {
return nil, err
}
if verbose {
fmt.Fprintf(os.Stderr, "\n")
}
return result, nil
}
// UploadMultipart performs a multipart POST upload with files
func (c *Client) UploadMultipart(ctx context.Context, targetURL string, files []UploadFile, formData map[string]string, verbose bool) (*UploadResult, error) {
var progressCallback func(int64, int64, float64)
if verbose {
progressCallback = func(current, total int64, speed float64) {
percent := float64(current) / float64(total) * 100
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
format.Bytes(current),
format.Bytes(total),
percent,
format.Speed(speed))
}
}
req := &UploadRequest{
URL: targetURL,
Method: "POST",
Files: files,
FormData: formData,
Timeout: c.config.Transport.Timeout,
ProgressCallback: progressCallback,
Headers: make(map[string]string),
}
for k, v := range c.config.Headers {
req.Headers[k] = v
}
result, err := c.Upload(ctx, req)
if err != nil {
return nil, err
}
if verbose {
fmt.Fprintf(os.Stderr, "\n")
}
return result, nil
}
// applyBasicAuth applies HTTP Basic Authentication to request
func applyBasicAuth(req *http.Request, username, password string) {
req.SetBasicAuth(username, password)
}
+151
View File
@@ -0,0 +1,151 @@
//go:build linux || freebsd
// +build linux freebsd
package protocol
import (
"fmt"
"net/url"
"codeberg.org/petrbalvin/goget/internal/core"
)
// ProtocolInfo contains metadata about a protocol
type ProtocolInfo struct {
// Protocol name
Name string
// URL scheme
Scheme string
// Protocol version
Version string
// Supported operations
Operations []string
// Supported features
Features []string
// Default port
DefaultPort int
}
// BaseProtocol is the base implementation for all protocols
type BaseProtocol struct {
info ProtocolInfo
}
// NewBaseProtocol creates a new base protocol
func NewBaseProtocol(info ProtocolInfo) *BaseProtocol {
return &BaseProtocol{info: info}
}
// Scheme returns the protocol scheme
func (bp *BaseProtocol) Scheme() string {
return bp.info.Scheme
}
// Name returns the protocol name
func (bp *BaseProtocol) Name() string {
return bp.info.Name
}
// Capabilities returns the list of supported features
func (bp *BaseProtocol) Capabilities() []string {
return bp.info.Features
}
// SupportsResume returns whether the protocol supports resume
func (bp *BaseProtocol) SupportsResume() bool {
return contains(bp.info.Features, "resume")
}
// SupportsCompression returns whether the protocol supports compression
func (bp *BaseProtocol) SupportsCompression() bool {
return contains(bp.info.Features, "compression")
}
// SupportsUpload returns whether the protocol supports upload
func (bp *BaseProtocol) SupportsUpload() bool {
return contains(bp.info.Features, "upload")
}
// SupportsRange returns whether the protocol supports range requests
func (bp *BaseProtocol) SupportsRange() bool {
return contains(bp.info.Features, "range")
}
// CanHandle checks if this protocol can handle the given URL.
func (bp *BaseProtocol) CanHandle(url *url.URL) bool {
if url == nil {
return false
}
return url.Scheme == bp.info.Scheme
}
// SupportsParallel returns whether the protocol supports parallel downloads.
func (bp *BaseProtocol) SupportsParallel() bool {
return contains(bp.info.Features, "parallel")
}
// SupportsRecursive returns whether the protocol supports recursive downloads.
func (bp *BaseProtocol) SupportsRecursive() bool {
return contains(bp.info.Features, "recursive")
}
// contains is a helper function
func contains(slice []string, item string) bool {
for _, s := range slice {
if s == item {
return true
}
}
return false
}
// ValidateURL validates a URL for the given protocol
func ValidateURL(u *url.URL, scheme string) error {
if u == nil {
return core.NewProtocolError("URL is nil", nil, "")
}
if u.Scheme != scheme {
return core.NewProtocolError(
fmt.Sprintf("expected scheme %s, got %s", scheme, u.Scheme),
nil,
u.String(),
)
}
if u.Host == "" {
return core.NewProtocolError("missing host in URL", nil, u.String())
}
return nil
}
// GetDefaultPort returns the default port for a scheme
func GetDefaultPort(scheme string) int {
switch scheme {
case "http":
return 80
case "https", "webdavs":
return 443
case "webdav":
// Plain WebDAV (no TLS) defaults to 80, matching HTTP.
// Previously this returned 443 — a leftover from bundling
// "webdav" with the TLS-bearing schemes — which made
// non-TLS WebDAV connections fail against servers not
// listening on 443.
return 80
case "ftp":
return 21
case "ftps":
return 990
case "sftp":
return 22
default:
return 0
}
}
File diff suppressed because it is too large Load Diff
+47
View File
@@ -0,0 +1,47 @@
//go:build linux || freebsd
// +build linux freebsd
// Package register provides a single registration point for all built-in goget protocols.
// Both CLI (cmd/goget) and library API (pkg/api) call RegisterAll to avoid
// duplicating the protocol registration list across packages.
package register
import (
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
"codeberg.org/petrbalvin/goget/internal/protocol/dataurl"
"codeberg.org/petrbalvin/goget/internal/protocol/file"
"codeberg.org/petrbalvin/goget/internal/protocol/ftp"
"codeberg.org/petrbalvin/goget/internal/protocol/gemini"
"codeberg.org/petrbalvin/goget/internal/protocol/gopher"
"codeberg.org/petrbalvin/goget/internal/protocol/http"
"codeberg.org/petrbalvin/goget/internal/protocol/sftp"
"codeberg.org/petrbalvin/goget/internal/protocol/webdav"
)
// RegisterAll registers all built-in protocols on the given registry.
// Protocols that are already registered are silently skipped, making this
// safe to call from both CLI and API initialization.
// Returns the HTTP protocol handler for further configuration (e.g., HSTS).
func RegisterAll(r *protocol.Registry) (*http.Client, error) {
registerIfMissing := func(p core.Protocol) {
if r.Supports(p.Scheme()) {
return
}
_ = r.Register(p)
}
httpClient, err := http.NewClient(http.DefaultConfig())
if err != nil {
return nil, err
}
registerIfMissing(httpClient)
registerIfMissing(ftp.NewProtocol())
registerIfMissing(file.NewProtocol())
registerIfMissing(dataurl.NewProtocol())
registerIfMissing(gopher.NewProtocol())
registerIfMissing(gemini.NewProtocol())
registerIfMissing(sftp.NewProtocol())
registerIfMissing(webdav.NewProtocol())
return httpClient, nil
}
+132
View File
@@ -0,0 +1,132 @@
//go:build linux || freebsd
// +build linux freebsd
package protocol
import (
"fmt"
"net/url"
"sort"
"strings"
"sync"
"codeberg.org/petrbalvin/goget/internal/core"
)
// Registry manages all registered protocols
type Registry struct {
mu sync.RWMutex
protocols map[string]core.Protocol
}
// NewRegistry creates a new registry
func NewRegistry() *Registry {
return &Registry{
protocols: make(map[string]core.Protocol),
}
}
// Register registers a protocol under its primary scheme
func (r *Registry) Register(p core.Protocol) error {
r.mu.Lock()
defer r.mu.Unlock()
scheme := p.Scheme()
if scheme == "" {
return fmt.Errorf("protocol scheme cannot be empty")
}
if _, exists := r.protocols[scheme]; exists {
return fmt.Errorf("protocol already registered: %s", scheme)
}
r.protocols[scheme] = p
return nil
}
// normalizeScheme normalizes a scheme for lookup (https -> http, webdavs -> webdav, etc.)
func normalizeScheme(scheme string) string {
scheme = strings.ToLower(scheme)
switch scheme {
case "https":
return "http"
case "webdavs":
return "webdav"
case "ftps":
return "ftp"
default:
return scheme
}
}
// Get gets a protocol for a given URL with scheme normalization
func (r *Registry) Get(u *url.URL) (core.Protocol, error) {
r.mu.RLock()
defer r.mu.RUnlock()
if u == nil {
return nil, fmt.Errorf("url cannot be nil")
}
scheme := normalizeScheme(u.Scheme)
p, exists := r.protocols[scheme]
if !exists {
return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme)
}
// Additional validation via CanHandle
if !p.CanHandle(u) {
return nil, fmt.Errorf("protocol %s cannot handle url: %s", scheme, u.String())
}
return p, nil
}
// List returns a list of all registered protocols
func (r *Registry) List() []string {
r.mu.RLock()
defer r.mu.RUnlock()
schemes := make([]string, 0, len(r.protocols))
for scheme := range r.protocols {
schemes = append(schemes, scheme)
}
sort.Strings(schemes)
return schemes
}
// Supports determines whether a protocol is supported (with normalization)
func (r *Registry) Supports(scheme string) bool {
r.mu.RLock()
defer r.mu.RUnlock()
_, exists := r.protocols[normalizeScheme(scheme)]
return exists
}
// Capabilities returns the capabilities of all protocols
func (r *Registry) Capabilities() map[string][]string {
r.mu.RLock()
defer r.mu.RUnlock()
caps := make(map[string][]string)
for scheme, p := range r.protocols {
caps[scheme] = p.Capabilities()
}
return caps
}
// GlobalRegistry is the global registry instance
var GlobalRegistry = NewRegistry()
// Register is a helper for registering in the global registry
func Register(p core.Protocol) error {
return GlobalRegistry.Register(p)
}
// Get is a helper for getting a protocol from the global registry
func Get(u *url.URL) (core.Protocol, error) {
return GlobalRegistry.Get(u)
}
+33
View File
@@ -0,0 +1,33 @@
//go:build linux || freebsd
// +build linux freebsd
package sftp
import (
"fmt"
"os"
"path/filepath"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, error) {
if insecure {
return ssh.InsecureIgnoreHostKey(), nil
}
if knownHostsPath == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, fmt.Errorf("known_hosts: %w", err)
}
knownHostsPath = filepath.Join(home, ".ssh", "known_hosts")
}
if _, err := os.Stat(knownHostsPath); err != nil {
return nil, fmt.Errorf("known_hosts file not found: %s (use --ssh-insecure to skip host key check)", knownHostsPath)
}
return knownhosts.New(knownHostsPath)
}
+25
View File
@@ -0,0 +1,25 @@
//go:build linux || freebsd
// +build linux freebsd
package sftp
import (
"testing"
)
func TestHostKeyCallbackInsecure(t *testing.T) {
cb, err := hostKeyCallback("", true)
if err != nil {
t.Fatalf("hostKeyCallback: %v", err)
}
if cb == nil {
t.Fatal("expected callback")
}
}
func TestHostKeyCallbackMissingFile(t *testing.T) {
_, err := hostKeyCallback("/nonexistent/known_hosts", false)
if err == nil {
t.Fatal("expected error for missing known_hosts")
}
}
@@ -0,0 +1,54 @@
//go:build linux || freebsd
// +build linux freebsd
package sftp
import (
"net/url"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
)
// TestSaveSFTPResume verifies the partial-progress sidecar for the
// SFTP single-file path. The SFTP parallel workers don't read attr.Size
// in the download loop, so Total is recorded as 0 (only Downloaded
// is known reliably). A future enhancement could plumb totalSize
// through sftpDownloadFile if more accurate accounting is needed.
func TestSaveSFTPResume(t *testing.T) {
tmp := t.TempDir()
out := filepath.Join(tmp, "file.bin")
u, err := url.Parse("sftp://user@host:22/file.bin")
if err != nil {
t.Fatalf("parse URL: %v", err)
}
req := &core.DownloadRequest{
URL: u,
Output: out,
}
saveSFTPResume(req, 12345)
meta, err := output.LoadResumeMetadata(out)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata, got nil")
}
if meta.Downloaded != 12345 {
t.Errorf("Downloaded = %d, want 12345", meta.Downloaded)
}
if meta.URL != "sftp://user@host:22/file.bin" {
t.Errorf("URL = %q", meta.URL)
}
// Total is 0 because the parallel worker doesn't read attr.Size;
// the resume code path uses this as "size unknown" and works
// regardless of Total.
if meta.Total != 0 {
t.Errorf("Total = %d, want 0 (size unknown for SFTP parallel workers)", meta.Total)
}
}
File diff suppressed because it is too large Load Diff
+623
View File
@@ -0,0 +1,623 @@
//go:build linux || freebsd
// +build linux freebsd
package sftp
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"encoding/binary"
"fmt"
"io"
"net"
"net/url"
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
"golang.org/x/crypto/ssh"
)
// startTestServer starts a minimal SSH+SFTP server for testing.
func startTestServer(t *testing.T) (addr string, cleanup func()) {
t.Helper()
config := &ssh.ServerConfig{
NoClientAuth: true,
}
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate key: %v", err)
}
signer, err := ssh.NewSignerFromKey(privateKey)
if err != nil {
t.Fatalf("new signer: %v", err)
}
config.AddHostKey(signer)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
done := make(chan struct{})
go func() {
defer close(done)
// Accept connections in a loop so the parallel-download tests
// (which open N SFTP sessions) can run. Each connection is
// served in its own goroutine, so the N pool connections can be
// alive concurrently as long as the test goroutine keeps
// accepting.
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func(conn net.Conn) {
defer conn.Close()
sshConn, chans, reqs, err := ssh.NewServerConn(conn, config)
if err != nil {
return
}
defer sshConn.Close()
go ssh.DiscardRequests(reqs)
for newChannel := range chans {
if newChannel.ChannelType() != "session" {
newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
continue
}
channel, requests, err := newChannel.Accept()
if err != nil {
return
}
go func(ch ssh.Channel, reqs <-chan *ssh.Request) {
for req := range reqs {
ok := false
if req.Type == "subsystem" && len(req.Payload) > 4 {
l := binary.BigEndian.Uint32(req.Payload[:4])
name := string(req.Payload[4 : 4+l])
if name == "sftp" {
ok = true
if req.WantReply {
req.Reply(ok, nil)
}
handleSFTPSession(ch)
return
}
}
if req.WantReply {
req.Reply(ok, nil)
}
}
ch.Close()
}(channel, requests)
}
}(conn)
}
}()
return listener.Addr().String(), func() {
listener.Close()
<-done
}
}
// handleSFTPSession handles a minimal SFTP v3 session.
func handleSFTPSession(ch ssh.Channel) {
defer ch.Close()
// Expect INIT
pkt, err := readTestPacket(ch)
if err != nil || len(pkt) < 5 || pkt[0] != sshFxpInit {
return
}
// Respond VERSION
var resp bytes.Buffer
resp.WriteByte(sshFxpVersion)
writeUint32(&resp, 3)
writeTestPacket(ch, resp.Bytes())
handles := make(map[string]string)
dirHandles := make(map[string][]string)
dirHandlePaths := make(map[string]string)
handleCounter := 0
for {
pkt, err := readTestPacket(ch)
if err != nil {
return
}
if len(pkt) < 1 {
continue
}
reqID, _ := readUint32(pkt, 1)
switch pkt[0] {
case sshFxpRealpath:
path, _ := readString(pkt, 5)
if path == "." {
path = "/home/test"
}
var r bytes.Buffer
r.WriteByte(sshFxpName)
writeUint32(&r, reqID)
writeUint32(&r, 1)
writeString(&r, path)
writeString(&r, path)
writeUint32(&r, 0) // attrs
writeTestPacket(ch, r.Bytes())
case sshFxpStat:
path, _ := readString(pkt, 5)
var r bytes.Buffer
r.WriteByte(sshFxpAttrs)
writeUint32(&r, reqID)
fs := testFSLookup(path)
if fs != nil && fs.isDir {
// mode flag only
writeUint32(&r, 0x4)
writeUint32(&r, 0x4000) // S_IFDIR
writeTestPacket(ch, r.Bytes())
continue
}
if fs != nil && !fs.isDir {
// size flag only
writeUint32(&r, 0x1)
writeUint64(&r, uint64(len(fs.content)))
writeTestPacket(ch, r.Bytes())
continue
}
// legacy single-file mock behaviour
writeUint32(&r, 0x1) // size flag
size := uint64(len(testFileContent))
if testOverrideAttrSize != nil {
size = *testOverrideAttrSize
}
writeUint64(&r, size)
writeTestPacket(ch, r.Bytes())
case sshFxpOpendir:
path, _ := readString(pkt, 5)
fs := testFSLookup(path)
if fs == nil || !fs.isDir {
status(ch, reqID, sshFxNoSuchFile, "no such directory")
continue
}
handleCounter++
h := fmt.Sprintf("dhandle%d", handleCounter)
dirHandles[h] = fs.children
dirHandlePaths[h] = path
var r bytes.Buffer
r.WriteByte(sshFxpHandle)
writeUint32(&r, reqID)
writeString(&r, h)
writeTestPacket(ch, r.Bytes())
case sshFxpReaddir:
handle, _ := readString(pkt, 5)
children, ok := dirHandles[handle]
if !ok {
status(ch, reqID, sshFxFailure, "no such dir handle")
continue
}
parentPath := dirHandlePaths[handle]
var r bytes.Buffer
r.WriteByte(sshFxpName)
writeUint32(&r, reqID)
writeUint32(&r, uint32(len(children)))
for _, name := range children {
writeString(&r, name)
writeString(&r, name)
childPath := parentPath + "/" + name
childFS := testFSLookup(childPath)
isDir := childFS != nil && childFS.isDir
// attrs: size flag (0x1) and mode flag (0x4) so the
// client can both classify the entry (dir vs file) and
// read the file size for parallel accounting.
if isDir {
writeUint32(&r, 0x4)
writeUint32(&r, 0x4000) // S_IFDIR
} else {
writeUint32(&r, 0x5) // size + mode
var sz uint64
if childFS != nil {
sz = uint64(len(childFS.content))
} else {
sz = uint64(len(testFileContent))
}
writeUint64(&r, sz)
writeUint32(&r, 0x8000) // S_IFREG
}
}
writeTestPacket(ch, r.Bytes())
case sshFxpOpen:
path, off := readString(pkt, 5)
pflags, _ := readUint32(pkt, off)
handleCounter++
h := fmt.Sprintf("handle%d", handleCounter)
handles[h] = path
var r bytes.Buffer
if pflags&sshFxfRead != 0 {
r.WriteByte(sshFxpHandle)
writeUint32(&r, reqID)
writeString(&r, h)
writeTestPacket(ch, r.Bytes())
} else if pflags&sshFxfWrite != 0 {
r.WriteByte(sshFxpHandle)
writeUint32(&r, reqID)
writeString(&r, h)
writeTestPacket(ch, r.Bytes())
} else {
status(ch, reqID, sshFxFailure, "unsupported flags")
}
case sshFxpRead:
handle, off := readString(pkt, 5)
offset, off := readUint64(pkt, off)
length, _ := readUint32(pkt, off)
_ = length
path := handles[handle]
if path == "" {
status(ch, reqID, sshFxNoSuchFile, "no such handle")
continue
}
var data []byte
if fs := testFSLookup(path); fs != nil && !fs.isDir {
data = fs.content
} else {
data = testFileContent
}
if int(offset) >= len(data) {
status(ch, reqID, sshFxEOF, "")
continue
}
end := int(offset) + int(length)
if end > len(data) {
end = len(data)
}
chunk := data[offset:end]
var r bytes.Buffer
r.WriteByte(sshFxpData)
writeUint32(&r, reqID)
writeUint32(&r, uint32(len(chunk)))
r.Write(chunk)
writeTestPacket(ch, r.Bytes())
case sshFxpClose:
handle, _ := readString(pkt, 5)
delete(handles, handle)
delete(dirHandles, handle)
var r bytes.Buffer
r.WriteByte(sshFxpStatus)
writeUint32(&r, reqID)
writeUint32(&r, sshFxOk)
writeString(&r, "")
writeString(&r, "")
writeTestPacket(ch, r.Bytes())
default:
status(ch, reqID, sshFxOpUnsupported, "unsupported")
}
}
}
// testFSLookup is a tiny helper that hides the nil-map check so the
// mock switch arms above can stay readable.
func testFSLookup(path string) *testFSEntry {
if testFS == nil {
return nil
}
return testFS[path]
}
func status(ch ssh.Channel, reqID, code uint32, msg string) {
var r bytes.Buffer
r.WriteByte(sshFxpStatus)
writeUint32(&r, reqID)
writeUint32(&r, code)
writeString(&r, msg)
writeString(&r, "")
writeTestPacket(ch, r.Bytes())
}
func readTestPacket(r io.Reader) ([]byte, error) {
var lenBuf [4]byte
if _, err := io.ReadFull(r, lenBuf[:]); err != nil {
return nil, err
}
length := binary.BigEndian.Uint32(lenBuf[:])
if length > 256*1024 {
return nil, fmt.Errorf("packet too large")
}
pkt := make([]byte, length)
if _, err := io.ReadFull(r, pkt); err != nil {
return nil, err
}
return pkt, nil
}
func writeTestPacket(w io.Writer, pkt []byte) error {
var lenBuf [4]byte
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(pkt)))
if _, err := w.Write(lenBuf[:]); err != nil {
return err
}
_, err := w.Write(pkt)
return err
}
var testFileContent = []byte("hello sftp world")
// testOverrideAttrSize, when non-nil, replaces the value reported by the
// mock server's sshFxpStat handler. Tests use it to simulate a server
// that returns a wrong (typically too large) size attribute so we can
// verify the client does not blindly trust attr.Size for
// BytesDownloaded accounting.
var testOverrideAttrSize *uint64
// testFSEntry describes a single in-memory file or directory used by
// the recursive-download tests. isDir==true entries carry a children
// slice; isDir==false entries carry the file content.
type testFSEntry struct {
isDir bool
content []byte
children []string
}
// testFS is an optional in-memory filesystem consulted by the mock
// server before falling back to the default testFileContent behaviour.
// Tests that exercise recursive downloads set it up and clear it via
// t.Cleanup. Keys are absolute paths; paths NOT present in the map
// keep the legacy single-file mock semantics so existing
// non-recursive tests continue to pass.
var testFS map[string]*testFSEntry
func TestDownload(t *testing.T) {
addr, cleanup := startTestServer(t)
defer cleanup()
u, _ := url.Parse("sftp://user@" + addr + "/testfile")
outDir := t.TempDir()
outputPath := filepath.Join(outDir, "out")
req := &core.DownloadRequest{
URL: u,
Output: outputPath,
Ctx: context.Background(),
SSHInsecure: true,
}
result, err := download(context.Background(), req)
if err != nil {
t.Fatalf("download failed: %v", err)
}
data, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read output: %v", err)
}
if !bytes.Equal(data, testFileContent) {
t.Errorf("got %q, want %q", data, testFileContent)
}
if result.BytesDownloaded != int64(len(testFileContent)) {
t.Errorf("bytes=%d, want %d", result.BytesDownloaded, len(testFileContent))
}
if result.Protocol != "sftp" {
t.Errorf("protocol=%q, want sftp", result.Protocol)
}
}
// TestDownloadWithStaleAttrSize is a regression guard for the BACKLOG
// entry "SFTP BytesDownloaded unconditionally overwritten". The previous
// implementation set result.BytesDownloaded = int64(attr.Size) when
// attr.Size > 0, ignoring the locally-tracked `total` counter. If the
// server reported a stale (typically too large) size, the result would
// over-report the bytes actually transferred — which is exactly what
// this test exercises.
func TestDownloadWithStaleAttrSize(t *testing.T) {
// Simulate a server that reports 10 KiB even though the file is
// only len(testFileContent) bytes long. After the fix, the result
// must reflect the actual bytes received, not the stale attribute.
wrong := uint64(10 * 1024)
testOverrideAttrSize = &wrong
t.Cleanup(func() { testOverrideAttrSize = nil })
addr, cleanup := startTestServer(t)
defer cleanup()
u, _ := url.Parse("sftp://user@" + addr + "/testfile")
outDir := t.TempDir()
outputPath := filepath.Join(outDir, "out")
req := &core.DownloadRequest{
URL: u,
Output: outputPath,
Ctx: context.Background(),
SSHInsecure: true,
}
result, err := download(context.Background(), req)
if err != nil {
t.Fatalf("download failed: %v", err)
}
// The downloaded file must still hold the real (small) payload.
data, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read output: %v", err)
}
if !bytes.Equal(data, testFileContent) {
t.Errorf("got %q, want %q", data, testFileContent)
}
// And BytesDownloaded must reflect the actual bytes received, not
// the stale attr.Size the server returned.
if result.BytesDownloaded != int64(len(testFileContent)) {
t.Errorf("BytesDownloaded = %d, want %d (real payload size, not stale attr.Size=%d)",
result.BytesDownloaded, len(testFileContent), wrong)
}
}
func TestParseHeader(t *testing.T) {
var buf bytes.Buffer
writeUint32(&buf, 0x05) // size + mode flags
writeUint64(&buf, 60)
writeUint32(&buf, 0644)
attr, off := readAttrs(buf.Bytes(), 0)
if attr.Size != 60 {
t.Errorf("size=%d, want 60", attr.Size)
}
if off != 16 {
t.Errorf("off=%d, want 16", off)
}
}
func TestIsEOF(t *testing.T) {
if !isEOF(fmt.Errorf("sftp status %d", sshFxEOF)) {
t.Error("expected true")
}
if isEOF(fmt.Errorf("sftp status %d", sshFxFailure)) {
t.Error("expected false")
}
}
func TestDialTimeout(t *testing.T) {
u, _ := url.Parse("sftp://user@127.0.0.1:1/")
_, err := dial(context.Background(), u, "", true)
if err == nil {
t.Fatal("expected error")
}
}
func TestSFTPRecursiveDownloadParallel(t *testing.T) {
addr, cleanup := startTestServer(t)
defer cleanup()
setupRecursiveFS := func(t *testing.T) {
t.Helper()
testFS = map[string]*testFSEntry{
"/home/test/dir": {isDir: true, children: []string{"a.txt", "b.txt", "c.txt", "sub"}},
"/home/test/dir/a.txt": {isDir: false, content: []byte("alpha")},
"/home/test/dir/b.txt": {isDir: false, content: []byte("bravo-bravo")},
"/home/test/dir/c.txt": {isDir: false, content: []byte("charlie")},
"/home/test/dir/sub": {isDir: true, children: []string{"x.txt", "y.txt"}},
"/home/test/dir/sub/x.txt": {isDir: false, content: []byte("x-ray")},
"/home/test/dir/sub/y.txt": {isDir: false, content: []byte("yankee")},
}
t.Cleanup(func() { testFS = nil })
}
t.Run("parallel=3 downloads all files", func(t *testing.T) {
setupRecursiveFS(t)
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
outDir := t.TempDir()
req := &core.DownloadRequest{
URL: u,
Output: outDir,
Recursive: true,
RecursiveParallel: 3,
SSHInsecure: true,
}
result, err := download(context.Background(), req)
if err != nil {
t.Fatalf("recursive parallel download failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5 (3 top-level files + 2 in subdir)", result.ChunksCount)
}
// 5 + 11 + 7 + 5 + 6 = 34 bytes total content
const wantBytes = int64(5 + 11 + 7 + 5 + 6)
if result.BytesDownloaded != wantBytes {
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
}
// Verify all 5 files actually exist with correct content.
wantFiles := map[string]string{
"a.txt": "alpha",
"b.txt": "bravo-bravo",
"c.txt": "charlie",
"sub/x.txt": "x-ray",
"sub/y.txt": "yankee",
}
for name, want := range wantFiles {
got, err := os.ReadFile(filepath.Join(outDir, name))
if err != nil {
t.Errorf("file %s: %v", name, err)
continue
}
if string(got) != want {
t.Errorf("file %s = %q, want %q", name, string(got), want)
}
}
})
t.Run("parallel=0 falls back to sequential", func(t *testing.T) {
setupRecursiveFS(t)
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
outDir := t.TempDir()
req := &core.DownloadRequest{
URL: u,
Output: outDir,
Recursive: true,
RecursiveParallel: 0,
SSHInsecure: true,
}
result, err := download(context.Background(), req)
if err != nil {
t.Fatalf("recursive sequential download failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
}
})
t.Run("parallel=1 falls back to sequential", func(t *testing.T) {
setupRecursiveFS(t)
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
outDir := t.TempDir()
req := &core.DownloadRequest{
URL: u,
Output: outDir,
Recursive: true,
RecursiveParallel: 1,
SSHInsecure: true,
}
result, err := download(context.Background(), req)
if err != nil {
t.Fatalf("download with parallel=1 failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
}
})
t.Run("subdir propagation with parallel", func(t *testing.T) {
setupRecursiveFS(t)
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
outDir := t.TempDir()
req := &core.DownloadRequest{
URL: u,
Output: outDir,
Recursive: true,
RecursiveParallel: 4,
SSHInsecure: true,
}
_, err := download(context.Background(), req)
if err != nil {
t.Fatalf("download failed: %v", err)
}
// Both subdir files should be present and correct.
for _, name := range []string{"sub/x.txt", "sub/y.txt"} {
path := filepath.Join(outDir, name)
if _, err := os.Stat(path); err != nil {
t.Errorf("expected file %s to exist: %v", name, err)
}
}
})
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,61 @@
//go:build linux || freebsd
// +build linux freebsd
package webdav
import (
"net/url"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
)
// TestSaveWebDAVResume verifies the partial-progress sidecar written
// by streamWebDAVBody on context cancellation. The ETag and
// Last-Modified headers are preserved so the next --resume attempt
// can send a matching If-Range and either continue or restart
// cleanly depending on the server's view of the file.
func TestSaveWebDAVResume(t *testing.T) {
tmp := t.TempDir()
out := filepath.Join(tmp, "file.bin")
u, _ := url.Parse("webdav://host/file.bin")
req := &core.DownloadRequest{
URL: u,
Output: out,
Resume: true,
}
resumed := true
saveWebDAVResume(req, 8192, 1<<20, `"etag-abc"`, "Mon, 01 Jan 2026 00:00:00 GMT", resumed)
meta, err := output.LoadResumeMetadata(out)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata, got nil")
}
if meta.Downloaded != 8192 {
t.Errorf("Downloaded = %d, want 8192", meta.Downloaded)
}
if meta.Total != 1<<20 {
t.Errorf("Total = %d, want %d", meta.Total, 1<<20)
}
if meta.ETag != `"etag-abc"` {
t.Errorf("ETag = %q", meta.ETag)
}
if meta.LastModified != "Mon, 01 Jan 2026 00:00:00 GMT" {
t.Errorf("LastModified = %q", meta.LastModified)
}
// Resume=false must short-circuit the save.
out2 := filepath.Join(tmp, "noresume.bin")
req.Resume = false
saveWebDAVResume(req, 4096, 1000, "", "", false)
if _, err := output.LoadResumeMetadata(out2); err != nil {
t.Fatalf("LoadResumeMetadata (noresume): %v", err)
}
}
File diff suppressed because it is too large Load Diff