feat: initial goget release — modern IPv6-first download utility
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package dataurl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
)
|
||||
|
||||
// Protocol handles data: URLs (RFC 2397).
|
||||
type Protocol struct {
|
||||
*protocol.BaseProtocol
|
||||
}
|
||||
|
||||
// NewProtocol creates a new data: URL protocol handler.
|
||||
func NewProtocol() *Protocol {
|
||||
return &Protocol{
|
||||
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
|
||||
Name: "DATA",
|
||||
Scheme: "data",
|
||||
DefaultPort: 0,
|
||||
Features: []string{},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
// data:[<mediatype>][;base64],<data>
|
||||
data, err := Decode(req.URL.String())
|
||||
if err != nil {
|
||||
return nil, core.NewProtocolError("failed to decode data URL", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
var writer io.Writer
|
||||
outputPath := req.Output
|
||||
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else if outputPath == "" || outputPath == "-" {
|
||||
writer = os.Stdout
|
||||
outputPath = "-"
|
||||
} else {
|
||||
f, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to create output file", err)
|
||||
}
|
||||
defer f.Close()
|
||||
writer = f
|
||||
}
|
||||
|
||||
n, err := writer.Write(data)
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to write data URL content", err)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: int64(n),
|
||||
TotalSize: int64(len(data)),
|
||||
Duration: duration,
|
||||
Protocol: "DATA",
|
||||
IPVersion: 0,
|
||||
Speed: float64(n) / duration.Seconds(),
|
||||
OutputPath: outputPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Decode parses and decodes a data: URL.
|
||||
func Decode(rawURL string) ([]byte, error) {
|
||||
if !strings.HasPrefix(rawURL, "data:") {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
content := strings.TrimPrefix(rawURL, "data:")
|
||||
isBase64 := false
|
||||
|
||||
if idx := strings.Index(content, ";base64,"); idx >= 0 {
|
||||
isBase64 = true
|
||||
content = content[idx+8:]
|
||||
} else if idx := strings.Index(content, ","); idx >= 0 {
|
||||
content = content[idx+1:]
|
||||
}
|
||||
|
||||
decoded, err := url.PathUnescape(content)
|
||||
if err != nil {
|
||||
decoded = content
|
||||
}
|
||||
|
||||
if isBase64 {
|
||||
return base64.StdEncoding.DecodeString(decoded)
|
||||
}
|
||||
return []byte(decoded), nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package dataurl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
func TestDecodePlainText(t *testing.T) {
|
||||
data, err := Decode("data:text/plain,hello%20world")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "hello world" {
|
||||
t.Errorf("got %q, want %q", data, "hello world")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBase64(t *testing.T) {
|
||||
data, err := Decode("data:text/plain;base64,SGVsbG8=")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "Hello" {
|
||||
t.Errorf("got %q, want %q", data, "Hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEmpty(t *testing.T) {
|
||||
data, err := Decode("data:,")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(data) != 0 {
|
||||
t.Errorf("expected empty, got %d bytes", len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeNotDataURL(t *testing.T) {
|
||||
data, err := Decode("https://example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data != nil {
|
||||
t.Error("expected nil for non-data URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPlainText(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
proto := NewProtocol()
|
||||
u, _ := url.Parse("data:text/plain,test123")
|
||||
result, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Writer: &buf,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.BytesDownloaded != 7 {
|
||||
t.Errorf("bytes = %d, want 7", result.BytesDownloaded)
|
||||
}
|
||||
if buf.String() != "test123" {
|
||||
t.Errorf("got %q, want %q", buf.String(), "test123")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadToFile(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
dst := filepath.Join(tmp, "out.txt")
|
||||
|
||||
proto := NewProtocol()
|
||||
u, _ := url.Parse("data:text/plain,hello")
|
||||
_, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: dst,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := os.ReadFile(dst)
|
||||
if string(data) != "hello" {
|
||||
t.Errorf("got %q, want %q", data, "hello")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
)
|
||||
|
||||
// Protocol implements file:// downloads.
|
||||
type Protocol struct {
|
||||
*protocol.BaseProtocol
|
||||
}
|
||||
|
||||
// NewProtocol creates a new file protocol handler.
|
||||
func NewProtocol() *Protocol {
|
||||
return &Protocol{
|
||||
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
|
||||
Name: "FILE",
|
||||
Scheme: "file",
|
||||
DefaultPort: 0,
|
||||
Features: []string{"resume", "recursive"},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
filePath := req.URL.Path
|
||||
if req.URL.Host != "" {
|
||||
filePath = filepath.Join("/", req.URL.Host, filePath)
|
||||
}
|
||||
filePath = filepath.Clean(filePath)
|
||||
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to stat source path", err)
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return p.downloadDirectory(ctx, req, filePath, startTime)
|
||||
}
|
||||
|
||||
src, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to open source file", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
totalSize := info.Size()
|
||||
var writer io.Writer
|
||||
var outputPath string
|
||||
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
outputPath = req.Output
|
||||
} else if req.Output == "" || req.Output == "-" {
|
||||
writer = os.Stdout
|
||||
outputPath = "-"
|
||||
} else {
|
||||
dst, err := os.Create(req.Output)
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to create output file", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
writer = dst
|
||||
outputPath = req.Output
|
||||
}
|
||||
|
||||
var written int64
|
||||
if req.Resume && outputPath != "" && outputPath != "-" {
|
||||
if fi, err := os.Stat(outputPath); err == nil {
|
||||
written = fi.Size()
|
||||
if written >= totalSize {
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalSize,
|
||||
TotalSize: totalSize,
|
||||
Duration: time.Since(startTime),
|
||||
Protocol: "FILE",
|
||||
OutputPath: outputPath,
|
||||
Resumed: true,
|
||||
}, nil
|
||||
}
|
||||
if _, err := src.Seek(written, io.SeekStart); err != nil {
|
||||
return nil, core.NewFileError("failed to seek for resume", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
n, err := src.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := writer.Write(buf[:n]); werr != nil {
|
||||
return nil, core.NewFileError("failed to write data", werr)
|
||||
}
|
||||
written += int64(n)
|
||||
if req.ProgressCallback != nil {
|
||||
speed := float64(written) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(written, totalSize, speed)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to read source file", err)
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(written) / duration.Seconds()
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: written,
|
||||
TotalSize: totalSize,
|
||||
Duration: duration,
|
||||
Protocol: "FILE",
|
||||
Speed: speed,
|
||||
OutputPath: outputPath,
|
||||
Resumed: written > 0 && req.Resume,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// downloadDirectory recursively copies a local directory.
|
||||
func (p *Protocol) downloadDirectory(ctx context.Context, req *core.DownloadRequest, dirPath string, startTime time.Time) (*core.DownloadResult, error) {
|
||||
outputBase := req.Output
|
||||
if outputBase == "" {
|
||||
outputBase = filepath.Base(dirPath)
|
||||
}
|
||||
|
||||
var totalBytes int64
|
||||
var fileCount int
|
||||
|
||||
err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
// Calculate relative path
|
||||
rel, err := filepath.Rel(dirPath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outputPath := filepath.Join(outputBase, rel)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
src, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
n, err := io.Copy(dst, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
totalBytes += n
|
||||
fileCount++
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to copy directory recursively", err)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(0)
|
||||
if duration.Seconds() > 0 {
|
||||
speed = float64(totalBytes) / duration.Seconds()
|
||||
}
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalBytes,
|
||||
TotalSize: totalBytes,
|
||||
Duration: duration,
|
||||
Protocol: "FILE",
|
||||
Speed: speed,
|
||||
OutputPath: outputBase,
|
||||
ChunksCount: fileCount,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
func TestDownloadFile(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
src := filepath.Join(tmp, "src.txt")
|
||||
dst := filepath.Join(tmp, "dst.txt")
|
||||
os.WriteFile(src, []byte("hello world"), 0644)
|
||||
|
||||
proto := NewProtocol()
|
||||
u := &url.URL{Scheme: "file", Path: src}
|
||||
result, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: dst,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.BytesDownloaded != 11 {
|
||||
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
|
||||
}
|
||||
data, _ := os.ReadFile(dst)
|
||||
if string(data) != "hello world" {
|
||||
t.Errorf("content = %q, want %q", data, "hello world")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileToWriter(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
src := filepath.Join(tmp, "src.txt")
|
||||
os.WriteFile(src, []byte("test"), 0644)
|
||||
|
||||
var buf bytes.Buffer
|
||||
proto := NewProtocol()
|
||||
u := &url.URL{Scheme: "file", Path: src}
|
||||
_, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: "-",
|
||||
Writer: &buf,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if buf.String() != "test" {
|
||||
t.Errorf("content = %q, want %q", buf.String(), "test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileNotFound(t *testing.T) {
|
||||
proto := NewProtocol()
|
||||
u := &url.URL{Scheme: "file", Path: "/nonexistent/path"}
|
||||
_, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: "/tmp/out",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for nonexistent file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadDirectory(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
srcDir := filepath.Join(tmp, "srcdir")
|
||||
dstDir := filepath.Join(tmp, "dstdir")
|
||||
os.MkdirAll(filepath.Join(srcDir, "sub"), 0755)
|
||||
os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("a"), 0644)
|
||||
os.WriteFile(filepath.Join(srcDir, "sub/b.txt"), []byte("b"), 0644)
|
||||
|
||||
proto := NewProtocol()
|
||||
u := &url.URL{Scheme: "file", Path: srcDir}
|
||||
result, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: dstDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.ChunksCount != 2 {
|
||||
t.Errorf("files = %d, want 2", result.ChunksCount)
|
||||
}
|
||||
if result.BytesDownloaded != 2 {
|
||||
t.Errorf("bytes = %d, want 2", result.BytesDownloaded)
|
||||
}
|
||||
// Verify files exist
|
||||
if _, err := os.Stat(filepath.Join(dstDir, "a.txt")); err != nil {
|
||||
t.Error("a.txt not found")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dstDir, "sub", "b.txt")); err != nil {
|
||||
t.Error("sub/b.txt not found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeFile(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
src := filepath.Join(tmp, "src.txt")
|
||||
dst := filepath.Join(tmp, "dst.txt")
|
||||
os.WriteFile(src, []byte("hello world"), 0644)
|
||||
os.WriteFile(dst, []byte("hello"), 0644) // partial: 5 bytes
|
||||
|
||||
proto := NewProtocol()
|
||||
u := &url.URL{Scheme: "file", Path: src}
|
||||
result, err := proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: dst,
|
||||
Resume: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Resumed {
|
||||
t.Error("expected Resumed=true")
|
||||
}
|
||||
if result.BytesDownloaded != 11 {
|
||||
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,817 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
conn net.Conn
|
||||
reader *textproto.Reader
|
||||
writer *bufio.Writer
|
||||
host string
|
||||
port int
|
||||
user string
|
||||
password string
|
||||
url string
|
||||
tlsConfig *tls.Config
|
||||
useTLS bool
|
||||
passive bool
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid url: %w", err)
|
||||
}
|
||||
|
||||
host := u.Hostname()
|
||||
port := 21
|
||||
if u.Port() != "" {
|
||||
p, err := strconv.Atoi(u.Port())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid port: %w", err)
|
||||
}
|
||||
port = p
|
||||
}
|
||||
|
||||
user := "anonymous"
|
||||
password := "anonymous@"
|
||||
if u.User != nil {
|
||||
user = u.User.Username()
|
||||
if pass, ok := u.User.Password(); ok {
|
||||
password = pass
|
||||
}
|
||||
}
|
||||
|
||||
useTLS := u.Scheme == "ftps"
|
||||
if !useTLS && (user != "anonymous" || (u.User != nil && u.User.Username() != "anonymous")) {
|
||||
fmt.Fprintf(os.Stderr, "Warning: FTP credentials sent in plaintext. Use ftps:// for encrypted transfers.\n")
|
||||
}
|
||||
|
||||
c := &Client{
|
||||
host: host,
|
||||
port: port,
|
||||
user: user,
|
||||
password: password,
|
||||
url: rawURL,
|
||||
useTLS: useTLS,
|
||||
passive: true, // Default to passive mode
|
||||
timeout: timeout,
|
||||
tlsConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ServerName: host,
|
||||
InsecureSkipVerify: false,
|
||||
},
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Client) Connect() error {
|
||||
addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port))
|
||||
conn, err := net.DialTimeout("tcp", addr, c.timeout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect: %w", err)
|
||||
}
|
||||
|
||||
c.conn = conn
|
||||
c.reader = textproto.NewReader(bufio.NewReader(conn))
|
||||
c.writer = bufio.NewWriter(conn)
|
||||
|
||||
// Read greeting
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read greeting: %w", err)
|
||||
}
|
||||
if code != 220 {
|
||||
return fmt.Errorf("unexpected greeting code: %d", code)
|
||||
}
|
||||
|
||||
// Upgrade to TLS if FTPS (explicit)
|
||||
if c.useTLS {
|
||||
if err := c.sendCommand("AUTH TLS"); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("auth tls failed: %w", err)
|
||||
}
|
||||
if code != 234 && code != 334 {
|
||||
// Try AUTH SSL as fallback
|
||||
if err := c.sendCommand("AUTH SSL"); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("auth ssl failed: %w", err)
|
||||
}
|
||||
if code != 234 && code != 334 {
|
||||
return fmt.Errorf("server does not support tls/ssl")
|
||||
}
|
||||
}
|
||||
|
||||
// Wrap connection with TLS
|
||||
tlsConn := tls.Client(c.conn, c.tlsConfig)
|
||||
if err := tlsConn.Handshake(); err != nil {
|
||||
return fmt.Errorf("tls handshake failed: %w", err)
|
||||
}
|
||||
c.conn = tlsConn
|
||||
c.reader = textproto.NewReader(bufio.NewReader(tlsConn))
|
||||
c.writer = bufio.NewWriter(tlsConn)
|
||||
}
|
||||
|
||||
// Send USER
|
||||
if err := c.sendCommand("USER %s", c.user); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("user command failed: %w", err)
|
||||
}
|
||||
|
||||
// 331 means password required, 230 means already logged in
|
||||
if code == 331 {
|
||||
// Send PASS
|
||||
if err := c.sendCommand("PASS %s", c.password); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("pass command failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if code != 230 && code != 200 {
|
||||
return fmt.Errorf("login failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Set binary mode
|
||||
if err := c.sendCommand("TYPE I"); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("type command failed: %w", err)
|
||||
}
|
||||
if code != 200 {
|
||||
return fmt.Errorf("failed to set binary mode")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) sendCommand(format string, args ...interface{}) error {
|
||||
cmd := fmt.Sprintf(format, args...)
|
||||
if _, err := c.writer.WriteString(cmd); err != nil {
|
||||
return fmt.Errorf("failed to send command: %w", err)
|
||||
}
|
||||
if _, err := c.writer.WriteString("\r\n"); err != nil {
|
||||
return fmt.Errorf("failed to send command terminator: %w", err)
|
||||
}
|
||||
if err := c.writer.Flush(); err != nil {
|
||||
return fmt.Errorf("failed to flush command: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) readResponse() (int, string, error) {
|
||||
code, msg, err := c.reader.ReadResponse(0)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return code, msg, nil
|
||||
}
|
||||
|
||||
func (c *Client) getFileSize(path string) (int64, error) {
|
||||
if err := c.sendCommand("SIZE %s", path); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
code, msg, err := c.readResponse()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("size command failed: %w", err)
|
||||
}
|
||||
if code != 213 {
|
||||
return 0, fmt.Errorf("size command failed with code: %d", code)
|
||||
}
|
||||
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(msg), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse size: %w", err)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
func (c *Client) establishDataConnection() (net.Conn, error) {
|
||||
if c.passive {
|
||||
return c.enterPassiveMode()
|
||||
}
|
||||
return c.enterActiveMode()
|
||||
}
|
||||
|
||||
func (c *Client) enterPassiveMode() (net.Conn, error) {
|
||||
if err := c.sendCommand("PASV"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
code, msg, err := c.readResponse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pasv command failed: %w", err)
|
||||
}
|
||||
if code != 227 {
|
||||
return nil, fmt.Errorf("pasv command failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Parse PASV response: 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2)
|
||||
start := strings.Index(msg, "(")
|
||||
end := strings.Index(msg, ")")
|
||||
if start == -1 || end == -1 {
|
||||
return nil, fmt.Errorf("invalid pasv response")
|
||||
}
|
||||
|
||||
parts := strings.Split(msg[start+1:end], ",")
|
||||
if len(parts) != 6 {
|
||||
return nil, fmt.Errorf("invalid pasv response format")
|
||||
}
|
||||
|
||||
h1, _ := strconv.Atoi(parts[0])
|
||||
h2, _ := strconv.Atoi(parts[1])
|
||||
h3, _ := strconv.Atoi(parts[2])
|
||||
h4, _ := strconv.Atoi(parts[3])
|
||||
p1, _ := strconv.Atoi(parts[4])
|
||||
p2, _ := strconv.Atoi(parts[5])
|
||||
|
||||
dataHost := fmt.Sprintf("%d.%d.%d.%d", h1, h2, h3, h4)
|
||||
dataPort := p1*256 + p2
|
||||
|
||||
// Use original host for TLS connections to match certificate
|
||||
if c.useTLS {
|
||||
dataHost = c.host
|
||||
}
|
||||
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", dataHost, dataPort), c.timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to data port: %w", err)
|
||||
}
|
||||
|
||||
if c.useTLS {
|
||||
tlsConn := tls.Client(conn, c.tlsConfig)
|
||||
if err := tlsConn.Handshake(); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("tls handshake on data connection failed: %w", err)
|
||||
}
|
||||
return tlsConn, nil
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (c *Client) enterActiveMode() (net.Conn, error) {
|
||||
// Get local IP from the control connection to bind only to that interface
|
||||
ctrlAddr, ok := c.conn.LocalAddr().(*net.TCPAddr)
|
||||
if !ok || ctrlAddr.IP == nil {
|
||||
return nil, fmt.Errorf("failed to get local address for active mode")
|
||||
}
|
||||
ctrlIP := ctrlAddr.IP.To4()
|
||||
if ctrlIP == nil {
|
||||
return nil, fmt.Errorf("no ipv4 address available for active mode")
|
||||
}
|
||||
|
||||
// Bind listener only to the interface of the control connection, not 0.0.0.0
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort(ctrlIP.String(), "0"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create listener: %w", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
// Set a deadline so Accept respects the timeout
|
||||
if tcpListener, ok := listener.(*net.TCPListener); ok {
|
||||
tcpListener.SetDeadline(time.Now().Add(c.timeout))
|
||||
}
|
||||
|
||||
listenAddr := listener.Addr().(*net.TCPAddr)
|
||||
|
||||
p1 := listenAddr.Port / 256
|
||||
p2 := listenAddr.Port % 256
|
||||
|
||||
portCmd := fmt.Sprintf("PORT %d,%d,%d,%d,%d,%d",
|
||||
ctrlIP[0], ctrlIP[1], ctrlIP[2], ctrlIP[3], p1, p2)
|
||||
|
||||
if err := c.sendCommand("%s", portCmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("port command failed: %w", err)
|
||||
}
|
||||
if code != 200 {
|
||||
return nil, fmt.Errorf("port command failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Accept incoming connection
|
||||
connChan := make(chan net.Conn, 1)
|
||||
errChan := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
connChan <- conn
|
||||
}()
|
||||
|
||||
select {
|
||||
case conn := <-connChan:
|
||||
return conn, nil
|
||||
case err := <-errChan:
|
||||
return nil, fmt.Errorf("failed to accept data connection: %w", err)
|
||||
case <-time.After(c.timeout):
|
||||
return nil, fmt.Errorf("timeout waiting for data connection")
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) DownloadFile(ctx context.Context, remotePath, localPath string, resumeOffset int64, progressFunc func(current, total int64)) error {
|
||||
// Get file size
|
||||
totalSize, err := c.getFileSize(remotePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file size: %w", err)
|
||||
}
|
||||
|
||||
// Check if we need to resume
|
||||
var startOffset int64 = 0
|
||||
if resumeOffset > 0 && resumeOffset < totalSize {
|
||||
startOffset = resumeOffset
|
||||
if err := c.sendCommand("REST %d", startOffset); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("rest command failed: %w", err)
|
||||
}
|
||||
if code != 350 {
|
||||
return fmt.Errorf("rest command failed with code: %d", code)
|
||||
}
|
||||
} else if resumeOffset >= totalSize {
|
||||
// File already complete
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open local file for writing
|
||||
mode := os.O_CREATE | os.O_WRONLY
|
||||
if startOffset > 0 {
|
||||
mode |= os.O_APPEND
|
||||
} else {
|
||||
mode |= os.O_TRUNC
|
||||
}
|
||||
|
||||
localFile, err := os.OpenFile(localPath, mode, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open local file: %w", err)
|
||||
}
|
||||
defer localFile.Close()
|
||||
|
||||
// Seek to start position if resuming
|
||||
if startOffset > 0 {
|
||||
if _, err := localFile.Seek(startOffset, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Establish data connection FIRST (before RETR command)
|
||||
dataConn, err := c.establishDataConnection()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to establish data connection: %w", err)
|
||||
}
|
||||
defer dataConn.Close()
|
||||
|
||||
// Send RETR command AFTER data connection is established
|
||||
if err := c.sendCommand("RETR %s", remotePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read response (should be 150 or 125)
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("retr command failed: %w", err)
|
||||
}
|
||||
if code != 150 && code != 125 {
|
||||
// Try waiting a bit for the server to respond
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("retr command failed with code: %d", code)
|
||||
}
|
||||
if code != 150 && code != 125 {
|
||||
return fmt.Errorf("retr command failed with code: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// Copy data with progress. Polls ctx between reads so a SIGINT
|
||||
// arriving mid-transfer is detected on the next iteration and the
|
||||
// partial progress is saved to ResumeMetadata before returning.
|
||||
var copied int64 = startOffset
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
if err := saveFTPResume(c.url, localPath, copied, totalSize); err != nil {
|
||||
return fmt.Errorf("failed to save resume metadata: %w", err)
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
n, err := dataConn.Read(buf)
|
||||
if n > 0 {
|
||||
written, werr := localFile.Write(buf[:n])
|
||||
copied += int64(written)
|
||||
if werr != nil {
|
||||
return fmt.Errorf("failed to write: %w", err)
|
||||
}
|
||||
if progressFunc != nil {
|
||||
progressFunc(copied, totalSize)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read from data connection: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Read final response
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read final response: %w", err)
|
||||
}
|
||||
if code != 226 && code != 250 {
|
||||
return fmt.Errorf("transfer failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Successful transfer — drop any stale resume sidecar.
|
||||
_ = output.DeleteResumeMetadata(localPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ListRemoteDir(path string) ([]string, error) {
|
||||
// Establish data connection
|
||||
dataConn, err := c.establishDataConnection()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to establish data connection: %w", err)
|
||||
}
|
||||
defer dataConn.Close()
|
||||
|
||||
// Send LIST command
|
||||
cmd := "LIST"
|
||||
if path != "" {
|
||||
cmd = fmt.Sprintf("LIST %s", path)
|
||||
}
|
||||
if err := c.sendCommand("%s", cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read response (should be 150 or 125)
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list command failed: %w", err)
|
||||
}
|
||||
if code != 150 && code != 125 {
|
||||
return nil, fmt.Errorf("list command failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Read directory listing
|
||||
var lines []string
|
||||
scanner := bufio.NewScanner(dataConn)
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to read directory listing: %w", err)
|
||||
}
|
||||
|
||||
// Read final response
|
||||
code, _, err = c.readResponse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read final response: %w", err)
|
||||
}
|
||||
if code != 226 && code != 250 {
|
||||
return nil, fmt.Errorf("list command failed with code: %d", code)
|
||||
}
|
||||
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func (c *Client) ChangeDir(path string) error {
|
||||
if err := c.sendCommand("CWD %s", path); err != nil {
|
||||
return err
|
||||
}
|
||||
code, _, err := c.readResponse()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cwd command failed: %w", err)
|
||||
}
|
||||
if code != 250 {
|
||||
return fmt.Errorf("cwd command failed with code: %d", code)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) GetCurrentDir() (string, error) {
|
||||
if err := c.sendCommand("PWD"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
code, msg, err := c.readResponse()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pwd command failed: %w", err)
|
||||
}
|
||||
if code != 257 {
|
||||
return "", fmt.Errorf("pwd command failed with code: %d", code)
|
||||
}
|
||||
|
||||
// Extract path from response (usually quoted)
|
||||
start := strings.Index(msg, "\"")
|
||||
end := strings.LastIndex(msg, "\"")
|
||||
if start != -1 && end > start {
|
||||
return msg[start+1 : end], nil
|
||||
}
|
||||
return strings.TrimSpace(msg), nil
|
||||
}
|
||||
|
||||
func (c *Client) Close() error {
|
||||
if err := c.sendCommand("QUIT"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _, _ = c.readResponse() // Ignore errors on close
|
||||
if c.conn != nil {
|
||||
return c.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper function to download entire directory recursively
|
||||
func (c *Client) DownloadDirectory(ctx context.Context, remotePath, localPath string, progressFunc func(file string, current, total int64)) error {
|
||||
// Create local directory
|
||||
if err := os.MkdirAll(localPath, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create local directory: %w", err)
|
||||
}
|
||||
|
||||
// Save current directory
|
||||
origDir, err := c.GetCurrentDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
c.ChangeDir(origDir)
|
||||
}()
|
||||
|
||||
// Change to remote directory
|
||||
if err := c.ChangeDir(remotePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// List directory contents
|
||||
entries, err := c.ListRemoteDir("")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Parse entries and download files
|
||||
for _, entry := range entries {
|
||||
fields := strings.Fields(entry)
|
||||
if len(fields) < 9 {
|
||||
continue
|
||||
}
|
||||
|
||||
name := fields[len(fields)-1]
|
||||
if name == "." || name == ".." {
|
||||
continue
|
||||
}
|
||||
|
||||
isDir := fields[0][0] == 'd'
|
||||
remoteFilePath := filepath.Join(remotePath, name)
|
||||
localFilePath := filepath.Join(localPath, name)
|
||||
|
||||
if isDir {
|
||||
if err := c.DownloadDirectory(ctx, remoteFilePath, localFilePath, progressFunc); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if progressFunc != nil {
|
||||
progressFunc(name, 0, 0)
|
||||
}
|
||||
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveFTPResume persists partial FTP download progress to the standard
|
||||
// `.goget.meta` sidecar so a subsequent `goget --resume` can continue
|
||||
// from `downloaded` bytes. Errors are wrapped to surface failures
|
||||
// without aborting the cancellation handling.
|
||||
func saveFTPResume(url, localPath string, downloaded, total int64) error {
|
||||
if localPath == "" || localPath == "-" || downloaded <= 0 {
|
||||
return nil
|
||||
}
|
||||
meta := output.NewResumeMetadata(url, "", "", downloaded, total)
|
||||
return meta.Save(localPath)
|
||||
}
|
||||
|
||||
// clientPool holds N pre-connected FTP clients. A single shared pool
|
||||
// removes the need to open/close connections for every file in a
|
||||
// recursive download, and is required for the parallel recursive
|
||||
// implementation because each *Client owns its own TCP control
|
||||
// connection and is not safe for concurrent use.
|
||||
type clientPool struct {
|
||||
available chan *Client
|
||||
all []*Client
|
||||
}
|
||||
|
||||
// newClientPool opens `size` independent FTP connections to the same
|
||||
// server. Connections are kept open until close() is called.
|
||||
func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) {
|
||||
if size < 1 {
|
||||
size = 1
|
||||
}
|
||||
all := make([]*Client, 0, size)
|
||||
avail := make(chan *Client, size)
|
||||
for i := 0; i < size; i++ {
|
||||
c, err := NewClient(rawURL, timeout)
|
||||
if err != nil {
|
||||
for _, existing := range all {
|
||||
_ = existing.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := c.Connect(); err != nil {
|
||||
for _, existing := range all {
|
||||
_ = existing.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, c)
|
||||
avail <- c
|
||||
}
|
||||
return &clientPool{available: avail, all: all}, nil
|
||||
}
|
||||
|
||||
// acquire blocks until a client is available, then returns it. The
|
||||
// caller must call release() when done.
|
||||
func (p *clientPool) acquire() *Client {
|
||||
return <-p.available
|
||||
}
|
||||
|
||||
// release returns a client to the pool. It must be called exactly once
|
||||
// per acquire().
|
||||
func (p *clientPool) release(c *Client) {
|
||||
p.available <- c
|
||||
}
|
||||
|
||||
// size returns the number of clients in the pool.
|
||||
func (p *clientPool) size() int {
|
||||
return len(p.all)
|
||||
}
|
||||
|
||||
// close shuts down every client in the pool. The pool must not be used
|
||||
// after this call.
|
||||
func (p *clientPool) close() {
|
||||
for _, c := range p.all {
|
||||
_ = c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadDirectoryConcurrent downloads a remote directory to localPath
|
||||
// using a pool of N FTP connections. All operations (LIST, RETR) use
|
||||
// absolute paths so concurrent workers never share or mutate CWD state
|
||||
// on the server side. The returned error is the first error encountered
|
||||
// by any worker; other workers continue in the background until they
|
||||
// hit ctx cancellation or finish their tasks.
|
||||
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error {
|
||||
if parallel < 1 {
|
||||
parallel = 1
|
||||
}
|
||||
pool, err := newClientPool(rawURL, timeout, parallel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.close()
|
||||
|
||||
sem := make(chan struct{}, parallel)
|
||||
var wg sync.WaitGroup
|
||||
var firstErr error
|
||||
var errMu sync.Mutex
|
||||
|
||||
setErr := func(err error) {
|
||||
errMu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
errMu.Unlock()
|
||||
}
|
||||
|
||||
processEntry(ctx, pool, sem, &wg, setErr, remotePath, localPath, progressFunc)
|
||||
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// processEntry processes one remote path: lists it, then dispatches
|
||||
// its child entries (file → download, subdir → recursive processEntry)
|
||||
// through the shared semaphore-bounded goroutine pool. A single
|
||||
// sem is threaded through the recursion so the total in-flight
|
||||
// goroutines never exceed `parallel`, regardless of directory depth.
|
||||
func processEntry(ctx context.Context, pool *clientPool, sem chan struct{}, wg *sync.WaitGroup, setErr func(error), remotePath, localPath string, progressFunc func(file string, current, total int64)) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(localPath, 0755); err != nil {
|
||||
setErr(fmt.Errorf("failed to create local dir %s: %w", localPath, err))
|
||||
return
|
||||
}
|
||||
|
||||
c := pool.acquire()
|
||||
entries, err := c.ListRemoteDir(remotePath)
|
||||
pool.release(c)
|
||||
if err != nil {
|
||||
setErr(fmt.Errorf("list %s: %w", remotePath, err))
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fields := strings.Fields(entry)
|
||||
if len(fields) < 9 {
|
||||
continue
|
||||
}
|
||||
name := fields[len(fields)-1]
|
||||
if name == "." || name == ".." {
|
||||
continue
|
||||
}
|
||||
isDir := fields[0][0] == 'd'
|
||||
remoteFilePath := filepath.Join(remotePath, name)
|
||||
localFilePath := filepath.Join(localPath, name)
|
||||
|
||||
if isDir {
|
||||
wg.Add(1)
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
go processEntry(ctx, pool, sem, wg, setErr, remoteFilePath, localFilePath, progressFunc)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
} else {
|
||||
wg.Add(1)
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
go func(remoteFilePath, localFilePath, name string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
c := pool.acquire()
|
||||
defer pool.release(c)
|
||||
if progressFunc != nil {
|
||||
progressFunc(name, 0, 0)
|
||||
}
|
||||
// DownloadFile's progress callback has a different
|
||||
// signature than our directory-level progressFunc, so
|
||||
// we wrap it here to keep the existing per-file
|
||||
// progress behaviour.
|
||||
var fileProgress func(current, total int64)
|
||||
if progressFunc != nil {
|
||||
fileProgress = func(current, total int64) {
|
||||
progressFunc(name, current, total)
|
||||
}
|
||||
}
|
||||
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, fileProgress); err != nil {
|
||||
setErr(fmt.Errorf("download %s: %w", remoteFilePath, err))
|
||||
}
|
||||
}(remoteFilePath, localFilePath, name)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------- Protocol tests ----------
|
||||
|
||||
func TestNewProtocol(t *testing.T) {
|
||||
p := NewProtocol()
|
||||
if p == nil {
|
||||
t.Fatal("NewProtocol() returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheme(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
if got := p.Scheme(); got != "ftp" {
|
||||
t.Errorf("Scheme() = %q, want %q", got, "ftp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanHandle(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want bool
|
||||
}{
|
||||
{"ftp scheme", "ftp://ftp.example.com/file.txt", true},
|
||||
{"ftps scheme", "ftps://ftp.example.com/file.txt", true},
|
||||
{"http scheme", "http://example.com/file.txt", false},
|
||||
{"https scheme", "https://example.com/file.txt", false},
|
||||
{"sftp scheme", "sftp://example.com/file.txt", false},
|
||||
{"empty scheme", "file.txt", false},
|
||||
{"nil URL", "", false},
|
||||
}
|
||||
|
||||
p := &Protocol{}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var u *url.URL
|
||||
if tt.raw != "" {
|
||||
var err error
|
||||
u, err = url.Parse(tt.raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse URL %q: %v", tt.raw, err)
|
||||
}
|
||||
}
|
||||
if got := p.CanHandle(u); got != tt.want {
|
||||
t.Errorf("CanHandle(%q) = %v, want %v", tt.raw, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanHandleCaseInsensitive(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
u, _ := url.Parse("FTP://example.com/file.txt")
|
||||
if !p.CanHandle(u) {
|
||||
t.Errorf("CanHandle should be case-insensitive for FTP")
|
||||
}
|
||||
u, _ = url.Parse("FTPS://example.com/file.txt")
|
||||
if !p.CanHandle(u) {
|
||||
t.Errorf("CanHandle should be case-insensitive for FTPS")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapabilities(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
caps := p.Capabilities()
|
||||
if caps == nil {
|
||||
t.Fatal("Capabilities() returned nil")
|
||||
}
|
||||
|
||||
expected := map[string]bool{
|
||||
"download": true,
|
||||
"resume": true,
|
||||
"recursive": true,
|
||||
"tls": true,
|
||||
}
|
||||
|
||||
if len(caps) != len(expected) {
|
||||
t.Errorf("Capabilities() length = %d, want %d", len(caps), len(expected))
|
||||
}
|
||||
|
||||
for _, cap := range caps {
|
||||
if !expected[cap] {
|
||||
t.Errorf("unexpected capability: %q", cap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsResume(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
if !p.SupportsResume() {
|
||||
t.Errorf("SupportsResume() should return true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsCompression(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
if p.SupportsCompression() {
|
||||
t.Errorf("SupportsCompression() should return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsParallel(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
if p.SupportsParallel() {
|
||||
t.Errorf("SupportsParallel() should return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsRecursive(t *testing.T) {
|
||||
p := &Protocol{}
|
||||
if !p.SupportsRecursive() {
|
||||
t.Errorf("SupportsRecursive() should return true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAll(t *testing.T) {
|
||||
t.Run("normal content", func(t *testing.T) {
|
||||
input := "Hello, FTP world!"
|
||||
r := strings.NewReader(input)
|
||||
data, err := readAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("readAll() returned error: %v", err)
|
||||
}
|
||||
if string(data) != input {
|
||||
t.Errorf("readAll() = %q, want %q", string(data), input)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty reader", func(t *testing.T) {
|
||||
r := strings.NewReader("")
|
||||
data, err := readAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("readAll() returned error: %v", err)
|
||||
}
|
||||
if len(data) != 0 {
|
||||
t.Errorf("readAll() returned %d bytes, want 0", len(data))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("large content", func(t *testing.T) {
|
||||
// Create content larger than the 32KB buffer to ensure multiple reads
|
||||
content := strings.Repeat("A", 100*1024)
|
||||
r := strings.NewReader(content)
|
||||
data, err := readAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("readAll() returned error: %v", err)
|
||||
}
|
||||
if len(data) != len(content) {
|
||||
t.Errorf("readAll() returned %d bytes, want %d", len(data), len(content))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reader error", func(t *testing.T) {
|
||||
expectedErr := errors.New("read error")
|
||||
r := &errorReader{err: expectedErr}
|
||||
_, err := readAll(r)
|
||||
if err == nil {
|
||||
t.Fatal("readAll() should return error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), expectedErr.Error()) {
|
||||
t.Errorf("readAll() error = %v, want %v", err, expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// errorReader implements io.Reader that always returns an error
|
||||
type errorReader struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *errorReader) Read(p []byte) (n int, err error) {
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
// ---------- Client tests ----------
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
rawURL string
|
||||
wantErr bool
|
||||
}{
|
||||
{"basic FTP URL", "ftp://ftp.example.com/pub/file.txt", false},
|
||||
{"FTP URL with path", "ftp://ftp.example.com/pub/", false},
|
||||
{"FTP URL root", "ftp://ftp.example.com", false},
|
||||
{"FTP URL with trailing slash", "ftp://ftp.example.com/", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client, err := NewClient(tt.rawURL, 30*time.Second)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("NewClient(%q) expected error", tt.rawURL)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(%q) returned error: %v", tt.rawURL, err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatal("NewClient() returned nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientWithAuth(t *testing.T) {
|
||||
client, err := NewClient("ftp://user:password@ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.user != "user" {
|
||||
t.Errorf("client.user = %q, want %q", client.user, "user")
|
||||
}
|
||||
if client.password != "password" {
|
||||
t.Errorf("client.password = %q, want %q", client.password, "password")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientWithUserOnly(t *testing.T) {
|
||||
client, err := NewClient("ftp://user@ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.user != "user" {
|
||||
t.Errorf("client.user = %q, want %q", client.user, "user")
|
||||
}
|
||||
if client.password != "anonymous@" {
|
||||
t.Errorf("client.password should default to %q when only username is provided, got %q", "anonymous@", client.password)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientFTPS(t *testing.T) {
|
||||
client, err := NewClient("ftps://ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if !client.useTLS {
|
||||
t.Errorf("client.useTLS should be true for ftps:// scheme")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientFTPSWithAuth(t *testing.T) {
|
||||
client, err := NewClient("ftps://alice:secret@ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if !client.useTLS {
|
||||
t.Errorf("client.useTLS should be true for ftps:// scheme")
|
||||
}
|
||||
if client.user != "alice" {
|
||||
t.Errorf("client.user = %q, want %q", client.user, "alice")
|
||||
}
|
||||
if client.password != "secret" {
|
||||
t.Errorf("client.password = %q, want %q", client.password, "secret")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientInvalidURL(t *testing.T) {
|
||||
// Only inputs where url.Parse itself fails
|
||||
_, err := NewClient("://invalid", 30*time.Second)
|
||||
if err == nil {
|
||||
t.Errorf("NewClient(%q) expected error", "://invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientEmptyURL(t *testing.T) {
|
||||
client, err := NewClient("", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(\"\") returned error: %v", err)
|
||||
}
|
||||
// url.Parse("") returns an empty URL struct, so host will be empty
|
||||
if client.host != "" {
|
||||
t.Errorf("client.host = %q, want empty", client.host)
|
||||
}
|
||||
if client.port != 21 {
|
||||
t.Errorf("client.port = %d, want %d", client.port, 21)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientNoScheme(t *testing.T) {
|
||||
client, err := NewClient("not-a-url", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(\"not-a-url\") returned error: %v", err)
|
||||
}
|
||||
// url.Parse("not-a-url") parses it as a path with empty host
|
||||
if client.host != "" {
|
||||
t.Errorf("client.host = %q, want empty", client.host)
|
||||
}
|
||||
if client.port != 21 {
|
||||
t.Errorf("client.port = %d, want %d", client.port, 21)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientDefaultPort(t *testing.T) {
|
||||
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.port != 21 {
|
||||
t.Errorf("client.port = %d, want %d", client.port, 21)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientCustomPort(t *testing.T) {
|
||||
client, err := NewClient("ftp://ftp.example.com:2121/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.port != 2121 {
|
||||
t.Errorf("client.port = %d, want %d", client.port, 2121)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientInvalidPort(t *testing.T) {
|
||||
_, err := NewClient("ftp://ftp.example.com:invalid/file.txt", 30*time.Second)
|
||||
if err == nil {
|
||||
t.Errorf("NewClient() expected error for invalid port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientDefaultCredentials(t *testing.T) {
|
||||
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.user != "anonymous" {
|
||||
t.Errorf("default user = %q, want %q", client.user, "anonymous")
|
||||
}
|
||||
if client.password != "anonymous@" {
|
||||
t.Errorf("default password = %q, want %q", client.password, "anonymous@")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientHostParsing(t *testing.T) {
|
||||
client, err := NewClient("ftp://ftp.example.com/path/to/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.host != "ftp.example.com" {
|
||||
t.Errorf("client.host = %q, want %q", client.host, "ftp.example.com")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientPassiveModeDefault(t *testing.T) {
|
||||
client, err := NewClient("ftp://ftp.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if !client.passive {
|
||||
t.Errorf("client.passive should default to true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientTLSConfig(t *testing.T) {
|
||||
client, err := NewClient("ftps://secure.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
if client.tlsConfig == nil {
|
||||
t.Fatal("client.tlsConfig should not be nil")
|
||||
}
|
||||
if client.tlsConfig.ServerName != "secure.example.com" {
|
||||
t.Errorf("tlsConfig.ServerName = %q, want %q", client.tlsConfig.ServerName, "secure.example.com")
|
||||
}
|
||||
if client.tlsConfig.InsecureSkipVerify {
|
||||
t.Errorf("tlsConfig.InsecureSkipVerify should be false by default")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
// Protocol implements the core.Protocol interface for FTP/FTPS
|
||||
type Protocol struct{}
|
||||
|
||||
// NewProtocol creates a new instance of FTP protocol
|
||||
func NewProtocol() *Protocol {
|
||||
return &Protocol{}
|
||||
}
|
||||
|
||||
// Scheme returns the protocol scheme
|
||||
func (p *Protocol) Scheme() string {
|
||||
return "ftp"
|
||||
}
|
||||
|
||||
// CanHandle checks whether the protocol can handle the given URL
|
||||
func (p *Protocol) CanHandle(u *url.URL) bool {
|
||||
if u == nil {
|
||||
return false
|
||||
}
|
||||
scheme := strings.ToLower(u.Scheme)
|
||||
return scheme == "ftp" || scheme == "ftps"
|
||||
}
|
||||
|
||||
// Capabilities returns the list of supported features
|
||||
func (p *Protocol) Capabilities() []string {
|
||||
return []string{
|
||||
"download",
|
||||
"resume",
|
||||
"recursive",
|
||||
"tls",
|
||||
}
|
||||
}
|
||||
|
||||
// SupportsResume returns whether the protocol supports resume
|
||||
func (p *Protocol) SupportsResume() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SupportsCompression returns whether the protocol supports compression
|
||||
func (p *Protocol) SupportsCompression() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// SupportsParallel returns whether the protocol supports parallel downloading
|
||||
func (p *Protocol) SupportsParallel() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// SupportsRecursive returns whether the protocol supports recursive downloading
|
||||
func (p *Protocol) SupportsRecursive() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Download downloads a file or directory via FTP/FTPS
|
||||
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
timeout := req.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
client, err := NewClient(req.URL.String(), timeout)
|
||||
if err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("failed to create FTP client: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
if err := client.Connect(); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("failed to connect: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
startTime := time.Now()
|
||||
var bytesDownloaded int64 = 0
|
||||
var chunksCount int
|
||||
|
||||
// Determine whether it is a file or directory
|
||||
remotePath := req.URL.Path
|
||||
if remotePath == "" {
|
||||
remotePath = "/"
|
||||
}
|
||||
|
||||
// Try to get file size - if it fails, it's probably a directory
|
||||
fileSize, err := client.getFileSize(remotePath)
|
||||
isFile := (err == nil)
|
||||
|
||||
if isFile {
|
||||
// Downloading file
|
||||
var outputFile string
|
||||
if req.Output != "" {
|
||||
outputFile = req.Output
|
||||
} else {
|
||||
outputFile = filepath.Base(remotePath)
|
||||
if outputFile == "" || outputFile == "/" {
|
||||
outputFile = "index.html"
|
||||
}
|
||||
}
|
||||
|
||||
// Kontrola resume
|
||||
var resumeOffset int64 = 0
|
||||
if req.Resume && req.Output != "" {
|
||||
if canResume, meta, _ := output.CanResume(req.Output, req.URL.String()); canResume && meta != nil {
|
||||
resumeOffset = meta.Downloaded
|
||||
if req.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "Resuming from offset %d\n", resumeOffset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Progress callback
|
||||
var progressCallback func(current, total int64)
|
||||
if req.ProgressCallback != nil && fileSize > 0 {
|
||||
totalSize := fileSize
|
||||
startOffset := resumeOffset
|
||||
progressCallback = func(current, _ int64) {
|
||||
req.ProgressCallback(startOffset+current, totalSize, 0)
|
||||
}
|
||||
}
|
||||
|
||||
if err := client.DownloadFile(ctx, remotePath, outputFile, resumeOffset, progressCallback); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("download failed: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
// Get the actual size of the downloaded file
|
||||
if info, err := os.Stat(outputFile); err == nil {
|
||||
bytesDownloaded = info.Size()
|
||||
}
|
||||
chunksCount = 1
|
||||
|
||||
} else {
|
||||
// Recursive download of directory
|
||||
if !req.Recursive {
|
||||
return nil, core.NewProtocolError("remote path is a directory, use --recursive to download", nil, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
outputDir := req.Output
|
||||
if outputDir == "" {
|
||||
outputDir = filepath.Base(remotePath)
|
||||
if outputDir == "" || outputDir == "/" {
|
||||
outputDir = "download"
|
||||
}
|
||||
}
|
||||
|
||||
var progressCallback func(file string, current, total int64)
|
||||
if req.Verbose {
|
||||
progressCallback = func(file string, current, total int64) {
|
||||
fmt.Fprintf(os.Stderr, "Downloading: %s\n", file)
|
||||
}
|
||||
}
|
||||
|
||||
// Dispatch to the parallel implementation when the user
|
||||
// requested worker-pool concurrency, otherwise fall back to the
|
||||
// single-connection sequential implementation. The parallel path
|
||||
// opens N control connections to avoid stepping on CWD state and
|
||||
// to keep the existing client-side protocol serialisation.
|
||||
if req.RecursiveParallel > 1 {
|
||||
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, progressCallback); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
} else {
|
||||
if err := client.DownloadDirectory(ctx, remotePath, outputDir, progressCallback); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate total size and file count
|
||||
filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
bytesDownloaded += info.Size()
|
||||
chunksCount++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: bytesDownloaded,
|
||||
Duration: duration,
|
||||
ChunksCount: chunksCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Helper function for reading from io.Reader
|
||||
func readAll(reader io.Reader) ([]byte, error) {
|
||||
var data []byte
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
data = append(data, buf[:n]...)
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
// TestSaveFTPResume verifies the helper persists partial FTP progress
|
||||
// to a .goget.meta sidecar and that the sidecar is suitable for a
|
||||
// subsequent goget --resume invocation. The function is called from
|
||||
// Client.DownloadFile when ctx is cancelled mid-transfer; without a
|
||||
// working sidecar the user's interrupted download cannot be continued.
|
||||
func TestSaveFTPResume(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
out := filepath.Join(tmp, "file.bin")
|
||||
|
||||
// 1. First call records partial progress.
|
||||
if err := saveFTPResume("ftp://example.com/file.bin", out, 4096, 1<<20); err != nil {
|
||||
t.Fatalf("saveFTPResume: %v", err)
|
||||
}
|
||||
meta, err := output.LoadResumeMetadata(out)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadResumeMetadata: %v", err)
|
||||
}
|
||||
if meta == nil {
|
||||
t.Fatal("expected resume metadata, got nil")
|
||||
}
|
||||
if meta.URL != "ftp://example.com/file.bin" {
|
||||
t.Errorf("URL = %q, want ftp://example.com/file.bin", meta.URL)
|
||||
}
|
||||
if meta.Downloaded != 4096 {
|
||||
t.Errorf("Downloaded = %d, want 4096", meta.Downloaded)
|
||||
}
|
||||
if meta.Total != 1<<20 {
|
||||
t.Errorf("Total = %d, want %d", meta.Total, 1<<20)
|
||||
}
|
||||
|
||||
// 2. Second call overwrites the sidecar with a later snapshot.
|
||||
if err := saveFTPResume("ftp://example.com/file.bin", out, 8192, 1<<20); err != nil {
|
||||
t.Fatalf("saveFTPResume (2): %v", err)
|
||||
}
|
||||
meta2, _ := output.LoadResumeMetadata(out)
|
||||
if meta2 == nil || meta2.Downloaded != 8192 {
|
||||
t.Errorf("second call did not update sidecar: %+v", meta2)
|
||||
}
|
||||
|
||||
// 3. Zero-byte save is a no-op (no point persisting a sidecar
|
||||
// for an empty transfer).
|
||||
out2 := filepath.Join(tmp, "empty.bin")
|
||||
if err := saveFTPResume("ftp://x", out2, 0, 100); err != nil {
|
||||
t.Errorf("zero-byte save returned error: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out2 + output.ResumeMetaFileSuffix); !os.IsNotExist(err) {
|
||||
t.Errorf("expected no sidecar for zero-byte save, got one")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
)
|
||||
|
||||
// Protocol implements gemini:// downloads.
|
||||
type Protocol struct {
|
||||
*protocol.BaseProtocol
|
||||
TLSInsecure bool // allow self-signed certs (testing only)
|
||||
}
|
||||
|
||||
// maxGeminiRedirects limits redirect chain depth to prevent stack overflow.
|
||||
const maxGeminiRedirects = 10
|
||||
|
||||
// NewProtocol creates a new Gemini protocol handler.
|
||||
func NewProtocol() *Protocol {
|
||||
return &Protocol{
|
||||
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
|
||||
Name: "GEMINI",
|
||||
Scheme: "gemini",
|
||||
DefaultPort: 1965,
|
||||
Features: []string{},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// Download starts a Gemini download. It handles redirect limits internally.
|
||||
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
return p.downloadWithRedirects(ctx, req, 0)
|
||||
}
|
||||
|
||||
func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.DownloadRequest, redirectCount int) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
host := req.URL.Hostname()
|
||||
port := req.URL.Port()
|
||||
if port == "" {
|
||||
port = "1965"
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(host, port)
|
||||
|
||||
tlsCfg := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ServerName: host,
|
||||
InsecureSkipVerify: p.TLSInsecure,
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{Timeout: 30 * time.Second}
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsCfg)
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to connect to gemini server", err, core.SafeURL(req.URL))
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
conn.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
|
||||
// Send request: URL + CRLF
|
||||
requestURL := req.URL.String()
|
||||
if _, err := fmt.Fprintf(conn, "%s\r\n", requestURL); err != nil {
|
||||
return nil, core.NewNetworkError("failed to send gemini request", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
// Read response header
|
||||
reader := bufio.NewReader(conn)
|
||||
header, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to read gemini response header", err, core.SafeURL(req.URL))
|
||||
}
|
||||
header = strings.TrimSpace(header)
|
||||
|
||||
// Parse status code (first two chars)
|
||||
if len(header) < 2 {
|
||||
return nil, core.NewProtocolError("invalid gemini response header", nil, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
statusCode, _ := strconv.Atoi(header[:2])
|
||||
meta := ""
|
||||
if len(header) > 3 {
|
||||
meta = header[3:]
|
||||
}
|
||||
|
||||
switch {
|
||||
case statusCode >= 10 && statusCode < 20:
|
||||
// 1x INPUT — server requests input (prompt user with meta)
|
||||
return nil, core.NewProtocolError(
|
||||
fmt.Sprintf("gemini input required: %s", meta), nil, core.SafeURL(req.URL))
|
||||
|
||||
case statusCode >= 30 && statusCode < 40:
|
||||
// 3x REDIRECT — follow the redirect
|
||||
if meta == "" {
|
||||
return nil, core.NewProtocolError("gemini redirect with no target URL", nil, core.SafeURL(req.URL))
|
||||
}
|
||||
if redirectCount >= maxGeminiRedirects {
|
||||
return nil, core.NewProtocolError(
|
||||
fmt.Sprintf("gemini redirect chain exceeded %d", maxGeminiRedirects), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
redirectURL, err := url.Parse(meta)
|
||||
if err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("invalid gemini redirect URL: %s", meta), err, core.SafeURL(req.URL))
|
||||
}
|
||||
// Resolve relative redirects
|
||||
resolved := req.URL.ResolveReference(redirectURL)
|
||||
if req.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[gemini] redirecting to: %s\n", resolved.String())
|
||||
}
|
||||
// Follow redirect with incremented depth
|
||||
redirectReq := &core.DownloadRequest{
|
||||
URL: resolved,
|
||||
Output: req.Output,
|
||||
Verbose: req.Verbose,
|
||||
Writer: req.Writer,
|
||||
Ctx: ctx,
|
||||
}
|
||||
return p.downloadWithRedirects(ctx, redirectReq, redirectCount+1)
|
||||
|
||||
case statusCode >= 40 && statusCode < 50:
|
||||
// 4x TEMPORARY FAILURE
|
||||
return nil, core.NewNetworkError(
|
||||
fmt.Sprintf("gemini temporary failure: %d %s", statusCode, meta),
|
||||
nil, core.SafeURL(req.URL))
|
||||
|
||||
case statusCode >= 50 && statusCode < 60:
|
||||
// 5x PERMANENT FAILURE
|
||||
return nil, core.NewProtocolError(
|
||||
fmt.Sprintf("gemini permanent failure: %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
|
||||
|
||||
case statusCode >= 60 && statusCode < 70:
|
||||
// 6x CLIENT CERTIFICATE REQUIRED
|
||||
return nil, core.NewProtocolError(
|
||||
fmt.Sprintf("gemini client certificate required: %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
|
||||
|
||||
case statusCode < 20 || statusCode >= 30:
|
||||
return nil, core.NewProtocolError(
|
||||
fmt.Sprintf("gemini server returned %d %s", statusCode, meta), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
// Category 2x: success — continue to read body
|
||||
var totalRead int64
|
||||
var writer io.Writer
|
||||
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
}
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := writer.Write(buf[:n]); werr != nil {
|
||||
return nil, core.NewFileError("failed to write gemini data", werr)
|
||||
}
|
||||
totalRead += int64(n)
|
||||
if req.ProgressCallback != nil {
|
||||
speed := float64(totalRead) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(totalRead, -1, speed)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to read gemini response body", err, core.SafeURL(req.URL))
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalRead) / duration.Seconds()
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalRead,
|
||||
TotalSize: totalRead,
|
||||
Duration: duration,
|
||||
Protocol: "GEMINI",
|
||||
IPVersion: 4,
|
||||
Speed: speed,
|
||||
OutputPath: req.Output,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
type stringWriter struct{ sb *strings.Builder }
|
||||
|
||||
func (w *stringWriter) Write(p []byte) (int, error) { return w.sb.Write(p) }
|
||||
|
||||
func testTLSConfig() *tls.Config {
|
||||
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "localhost"},
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
}
|
||||
certDER, _ := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{{
|
||||
Certificate: [][]byte{certDER},
|
||||
PrivateKey: key,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiSuccess(t *testing.T) {
|
||||
cfg := testTLSConfig()
|
||||
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, _ := listener.Accept()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
r := bufio.NewReader(conn)
|
||||
r.ReadString('\n')
|
||||
conn.Write([]byte("20 text/gemini\r\n# Hello\n\nContent.\n"))
|
||||
}()
|
||||
|
||||
proto := NewProtocol()
|
||||
proto.TLSInsecure = true
|
||||
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
|
||||
var buf strings.Builder
|
||||
_, err = proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Writer: &stringWriter{&buf},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(buf.String(), "Hello") {
|
||||
t.Errorf("got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiTemporaryFailure(t *testing.T) {
|
||||
cfg := testTLSConfig()
|
||||
listener, _ := tls.Listen("tcp", "127.0.0.1:0", cfg)
|
||||
defer listener.Close()
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, _ := listener.Accept()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
r := bufio.NewReader(conn)
|
||||
r.ReadString('\n')
|
||||
conn.Write([]byte("40 overloaded\r\n"))
|
||||
}()
|
||||
|
||||
proto := NewProtocol()
|
||||
proto.TLSInsecure = true
|
||||
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
|
||||
_, err := proto.Download(context.Background(), &core.DownloadRequest{URL: u})
|
||||
if err == nil {
|
||||
t.Error("expected temporary failure error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiPermanentFailure(t *testing.T) {
|
||||
cfg := testTLSConfig()
|
||||
listener, _ := tls.Listen("tcp", "127.0.0.1:0", cfg)
|
||||
defer listener.Close()
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, _ := listener.Accept()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
r := bufio.NewReader(conn)
|
||||
r.ReadString('\n')
|
||||
conn.Write([]byte("51 not found\r\n"))
|
||||
}()
|
||||
|
||||
proto := NewProtocol()
|
||||
proto.TLSInsecure = true
|
||||
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
|
||||
_, err := proto.Download(context.Background(), &core.DownloadRequest{URL: u})
|
||||
if err == nil {
|
||||
t.Error("expected permanent failure error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package gopher
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
)
|
||||
|
||||
// Gopher item types (RFC 1436).
|
||||
const (
|
||||
typeTextFile = '0'
|
||||
typeDirectory = '1'
|
||||
typeError = '3'
|
||||
typeBinaryFile = '9'
|
||||
typeGIF = 'g'
|
||||
typeImage = 'I'
|
||||
typeHTML = 'h'
|
||||
typeInformation = 'i'
|
||||
typeSound = 's'
|
||||
typeTelnet = '8'
|
||||
typeSearch = '7' // index search
|
||||
)
|
||||
|
||||
// Protocol implements gopher:// downloads (RFC 1436).
|
||||
type Protocol struct {
|
||||
*protocol.BaseProtocol
|
||||
}
|
||||
|
||||
// NewProtocol creates a new Gopher protocol handler.
|
||||
func NewProtocol() *Protocol {
|
||||
return &Protocol{
|
||||
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
|
||||
Name: "GOPHER",
|
||||
Scheme: "gopher",
|
||||
DefaultPort: 70,
|
||||
Features: []string{},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
host := req.URL.Hostname()
|
||||
port := req.URL.Port()
|
||||
if port == "" {
|
||||
port = "70"
|
||||
}
|
||||
selector := req.URL.Path
|
||||
if req.URL.RawQuery != "" {
|
||||
selector += "?" + req.URL.RawQuery
|
||||
}
|
||||
if selector == "" {
|
||||
selector = "/"
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(host, port)
|
||||
conn, err := net.DialTimeout("tcp", addr, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to connect to gopher server", err, core.SafeURL(req.URL))
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
|
||||
// Send selector + CRLF
|
||||
if _, err := fmt.Fprintf(conn, "%s\r\n", selector); err != nil {
|
||||
return nil, core.NewNetworkError("failed to send gopher selector", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
// Peek at first byte to determine response type
|
||||
reader := bufio.NewReader(conn)
|
||||
firstByte, err := reader.Peek(1)
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to read gopher response", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
itemType := firstByte[0]
|
||||
|
||||
switch itemType {
|
||||
case typeError:
|
||||
// Type 3: error — read the error message
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, core.NewNetworkError("failed to read gopher error", err, core.SafeURL(req.URL))
|
||||
}
|
||||
msg := strings.TrimSpace(line)
|
||||
if len(msg) > 2 {
|
||||
msg = msg[2:] // strip type + first char
|
||||
}
|
||||
return nil, core.NewProtocolError("gopher server error: "+msg, nil, core.SafeURL(req.URL))
|
||||
|
||||
case typeDirectory:
|
||||
// Type 1: directory listing — format as readable text
|
||||
return p.downloadDirectory(reader, req, startTime)
|
||||
|
||||
case typeTextFile, typeHTML, typeInformation:
|
||||
// Text-based types — strip item-type prefix, handle .\r\n terminator
|
||||
return p.downloadText(ctx, reader, req, startTime, itemType)
|
||||
|
||||
case typeSearch:
|
||||
// Type 7: index search — server expects a query
|
||||
return nil, core.NewProtocolError(
|
||||
"gopher search server requires a query (append ?query to URL)", nil, core.SafeURL(req.URL))
|
||||
|
||||
case typeTelnet:
|
||||
// Type 8: telnet session — interactive, cannot download
|
||||
return nil, core.NewProtocolError(
|
||||
"gopher telnet session type cannot be downloaded", nil, core.SafeURL(req.URL))
|
||||
|
||||
default:
|
||||
// Binary types (9, g, I, s, etc.) — raw data with .\r\n termination
|
||||
// But first check which ones have the type-prefix-per-line format
|
||||
if isTextGopherType(itemType) {
|
||||
return p.downloadText(ctx, reader, req, startTime, itemType)
|
||||
}
|
||||
return p.downloadBinary(ctx, reader, req, startTime)
|
||||
}
|
||||
}
|
||||
|
||||
// isTextGopherType returns true for types that use line-prefix format.
|
||||
func isTextGopherType(t byte) bool {
|
||||
switch t {
|
||||
case typeTextFile, typeDirectory, typeError, typeInformation, typeHTML:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// downloadText handles text responses where each line has a type prefix.
|
||||
func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time, firstType byte) (*core.DownloadResult, error) {
|
||||
var writer io.Writer
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
}
|
||||
|
||||
var totalRead int64
|
||||
lineCount := 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, core.NewNetworkError("failed to read gopher line", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
// End-of-transmission: line containing only ".\r\n"
|
||||
if line == ".\r\n" || line == ".\n" {
|
||||
break
|
||||
}
|
||||
|
||||
// Strip item-type prefix if present (first char = type, second char is usually tab/space)
|
||||
displayLine := line
|
||||
if len(line) > 1 && line[1] != '.' {
|
||||
// Full menu line: type + display string + tab + selector + tab + host + tab + port
|
||||
if true {
|
||||
// For text downloads, strip the type and metadata, show only display string + \n
|
||||
clean := stripGopherMeta(line)
|
||||
displayLine = clean + "\n"
|
||||
}
|
||||
}
|
||||
|
||||
// For informational lines (type i), don't strip anything — just pass through
|
||||
if firstType == typeInformation {
|
||||
displayLine = line
|
||||
}
|
||||
|
||||
n, err := writer.Write([]byte(displayLine))
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to write gopher text", err)
|
||||
}
|
||||
totalRead += int64(n)
|
||||
lineCount++
|
||||
|
||||
if req.ProgressCallback != nil {
|
||||
speed := float64(totalRead) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(totalRead, -1, speed)
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalRead) / duration.Seconds()
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalRead,
|
||||
TotalSize: totalRead,
|
||||
Duration: duration,
|
||||
Protocol: "GOPHER",
|
||||
IPVersion: 4,
|
||||
Speed: speed,
|
||||
OutputPath: req.Output,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// downloadDirectory formats a directory listing as readable text.
|
||||
func (p *Protocol) downloadDirectory(reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
|
||||
var writer io.Writer
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
}
|
||||
|
||||
var totalRead int64
|
||||
var sb strings.Builder
|
||||
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, core.NewNetworkError("failed to read gopher directory", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
if line == ".\r\n" || line == ".\n" {
|
||||
break
|
||||
}
|
||||
|
||||
// Parse gopher menu line: type + display string + tab + selector + tab + host + tab + port
|
||||
formatted := formatGopherMenu(line)
|
||||
sb.WriteString(formatted)
|
||||
sb.WriteByte('\n')
|
||||
}
|
||||
|
||||
output := sb.String()
|
||||
n, err := writer.Write([]byte(output))
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to write gopher directory", err)
|
||||
}
|
||||
totalRead = int64(n)
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalRead) / duration.Seconds()
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalRead,
|
||||
TotalSize: totalRead,
|
||||
Duration: duration,
|
||||
Protocol: "GOPHER",
|
||||
IPVersion: 4,
|
||||
Speed: speed,
|
||||
OutputPath: req.Output,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// downloadBinary handles binary responses.
|
||||
func (p *Protocol) downloadBinary(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
|
||||
var writer io.Writer
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
}
|
||||
|
||||
var totalRead int64
|
||||
buf := make([]byte, 32*1024)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := writer.Write(buf[:n]); werr != nil {
|
||||
return nil, core.NewFileError("failed to write gopher binary data", werr)
|
||||
}
|
||||
totalRead += int64(n)
|
||||
if req.ProgressCallback != nil {
|
||||
speed := float64(totalRead) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(totalRead, -1, speed)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, core.NewNetworkError("failed to read gopher binary", err, core.SafeURL(req.URL))
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalRead) / duration.Seconds()
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalRead,
|
||||
TotalSize: totalRead,
|
||||
Duration: duration,
|
||||
Protocol: "GOPHER",
|
||||
IPVersion: 4,
|
||||
Speed: speed,
|
||||
OutputPath: req.Output,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// stripGopherMeta removes the gopher metadata (tabs + selector + host + port) from a menu line.
|
||||
func stripGopherMeta(line string) string {
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
if len(line) < 2 {
|
||||
return line
|
||||
}
|
||||
// Format: Xdisplay string\tselector\thost\tport
|
||||
// Everything before the first tab is the type + display string
|
||||
tabIdx := strings.IndexByte(line, '\t')
|
||||
if tabIdx <= 1 {
|
||||
return line // no metadata
|
||||
}
|
||||
return line[1:tabIdx] // strip type char, keep display text
|
||||
}
|
||||
|
||||
// formatGopherMenu formats a gopher menu line for human display.
|
||||
func formatGopherMenu(line string) string {
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
if len(line) < 2 {
|
||||
return line
|
||||
}
|
||||
|
||||
itemType := line[0]
|
||||
rest := line[1:]
|
||||
|
||||
// Split by tabs: display\0selector\0host\0port
|
||||
parts := strings.Split(rest, "\t")
|
||||
display := parts[0]
|
||||
selector := ""
|
||||
if len(parts) > 1 {
|
||||
selector = parts[1]
|
||||
}
|
||||
|
||||
icon := gopherIcon(itemType)
|
||||
|
||||
if selector != "" && selector != "(null)" {
|
||||
return fmt.Sprintf("%s %s [%s]", icon, display, selector)
|
||||
}
|
||||
return fmt.Sprintf("%s %s", icon, display)
|
||||
}
|
||||
|
||||
// gopherIcon returns an icon for a gopher item type.
|
||||
func gopherIcon(t byte) string {
|
||||
switch t {
|
||||
case typeDirectory:
|
||||
return "📁"
|
||||
case typeTextFile:
|
||||
return "📄"
|
||||
case typeBinaryFile:
|
||||
return "💾"
|
||||
case typeGIF, typeImage:
|
||||
return "🖼"
|
||||
case typeHTML:
|
||||
return "🌐"
|
||||
case typeSound:
|
||||
return "🔊"
|
||||
case typeSearch:
|
||||
return "🔍"
|
||||
case typeTelnet:
|
||||
return "🖥"
|
||||
case typeInformation:
|
||||
return "ℹ️"
|
||||
default:
|
||||
return "❓"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package gopher
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
func TestFormatGopherMenu(t *testing.T) {
|
||||
tests := []struct {
|
||||
line string
|
||||
expected string
|
||||
}{
|
||||
{"0About\t/about\tgopher.floodgap.com\t70\r\n", "📄 About [/about]"},
|
||||
{"1Fun\t/fun\tgopher.floodgap.com\t70\r\n", "📁 Fun [/fun]"},
|
||||
{"iWelcome to Gopher\t(null)\t(null)\t0\r\n", "ℹ️ Welcome to Gopher"},
|
||||
{"hNews\t/news\tgopher.floodgap.com\t70\r\n", "🌐 News [/news]"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
result := formatGopherMenu(tc.line)
|
||||
if result != tc.expected {
|
||||
t.Errorf("formatGopherMenu(%q) = %q, want %q", tc.line, result, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripGopherMeta(t *testing.T) {
|
||||
result := stripGopherMeta("0About\t/about\thost\t70")
|
||||
if result != "About" {
|
||||
t.Errorf("got %q, want %q", result, "About")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGopherIcon(t *testing.T) {
|
||||
if i := gopherIcon('0'); i != "📄" {
|
||||
t.Errorf("got %q", i)
|
||||
}
|
||||
if i := gopherIcon('1'); i != "📁" {
|
||||
t.Errorf("got %q", i)
|
||||
}
|
||||
if i := gopherIcon('9'); i != "💾" {
|
||||
t.Errorf("got %q", i)
|
||||
}
|
||||
if i := gopherIcon('h'); i != "🌐" {
|
||||
t.Errorf("got %q", i)
|
||||
}
|
||||
if i := gopherIcon('7'); i != "🔍" {
|
||||
t.Errorf("got %q", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTextGopherType(t *testing.T) {
|
||||
if !isTextGopherType('0') {
|
||||
t.Error("0 should be text")
|
||||
}
|
||||
if isTextGopherType('9') {
|
||||
t.Error("9 should not be text")
|
||||
}
|
||||
}
|
||||
|
||||
type stringWriter struct{ sb *strings.Builder }
|
||||
|
||||
func (w *stringWriter) Write(p []byte) (int, error) { return w.sb.Write(p) }
|
||||
|
||||
func TestGopherDownloadMock(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, _ := listener.Accept()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
reader := bufio.NewReader(conn)
|
||||
reader.ReadString('\n') // selector
|
||||
conn.Write([]byte("0Hello from gopher\t(null)\t(null)\t0\r\n"))
|
||||
conn.Write([]byte(".\r\n"))
|
||||
}()
|
||||
|
||||
proto := NewProtocol()
|
||||
u, _ := url.Parse("gopher://127.0.0.1:" + strconv.Itoa(port) + "/")
|
||||
|
||||
var buf strings.Builder
|
||||
_, err = proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
Writer: &stringWriter{&buf},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(buf.String(), "Hello from gopher") {
|
||||
t.Errorf("output missing text: %q", buf.String())
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
"codeberg.org/petrbalvin/goget/internal/hsts"
|
||||
"codeberg.org/petrbalvin/goget/internal/transport"
|
||||
)
|
||||
|
||||
// TransportConfig groups network, TLS, proxy, and connection settings.
|
||||
type TransportConfig struct {
|
||||
// Connection
|
||||
Timeout time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
MaxTime time.Duration
|
||||
MaxFileSize int64
|
||||
BufferSize int
|
||||
FollowRedirects bool
|
||||
MaxRedirects int
|
||||
NoClobber bool
|
||||
|
||||
// TLS and certificates
|
||||
TLS *transport.TLSConfig
|
||||
CACertFile string
|
||||
HSTSCache *hsts.Cache
|
||||
|
||||
// Proxy and dialer
|
||||
Proxy *transport.ProxyConfig
|
||||
Dialer *transport.DialerConfig
|
||||
BindInterface string
|
||||
|
||||
// DNS and IP filtering
|
||||
DNSServers []string
|
||||
BlockPrivateIPs bool
|
||||
|
||||
// Retry
|
||||
Retry *transport.RetryConfig
|
||||
MaxRetries int
|
||||
RetryDelay time.Duration
|
||||
RetryHTTPCodes []int
|
||||
RetryAllErrors bool
|
||||
}
|
||||
|
||||
// AuthConfig groups authentication settings.
|
||||
type AuthConfig struct {
|
||||
Username string
|
||||
Password string
|
||||
AuthType string // "basic", "digest", "auto"
|
||||
OAuth *auth.OAuthConfig
|
||||
}
|
||||
|
||||
// RecursiveConfig groups recursive download and mirroring settings.
|
||||
type RecursiveConfig struct {
|
||||
// Core recursion
|
||||
Enabled bool
|
||||
MaxDepth int
|
||||
FollowExternal bool
|
||||
ExcludePatterns []string
|
||||
Accept string
|
||||
Reject []string
|
||||
|
||||
// Parallel worker count for recursive downloads. 0 falls back to
|
||||
// Parallel.Connections (or the default). Applies to all recursive
|
||||
// protocols that support parallel file processing.
|
||||
Parallel int
|
||||
|
||||
// Scope control
|
||||
NoParent bool
|
||||
SpanHosts bool
|
||||
PageRequisites bool
|
||||
Domains []string
|
||||
|
||||
// Rate limiting for recursion
|
||||
Wait time.Duration
|
||||
RandomWait bool
|
||||
|
||||
// Output control
|
||||
CutDirs int
|
||||
AdjustExt bool
|
||||
DryRun bool
|
||||
|
||||
// Mirror mode
|
||||
Mirror bool
|
||||
MirrorAssets bool
|
||||
ConvertLinks bool
|
||||
RespectRobots bool
|
||||
}
|
||||
|
||||
// OutputConfig groups output, progress, and display settings.
|
||||
type OutputConfig struct {
|
||||
OutputDir string
|
||||
Verbose bool
|
||||
JSONOutput bool
|
||||
ShowHeaders bool
|
||||
ContentDisposition bool
|
||||
KeepTimestamps bool
|
||||
WriteOut string
|
||||
TraceTime bool
|
||||
OnComplete string
|
||||
WarcFile string
|
||||
AutoExtract bool
|
||||
ExtractDir string
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// URLInfo contains extracted information from an HTTP URL
|
||||
type URLInfo struct {
|
||||
Scheme string
|
||||
Host string
|
||||
Port string
|
||||
Path string
|
||||
Query string
|
||||
Fragment string
|
||||
User *url.Userinfo
|
||||
}
|
||||
|
||||
// Handler contains HTTP-specific logic
|
||||
type Handler struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewHandler creates new handler
|
||||
func NewHandler(client *Client) *Handler {
|
||||
return &Handler{client: client}
|
||||
}
|
||||
|
||||
// ParseHTTPURL parses an HTTP URL and extracts information
|
||||
func ParseHTTPURL(u *url.URL) (*URLInfo, error) {
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid http url scheme: %s", u.Scheme)
|
||||
}
|
||||
|
||||
host := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
if u.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
return &URLInfo{
|
||||
Scheme: u.Scheme,
|
||||
Host: host,
|
||||
Port: port,
|
||||
Path: u.Path,
|
||||
Query: u.RawQuery,
|
||||
Fragment: u.Fragment,
|
||||
User: u.User,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetContentType extracts Content-Type from the response
|
||||
func GetContentType(resp *http.Response) string {
|
||||
return resp.Header.Get("Content-Type")
|
||||
}
|
||||
|
||||
// GetContentLength extracts Content-Length from the response
|
||||
func GetContentLength(resp *http.Response) int64 {
|
||||
return resp.ContentLength
|
||||
}
|
||||
|
||||
// GetContentEncoding extracts Content-Encoding from the response
|
||||
func GetContentEncoding(resp *http.Response) string {
|
||||
return resp.Header.Get("Content-Encoding")
|
||||
}
|
||||
|
||||
// GetETag extracts ETag from the response
|
||||
func GetETag(resp *http.Response) string {
|
||||
return resp.Header.Get("ETag")
|
||||
}
|
||||
|
||||
// GetLastModified extracts Last-Modified from the response
|
||||
func GetLastModified(resp *http.Response) string {
|
||||
return resp.Header.Get("Last-Modified")
|
||||
}
|
||||
|
||||
// SupportsRangeCheck checks whether the server supports range requests
|
||||
func SupportsRangeCheck(resp *http.Response) bool {
|
||||
acceptRanges := resp.Header.Get("Accept-Ranges")
|
||||
return strings.ToLower(acceptRanges) == "bytes"
|
||||
}
|
||||
|
||||
// IsRedirect checks whether the response is a redirect
|
||||
func IsRedirect(statusCode int) bool {
|
||||
return statusCode >= 300 && statusCode < 400
|
||||
}
|
||||
|
||||
// GetRedirectLocation gets the Location header from a redirect response
|
||||
func GetRedirectLocation(resp *http.Response) string {
|
||||
return resp.Header.Get("Location")
|
||||
}
|
||||
|
||||
// BuildURLWithQuery creates a URL with query parameters
|
||||
func BuildURLWithQuery(base *url.URL, params map[string]string) (*url.URL, error) {
|
||||
if base == nil {
|
||||
return nil, fmt.Errorf("base url cannot be nil")
|
||||
}
|
||||
|
||||
result := *base
|
||||
query := result.Query()
|
||||
|
||||
for key, value := range params {
|
||||
query.Set(key, value)
|
||||
}
|
||||
|
||||
result.RawQuery = query.Encode()
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// GetUserinfo extracts authentication credentials from the URL
|
||||
func GetUserinfo(u *url.URL) (username, password string, ok bool) {
|
||||
if u.User == nil {
|
||||
return "", "", false
|
||||
}
|
||||
username = u.User.Username()
|
||||
password, ok = u.User.Password()
|
||||
return username, password, true
|
||||
}
|
||||
|
||||
// IsSecure checks whether the URL uses HTTPS
|
||||
func IsSecure(u *url.URL) bool {
|
||||
return u.Scheme == "https"
|
||||
}
|
||||
|
||||
// GetDefaultPortForScheme returns the default port for the scheme
|
||||
func GetDefaultPortForScheme(scheme string) string {
|
||||
switch scheme {
|
||||
case "https":
|
||||
return "443"
|
||||
case "http":
|
||||
return "80"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ParseContentDisposition extracts filename from Content-Disposition header
|
||||
func ParseContentDisposition(headerValue string) string {
|
||||
if headerValue == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
_, params, err := mime.ParseMediaType(headerValue)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Try filename* first (RFC 5987, extended filename)
|
||||
if filename, ok := params["filename*"]; ok {
|
||||
return filename
|
||||
}
|
||||
|
||||
// Fall back to filename
|
||||
if filename, ok := params["filename"]; ok {
|
||||
return filename
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// SanitizeFilename removes path components from a filename for security
|
||||
func SanitizeFilename(name string) string {
|
||||
name = filepath.Base(name)
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/transport"
|
||||
)
|
||||
|
||||
// testWriter implements io.Writer for testing
|
||||
type testWriter struct {
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (w *testWriter) Write(p []byte) (int, error) {
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
|
||||
func TestDownloadSequentialIntegration(t *testing.T) {
|
||||
// Create a test server that returns known content
|
||||
testContent := "Hello, this is test content for download integration testing!"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Parse the server URL
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
// Create the HTTP client
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Create download request
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Execute download
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded != int64(len(testContent)) {
|
||||
t.Errorf("Expected %d bytes downloaded, got %d", len(testContent), result.BytesDownloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadParallelIntegration(t *testing.T) {
|
||||
// Create test content for parallel download
|
||||
testContent := strings.Repeat("B", 1024*1024) // 1MB
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Support range requests
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start, end int64
|
||||
n, _ := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
|
||||
if n >= 1 {
|
||||
if end == 0 || end >= int64(len(testContent)) {
|
||||
end = int64(len(testContent)) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(testContent)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write([]byte(testContent[start : end+1]))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Parallel = &core.ParallelConfig{
|
||||
Connections: 4,
|
||||
MinSize: 100, // Force parallel for small files
|
||||
MinChunkSize: 100,
|
||||
MaxChunkSize: 1024 * 1024,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
Parallel: cfg.Parallel,
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if !result.Parallel {
|
||||
t.Log("Download was not parallel (may be expected depending on server negotiation)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadResumeIntegration(t *testing.T) {
|
||||
testContent := "This is content for resume testing!"
|
||||
var requestCount int
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("ETag", "\"test-etag-123\"")
|
||||
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start int64
|
||||
fmt.Sscanf(rangeHeader, "bytes=%d-", &start)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(testContent)-1, len(testContent)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write([]byte(testContent[start:]))
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(testContent)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(testContent))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Create resume info simulating a partially downloaded file
|
||||
resumeInfo := &core.ResumeInfo{
|
||||
DownloadedBytes: 10,
|
||||
ETag: "\"test-etag-123\"",
|
||||
URL: server.URL,
|
||||
}
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Output: "",
|
||||
Writer: &testWriter{},
|
||||
Resume: true,
|
||||
ResumeInfo: resumeInfo,
|
||||
Verbose: false,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesDownloaded <= 0 {
|
||||
t.Errorf("Expected bytes downloaded > 0, got %d", result.BytesDownloaded)
|
||||
}
|
||||
|
||||
if result.Resumed {
|
||||
t.Log("Download was resumed successfully")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadIntegration(t *testing.T) {
|
||||
var receivedBody string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read body", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
receivedBody = string(body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
// Test POST upload with form data
|
||||
formReq := &UploadRequest{
|
||||
URL: server.URL,
|
||||
Method: "POST",
|
||||
FormData: map[string]string{"key": "value"},
|
||||
Timeout: 0,
|
||||
}
|
||||
|
||||
result, err := client.Upload(context.Background(), formReq)
|
||||
if err != nil {
|
||||
t.Fatalf("Upload failed: %v", err)
|
||||
}
|
||||
|
||||
if result.StatusCode != http.StatusOK {
|
||||
t.Errorf("Expected status 200, got %d", result.StatusCode)
|
||||
}
|
||||
|
||||
if result.BytesUploaded <= 0 {
|
||||
t.Errorf("Expected bytes uploaded > 0, got %d", result.BytesUploaded)
|
||||
}
|
||||
|
||||
if !strings.Contains(receivedBody, "key=value") {
|
||||
t.Errorf("Expected body to contain form data, got: %s", receivedBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFollowsRedirectIntegration(t *testing.T) {
|
||||
const finalBody = "redirected content"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/redirect" {
|
||||
http.Redirect(w, r, "/target", http.StatusFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(finalBody))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL + "/redirect")
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
client, err := NewClient(DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if result.BytesDownloaded != int64(len(finalBody)) {
|
||||
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetryAllErrorsClosesBody exercises the RetryAllErrors loop with a
|
||||
// server that always answers 500. It verifies the loop actually attempts the
|
||||
// configured number of retries and that each attempt's response body is
|
||||
// closed — guarding against the regression where http.Client.Do returned a
|
||||
// non-nil resp alongside an error and the body was never closed, exhausting
|
||||
// the connection pool.
|
||||
func TestRetryAllErrorsClosesBody(t *testing.T) {
|
||||
var requestCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
// Write a non-empty body so a leaking response body would matter
|
||||
// (an empty body is closed trivially by net/http's chunked reader).
|
||||
_, _ = w.Write([]byte("server error"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Transport.RetryAllErrors = true
|
||||
cfg.Transport.MaxRetries = 3
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 3,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error after exhausting retries on 500 responses")
|
||||
}
|
||||
|
||||
// We expect at least 1 (initial transport.Retry attempt) + MaxRetries
|
||||
// (RetryAllErrors loop). The exact number may be higher because Go's
|
||||
// HTTP transport may also retry once on its own when an attempt fails
|
||||
// (e.g. HTTP/2 negotiation with an HTTP/1.1 test server), so we assert
|
||||
// the lower bound only — the goal is to confirm the retry loop runs and
|
||||
// that every failed response body is closed, not to pin a Go-internal
|
||||
// detail.
|
||||
minRequests := int32(1 + cfg.Transport.MaxRetries)
|
||||
if got := atomic.LoadInt32(&requestCount); got < minRequests {
|
||||
t.Errorf("request count = %d, want >= %d (1 initial + %d retries)", got, minRequests, cfg.Transport.MaxRetries)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadRateLimiterAndProgressAreChained is a regression guard for the
|
||||
// BACKLOG entry "Rate limiter silently dropped when progress wrapper
|
||||
// enabled". The two wrappers must compose — httpReq.Body must end up
|
||||
// reading through the rate-limited reader, with progress reported on top.
|
||||
func TestUploadRateLimiterAndProgressAreChained(t *testing.T) {
|
||||
const payload = "the quick brown fox jumps over the lazy dog"
|
||||
|
||||
var receivedBody string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "read error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
receivedBody = string(body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Write payload to a temp file (Upload's PUT path opens the file).
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := filepath.Join(tmpDir, "payload.txt")
|
||||
if err := os.WriteFile(tmpFile, []byte(payload), 0600); err != nil {
|
||||
t.Fatalf("write tmp: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
// Generous rate limit so the test isn't dominated by the limiter.
|
||||
// limitedReader.Read asks for len(p) tokens on every call, and the
|
||||
// HTTP transport uses a 32 KB buffer, so burst must comfortably exceed
|
||||
// that or Allow() blocks forever.
|
||||
cfg.RateLimiter = transport.NewTokenBucket(8*1024*1024, 8*1024*1024)
|
||||
cfg.Transport.MaxRetries = 0
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 0,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
progressCalls := 0
|
||||
var lastCurrent int64
|
||||
progressCb := func(current, total int64, _ float64) {
|
||||
progressCalls++
|
||||
if current < lastCurrent {
|
||||
t.Errorf("progress went backwards: %d -> %d", lastCurrent, current)
|
||||
}
|
||||
lastCurrent = current
|
||||
}
|
||||
|
||||
_, err = client.Upload(context.Background(), &UploadRequest{
|
||||
URL: server.URL,
|
||||
Method: "PUT",
|
||||
FilePath: tmpFile,
|
||||
ProgressCallback: progressCb,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
|
||||
// The server must receive the full, unmodified payload. If the rate
|
||||
// limiter wrapper was silently dropped (e.g. httpReq.Body not updated
|
||||
// after body was reassigned), the server would still get the data via
|
||||
// the original reader — so this check alone doesn't expose the bug.
|
||||
if receivedBody != payload {
|
||||
t.Errorf("server body mismatch: got %q, want %q", receivedBody, payload)
|
||||
}
|
||||
|
||||
// Progress callback must fire at least once and reach the full payload
|
||||
// length. The exact call count depends on the read-buffer behaviour of
|
||||
// net/http (a 43-byte payload may be read in a single Read), but a count
|
||||
// of zero would prove the progress reader was bypassed — which is the
|
||||
// exact regression the BACKLOG entry warns about.
|
||||
if progressCalls < 1 {
|
||||
t.Errorf("progress callback called %d times, want >= 1 (progress reader must be in the chain)", progressCalls)
|
||||
}
|
||||
if lastCurrent != int64(len(payload)) {
|
||||
t.Errorf("final progress current = %d, want %d", lastCurrent, len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetryAllErrorsRecoversOn200 verifies that a 200 response after a few
|
||||
// 500s short-circuits the retry loop and produces a successful download —
|
||||
// another code path that must close every preceding failed response body
|
||||
// before jumping to okStatus.
|
||||
func TestRetryAllErrorsRecoversOn200(t *testing.T) {
|
||||
const finalBody = "ok after retries"
|
||||
var requestCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&requestCount, 1)
|
||||
if n < 3 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte("server error"))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(finalBody)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(finalBody))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Transport.RetryAllErrors = true
|
||||
cfg.Transport.MaxRetries = 5
|
||||
cfg.Transport.Retry = &transport.RetryConfig{
|
||||
MaxRetries: 5,
|
||||
InitialDelay: time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("create client: %v", err)
|
||||
}
|
||||
|
||||
result, err := client.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if result.BytesDownloaded != int64(len(finalBody)) {
|
||||
t.Errorf("bytes = %d, want %d", result.BytesDownloaded, len(finalBody))
|
||||
}
|
||||
if got := atomic.LoadInt32(&requestCount); got != 3 {
|
||||
t.Errorf("request count = %d, want 3 (2 failures + 1 success)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTraceTimeTTFBOnReusedConnection is a regression guard for the
|
||||
// BACKLOG entry "Trace timing dnsStart zero-value gives bogus TTFB on
|
||||
// reused connections". The previous implementation measured TTFB as
|
||||
// time.Since(dnsStart); on a pooled HTTP connection the DNSStart
|
||||
// callback is silent, dnsStart stayed zero, and the result was ~55 years
|
||||
// instead of a plausible request-time duration. The fix anchors TTFB to
|
||||
// a requestStart timestamp captured before the trace is attached, so
|
||||
// the value is meaningful whether or not the connection is reused.
|
||||
func TestTraceTimeTTFBOnReusedConnection(t *testing.T) {
|
||||
var connectionCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&connectionCount, 1)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
parsedURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse server URL: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Output.TraceTime = true
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
|
||||
reqTemplate := func() *core.DownloadRequest {
|
||||
return &core.DownloadRequest{
|
||||
URL: parsedURL,
|
||||
Writer: &testWriter{},
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// First request opens the connection — DNSStart/ConnectStart/
|
||||
// TLSHandshakeStart all fire, populating the trace timestamps.
|
||||
if _, err := client.Download(context.Background(), reqTemplate()); err != nil {
|
||||
t.Fatalf("first download failed: %v", err)
|
||||
}
|
||||
|
||||
// Second request should be served from the pool. Whether or not
|
||||
// keep-alive actually reuses the connection in this test depends
|
||||
// on the transport's MaxIdleConnsPerHost, but the assertion is the
|
||||
// same: TTFB must be a plausible duration in either case.
|
||||
result, err := client.Download(context.Background(), reqTemplate())
|
||||
if err != nil {
|
||||
t.Fatalf("second download failed: %v", err)
|
||||
}
|
||||
|
||||
if result.TraceTimings == nil {
|
||||
t.Fatal("TraceTimings is nil despite TraceTime=true")
|
||||
}
|
||||
|
||||
// 1 hour is a generous upper bound that flags the original ~55-year
|
||||
// regression (time.Since(time.Time{}) on a fresh time.Time) without
|
||||
// being flaky on a busy CI runner.
|
||||
if ttfb := result.TraceTimings.TTFB; ttfb > time.Hour {
|
||||
t.Errorf("TTFB = %v on second request; want a plausible duration (<1h); the zero-dnsStart regression typically produces ~55 years", ttfb)
|
||||
}
|
||||
if ttfb := result.TraceTimings.TTFB; ttfb < 0 {
|
||||
t.Errorf("TTFB = %v; want non-negative", ttfb)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
func (c *Client) downloadParallel(ctx context.Context, req *core.DownloadRequest, totalSize int64, connections int, resumeChunks map[int]int64, username, password string) (*core.DownloadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
chunks := createChunks(totalSize, connections, c.config.Parallel)
|
||||
if c.config.Output.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[parallel] Split into %d chunks\n", len(chunks))
|
||||
}
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "goget-chunks-*")
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to create temp dir", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// completedChunks tracks per-chunk downloaded bytes for resume metadata.
|
||||
completedChunks := make(map[int]int64)
|
||||
var completedChunksMu sync.Mutex
|
||||
|
||||
for id, n := range resumeChunks {
|
||||
completedChunks[id] = n
|
||||
}
|
||||
|
||||
progressChan := make(chan int64, len(chunks)*2)
|
||||
var wg sync.WaitGroup
|
||||
var firstError error
|
||||
var errorMu sync.Mutex
|
||||
|
||||
for i := range chunks {
|
||||
chunk := &chunks[i]
|
||||
wg.Add(1)
|
||||
|
||||
go func(chunk *core.ChunkInfo) {
|
||||
defer wg.Done()
|
||||
|
||||
if resumeChunks != nil {
|
||||
if done, ok := resumeChunks[chunk.ID]; ok && done >= (chunk.End-chunk.Start+1) {
|
||||
progressChan <- done
|
||||
chunk.Status = 2
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
|
||||
err := c.downloadChunk(ctx, req, chunk, chunkPath, progressChan, username, password)
|
||||
if err != nil {
|
||||
errorMu.Lock()
|
||||
if firstError == nil {
|
||||
firstError = err
|
||||
}
|
||||
errorMu.Unlock()
|
||||
chunk.Status = 3
|
||||
chunk.Error = err
|
||||
return
|
||||
}
|
||||
// Save per-chunk progress for resume on interruption.
|
||||
completedChunksMu.Lock()
|
||||
completedChunks[chunk.ID] = chunk.End - chunk.Start + 1
|
||||
saveParallelResumeMetadata(req, totalSize, completedChunks)
|
||||
completedChunksMu.Unlock()
|
||||
chunk.Status = 2
|
||||
}(chunk)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(progressChan)
|
||||
}()
|
||||
|
||||
var totalDownloaded int64
|
||||
var cancelled bool
|
||||
aggregationLoop:
|
||||
for {
|
||||
select {
|
||||
case progress, ok := <-progressChan:
|
||||
if !ok {
|
||||
break aggregationLoop
|
||||
}
|
||||
totalDownloaded += progress
|
||||
if req.ProgressCallback != nil {
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalDownloaded) / duration.Seconds()
|
||||
req.ProgressCallback(totalDownloaded, totalSize, speed)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
cancelled = true
|
||||
break aggregationLoop
|
||||
}
|
||||
}
|
||||
|
||||
if cancelled {
|
||||
// Context was cancelled (SIGINT/SIGTERM). Return the error so
|
||||
// the caller can show "cancelled" instead of "complete", and
|
||||
// the progress bar renders the interrupted state. Partial
|
||||
// chunks are NOT merged — they remain in temp files for a
|
||||
// potential resume later.
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
if firstError != nil {
|
||||
return nil, firstError
|
||||
}
|
||||
|
||||
if req.Output != "" && req.Output != "-" {
|
||||
if err := mergeChunks(req.Output, chunks, tempDir); err != nil {
|
||||
return nil, core.NewFileError("failed to merge chunks", err)
|
||||
}
|
||||
}
|
||||
|
||||
if req.Output != "" {
|
||||
output.DeleteResumeMetadata(req.Output)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(0)
|
||||
if duration.Seconds() > 0 {
|
||||
speed = float64(totalSize) / duration.Seconds()
|
||||
}
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalSize,
|
||||
TotalSize: totalSize,
|
||||
Duration: duration,
|
||||
Speed: speed,
|
||||
Protocol: "HTTP/HTTPS",
|
||||
IPVersion: 6,
|
||||
OutputPath: req.Output,
|
||||
Parallel: true,
|
||||
ChunksCount: len(chunks),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// saveParallelResumeMetadata saves per-chunk progress to the resume metadata file.
|
||||
func saveParallelResumeMetadata(req *core.DownloadRequest, totalSize int64, chunks map[int]int64) {
|
||||
if req.Output == "" || req.Output == "-" {
|
||||
return
|
||||
}
|
||||
// Build a snapshot of the current chunk progress.
|
||||
snapshot := make(map[int]int64, len(chunks))
|
||||
for id, n := range chunks {
|
||||
snapshot[id] = n
|
||||
}
|
||||
meta := &output.ResumeMetadata{
|
||||
URL: req.URL.String(),
|
||||
Total: totalSize,
|
||||
Downloaded: totalChunkBytes(snapshot),
|
||||
LastWrite: time.Now(),
|
||||
Version: "1.0",
|
||||
Chunks: snapshot,
|
||||
}
|
||||
_ = meta.Save(req.Output)
|
||||
}
|
||||
|
||||
// totalChunkBytes sums the byte count of all completed chunks.
|
||||
func totalChunkBytes(chunks map[int]int64) int64 {
|
||||
var total int64
|
||||
for _, n := range chunks {
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func createChunks(totalSize int64, count int, cfg *core.ParallelConfig) []core.ChunkInfo {
|
||||
if count <= 1 || totalSize <= cfg.MinSize {
|
||||
return []core.ChunkInfo{
|
||||
{ID: 0, Start: 0, End: totalSize - 1},
|
||||
}
|
||||
}
|
||||
|
||||
// Divide the file into `count` equal-sized chunks. The last chunk
|
||||
// absorbs any remainder. MaxChunkSize is NOT used to further
|
||||
// subdivide — the user explicitly asked for N parallel connections
|
||||
// via --parallel, not N×M tiny range requests. Each chunk becomes
|
||||
// one HTTP Range request handled by one of the N worker goroutines.
|
||||
chunkSize := totalSize / int64(count)
|
||||
if chunkSize < cfg.MinChunkSize {
|
||||
chunkSize = cfg.MinChunkSize
|
||||
count = int((totalSize + chunkSize - 1) / chunkSize)
|
||||
}
|
||||
|
||||
chunks := make([]core.ChunkInfo, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
start := int64(i) * chunkSize
|
||||
end := start + chunkSize - 1
|
||||
if i == count-1 || end >= totalSize {
|
||||
end = totalSize - 1
|
||||
}
|
||||
chunks = append(chunks, core.ChunkInfo{
|
||||
ID: i,
|
||||
Start: start,
|
||||
End: end,
|
||||
})
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
func (c *Client) downloadChunk(ctx context.Context, req *core.DownloadRequest, chunk *core.ChunkInfo, outputPath string, progressChan chan<- int64, username, password string) error {
|
||||
chunk.Status = 1
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", req.URL.String(), nil)
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to create chunk request", err, req.URL.String())
|
||||
}
|
||||
|
||||
httpReq.Header.Set("User-Agent", c.config.UserAgent)
|
||||
httpReq.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", chunk.Start, chunk.End))
|
||||
|
||||
if req.ResumeInfo != nil {
|
||||
if req.ResumeInfo.ETag != "" {
|
||||
httpReq.Header.Set("If-Range", req.ResumeInfo.ETag)
|
||||
} else if req.ResumeInfo.LastModified != "" {
|
||||
httpReq.Header.Set("If-Range", req.ResumeInfo.LastModified)
|
||||
}
|
||||
}
|
||||
|
||||
if username != "" {
|
||||
auth.BasicAuth(httpReq, username, password)
|
||||
}
|
||||
|
||||
for key, value := range req.Headers {
|
||||
httpReq.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to fetch chunk", err, req.URL.String())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
c.updateHSTS(resp)
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return core.NewProtocolError(
|
||||
fmt.Sprintf("chunk HTTP error: %s", resp.Status),
|
||||
nil,
|
||||
req.URL.String(),
|
||||
)
|
||||
}
|
||||
|
||||
file, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return core.NewFileError("failed to create chunk file", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := io.Reader(resp.Body)
|
||||
if c.config.RateLimiter != nil {
|
||||
reader = c.config.RateLimiter.WrapReader(reader)
|
||||
}
|
||||
|
||||
buf := make([]byte, c.config.Transport.BufferSize)
|
||||
var downloaded int64
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
_, wErr := file.Write(buf[:n])
|
||||
if wErr != nil {
|
||||
return core.NewFileError("failed to write chunk", wErr)
|
||||
}
|
||||
downloaded += int64(n)
|
||||
if progressChan != nil {
|
||||
progressChan <- int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return core.NewNetworkError("failed to read chunk", err, req.URL.String())
|
||||
}
|
||||
}
|
||||
|
||||
chunk.Downloaded = downloaded
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeChunks(outputPath string, chunks []core.ChunkInfo, tempDir string) error {
|
||||
outFile, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
for _, chunk := range chunks {
|
||||
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%04d", chunk.ID))
|
||||
chunkFile, err := os.Open(chunkPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open chunk %d: %w", chunk.ID, err)
|
||||
}
|
||||
_, err = io.Copy(outFile, chunkFile)
|
||||
chunkFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy chunk %d: %w", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/auth"
|
||||
format "codeberg.org/petrbalvin/goget/internal/format"
|
||||
)
|
||||
|
||||
// UploadRequest represents an upload request
|
||||
type UploadRequest struct {
|
||||
// URL to upload to
|
||||
URL string
|
||||
|
||||
// Method: PUT or POST
|
||||
Method string
|
||||
|
||||
// Path to the file for upload
|
||||
FilePath string
|
||||
|
||||
// Form data fields (for POST)
|
||||
FormData map[string]string
|
||||
|
||||
// Files for upload (for POST multipart)
|
||||
Files []UploadFile
|
||||
|
||||
// Custom headers
|
||||
Headers map[string]string
|
||||
|
||||
// Timeout
|
||||
Timeout time.Duration
|
||||
|
||||
// Callback pro progress
|
||||
ProgressCallback func(current, total int64, speed float64)
|
||||
}
|
||||
|
||||
// UploadFile represents a file for multipart upload
|
||||
type UploadFile struct {
|
||||
// Field name in the form data
|
||||
FieldName string
|
||||
|
||||
// Path to the file
|
||||
FilePath string
|
||||
|
||||
// Custom file name (optional)
|
||||
FileName string
|
||||
|
||||
// Content-Type (optional)
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// UploadResult is the result of an upload
|
||||
type UploadResult struct {
|
||||
StatusCode int
|
||||
ContentLength int64
|
||||
Duration time.Duration
|
||||
BytesUploaded int64
|
||||
Speed float64
|
||||
ResponseBody string
|
||||
}
|
||||
|
||||
// Upload uploads a file using PUT or POST
|
||||
func (c *Client) Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
if req.Method == "" {
|
||||
req.Method = "PUT"
|
||||
}
|
||||
|
||||
// Create request body
|
||||
var body io.Reader
|
||||
var contentType string
|
||||
var totalSize int64
|
||||
|
||||
if req.Method == "PUT" {
|
||||
// PUT upload - direct file
|
||||
if req.FilePath == "" {
|
||||
return nil, fmt.Errorf("put upload requires --upload-file")
|
||||
}
|
||||
|
||||
fileInfo, err := os.Stat(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
totalSize = fileInfo.Size()
|
||||
|
||||
file, err := os.Open(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
body = file
|
||||
contentType = "application/octet-stream"
|
||||
|
||||
} else if req.Method == "POST" {
|
||||
// POST upload - form data or multipart
|
||||
if len(req.Files) > 0 {
|
||||
// Multipart form data
|
||||
var bodyBuffer bytes.Buffer
|
||||
writer := multipart.NewWriter(&bodyBuffer)
|
||||
|
||||
// Add regular form fields
|
||||
for key, value := range req.FormData {
|
||||
if err := writer.WriteField(key, value); err != nil {
|
||||
return nil, fmt.Errorf("failed to write form field: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add files
|
||||
for _, uf := range req.Files {
|
||||
file, err := os.Open(uf.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file %s: %w", uf.FilePath, err)
|
||||
}
|
||||
|
||||
fileName := uf.FileName
|
||||
if fileName == "" {
|
||||
fileName = filepath.Base(uf.FilePath)
|
||||
}
|
||||
|
||||
contentType := uf.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set("Content-Disposition",
|
||||
fmt.Sprintf(`form-data; name="%s"; filename="%s"`, uf.FieldName, fileName))
|
||||
h.Set("Content-Type", contentType)
|
||||
|
||||
part, err := writer.CreatePart(h)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, fmt.Errorf("failed to create form part: %w", err)
|
||||
}
|
||||
|
||||
fileInfo, _ := os.Stat(uf.FilePath)
|
||||
totalSize += fileInfo.Size()
|
||||
|
||||
_, err = io.Copy(part, file)
|
||||
file.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write file to form: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
body = &bodyBuffer
|
||||
contentType = writer.FormDataContentType()
|
||||
|
||||
} else if len(req.FormData) > 0 {
|
||||
// URL-encoded form data
|
||||
form := url.Values{}
|
||||
for key, value := range req.FormData {
|
||||
form.Set(key, value)
|
||||
}
|
||||
|
||||
body = strings.NewReader(form.Encode())
|
||||
contentType = "application/x-www-form-urlencoded"
|
||||
totalSize = int64(len(form.Encode()))
|
||||
|
||||
} else if req.FilePath != "" {
|
||||
// POST with raw file body
|
||||
fileInfo, err := os.Stat(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
totalSize = fileInfo.Size()
|
||||
|
||||
file, err := os.Open(req.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
body = file
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
if body == nil {
|
||||
return nil, fmt.Errorf("no upload data provided")
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, req.Method, req.URL, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
httpReq.Header.Set("Content-Type", contentType)
|
||||
httpReq.Header.Set("User-Agent", c.config.UserAgent)
|
||||
|
||||
for key, value := range req.Headers {
|
||||
httpReq.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Apply authentication
|
||||
if c.config.Auth.Username != "" {
|
||||
authType := c.config.Auth.AuthType
|
||||
if authType == "" {
|
||||
authType = "auto"
|
||||
}
|
||||
|
||||
if authType == "auto" || authType == "basic" {
|
||||
// Try basic auth first
|
||||
applyBasicAuth(httpReq, c.config.Auth.Username, c.config.Auth.Password)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply OAuth token if configured
|
||||
if c.config.Auth.OAuth != nil && c.config.Auth.OAuth.AccessToken != "" {
|
||||
oauthClient := auth.NewOAuthClient(c.config.Auth.OAuth)
|
||||
if oauthClient.IsTokenValid() {
|
||||
oauthClient.ApplyToken(httpReq)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute request
|
||||
|
||||
if c.config.RateLimiter != nil {
|
||||
// Wrap body with rate limiter for upload
|
||||
body = c.config.RateLimiter.WrapReader(body)
|
||||
httpReq.Body = io.NopCloser(body)
|
||||
}
|
||||
|
||||
// Track progress
|
||||
if req.ProgressCallback != nil && totalSize > 0 {
|
||||
body = &progressReader{
|
||||
r: body,
|
||||
callback: req.ProgressCallback,
|
||||
total: totalSize,
|
||||
start: startTime,
|
||||
offset: 0,
|
||||
}
|
||||
httpReq.Body = io.NopCloser(body)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upload request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
return &UploadResult{
|
||||
StatusCode: resp.StatusCode,
|
||||
ContentLength: resp.ContentLength,
|
||||
Duration: duration,
|
||||
BytesUploaded: totalSize,
|
||||
Speed: float64(totalSize) / duration.Seconds(),
|
||||
ResponseBody: string(respBody),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// UploadSimple performs a simple file upload with progress tracking
|
||||
func (c *Client) UploadSimple(ctx context.Context, method, targetURL, filePath string, verbose bool) (*UploadResult, error) {
|
||||
fileInfo, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
|
||||
totalSize := fileInfo.Size()
|
||||
_ = totalSize // Used in progress callback
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var progressCallback func(int64, int64, float64)
|
||||
if verbose {
|
||||
progressCallback = func(current, total int64, speed float64) {
|
||||
percent := float64(current) / float64(total) * 100
|
||||
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
|
||||
format.Bytes(current),
|
||||
format.Bytes(total),
|
||||
percent,
|
||||
format.Speed(speed))
|
||||
}
|
||||
}
|
||||
|
||||
req := &UploadRequest{
|
||||
URL: targetURL,
|
||||
Method: method,
|
||||
FilePath: filePath,
|
||||
Timeout: c.config.Transport.Timeout,
|
||||
ProgressCallback: progressCallback,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Copy headers from client config
|
||||
for k, v := range c.config.Headers {
|
||||
req.Headers[k] = v
|
||||
}
|
||||
|
||||
result, err := c.Upload(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// UploadMultipart performs a multipart POST upload with files
|
||||
func (c *Client) UploadMultipart(ctx context.Context, targetURL string, files []UploadFile, formData map[string]string, verbose bool) (*UploadResult, error) {
|
||||
var progressCallback func(int64, int64, float64)
|
||||
if verbose {
|
||||
progressCallback = func(current, total int64, speed float64) {
|
||||
percent := float64(current) / float64(total) * 100
|
||||
fmt.Fprintf(os.Stderr, "\r[upload] %s/%s (%.1f%%) %s/s",
|
||||
format.Bytes(current),
|
||||
format.Bytes(total),
|
||||
percent,
|
||||
format.Speed(speed))
|
||||
}
|
||||
}
|
||||
|
||||
req := &UploadRequest{
|
||||
URL: targetURL,
|
||||
Method: "POST",
|
||||
Files: files,
|
||||
FormData: formData,
|
||||
Timeout: c.config.Transport.Timeout,
|
||||
ProgressCallback: progressCallback,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
for k, v := range c.config.Headers {
|
||||
req.Headers[k] = v
|
||||
}
|
||||
|
||||
result, err := c.Upload(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if verbose {
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// applyBasicAuth applies HTTP Basic Authentication to request
|
||||
func applyBasicAuth(req *http.Request, username, password string) {
|
||||
req.SetBasicAuth(username, password)
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
// ProtocolInfo contains metadata about a protocol
|
||||
type ProtocolInfo struct {
|
||||
// Protocol name
|
||||
Name string
|
||||
|
||||
// URL scheme
|
||||
Scheme string
|
||||
|
||||
// Protocol version
|
||||
Version string
|
||||
|
||||
// Supported operations
|
||||
Operations []string
|
||||
|
||||
// Supported features
|
||||
Features []string
|
||||
|
||||
// Default port
|
||||
DefaultPort int
|
||||
}
|
||||
|
||||
// BaseProtocol is the base implementation for all protocols
|
||||
type BaseProtocol struct {
|
||||
info ProtocolInfo
|
||||
}
|
||||
|
||||
// NewBaseProtocol creates a new base protocol
|
||||
func NewBaseProtocol(info ProtocolInfo) *BaseProtocol {
|
||||
return &BaseProtocol{info: info}
|
||||
}
|
||||
|
||||
// Scheme returns the protocol scheme
|
||||
func (bp *BaseProtocol) Scheme() string {
|
||||
return bp.info.Scheme
|
||||
}
|
||||
|
||||
// Name returns the protocol name
|
||||
func (bp *BaseProtocol) Name() string {
|
||||
return bp.info.Name
|
||||
}
|
||||
|
||||
// Capabilities returns the list of supported features
|
||||
func (bp *BaseProtocol) Capabilities() []string {
|
||||
return bp.info.Features
|
||||
}
|
||||
|
||||
// SupportsResume returns whether the protocol supports resume
|
||||
func (bp *BaseProtocol) SupportsResume() bool {
|
||||
return contains(bp.info.Features, "resume")
|
||||
}
|
||||
|
||||
// SupportsCompression returns whether the protocol supports compression
|
||||
func (bp *BaseProtocol) SupportsCompression() bool {
|
||||
return contains(bp.info.Features, "compression")
|
||||
}
|
||||
|
||||
// SupportsUpload returns whether the protocol supports upload
|
||||
func (bp *BaseProtocol) SupportsUpload() bool {
|
||||
return contains(bp.info.Features, "upload")
|
||||
}
|
||||
|
||||
// SupportsRange returns whether the protocol supports range requests
|
||||
func (bp *BaseProtocol) SupportsRange() bool {
|
||||
return contains(bp.info.Features, "range")
|
||||
}
|
||||
|
||||
// CanHandle checks if this protocol can handle the given URL.
|
||||
func (bp *BaseProtocol) CanHandle(url *url.URL) bool {
|
||||
if url == nil {
|
||||
return false
|
||||
}
|
||||
return url.Scheme == bp.info.Scheme
|
||||
}
|
||||
|
||||
// SupportsParallel returns whether the protocol supports parallel downloads.
|
||||
func (bp *BaseProtocol) SupportsParallel() bool {
|
||||
return contains(bp.info.Features, "parallel")
|
||||
}
|
||||
|
||||
// SupportsRecursive returns whether the protocol supports recursive downloads.
|
||||
func (bp *BaseProtocol) SupportsRecursive() bool {
|
||||
return contains(bp.info.Features, "recursive")
|
||||
}
|
||||
|
||||
// contains is a helper function
|
||||
func contains(slice []string, item string) bool {
|
||||
for _, s := range slice {
|
||||
if s == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ValidateURL validates a URL for the given protocol
|
||||
func ValidateURL(u *url.URL, scheme string) error {
|
||||
if u == nil {
|
||||
return core.NewProtocolError("URL is nil", nil, "")
|
||||
}
|
||||
|
||||
if u.Scheme != scheme {
|
||||
return core.NewProtocolError(
|
||||
fmt.Sprintf("expected scheme %s, got %s", scheme, u.Scheme),
|
||||
nil,
|
||||
u.String(),
|
||||
)
|
||||
}
|
||||
|
||||
if u.Host == "" {
|
||||
return core.NewProtocolError("missing host in URL", nil, u.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDefaultPort returns the default port for a scheme
|
||||
func GetDefaultPort(scheme string) int {
|
||||
switch scheme {
|
||||
case "http":
|
||||
return 80
|
||||
case "https", "webdavs":
|
||||
return 443
|
||||
case "webdav":
|
||||
// Plain WebDAV (no TLS) defaults to 80, matching HTTP.
|
||||
// Previously this returned 443 — a leftover from bundling
|
||||
// "webdav" with the TLS-bearing schemes — which made
|
||||
// non-TLS WebDAV connections fail against servers not
|
||||
// listening on 443.
|
||||
return 80
|
||||
case "ftp":
|
||||
return 21
|
||||
case "ftps":
|
||||
return 990
|
||||
case "sftp":
|
||||
return 22
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,47 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
// Package register provides a single registration point for all built-in goget protocols.
|
||||
// Both CLI (cmd/goget) and library API (pkg/api) call RegisterAll to avoid
|
||||
// duplicating the protocol registration list across packages.
|
||||
package register
|
||||
|
||||
import (
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/dataurl"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/file"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/ftp"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/gemini"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/gopher"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/http"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/sftp"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol/webdav"
|
||||
)
|
||||
|
||||
// RegisterAll registers all built-in protocols on the given registry.
|
||||
// Protocols that are already registered are silently skipped, making this
|
||||
// safe to call from both CLI and API initialization.
|
||||
// Returns the HTTP protocol handler for further configuration (e.g., HSTS).
|
||||
func RegisterAll(r *protocol.Registry) (*http.Client, error) {
|
||||
registerIfMissing := func(p core.Protocol) {
|
||||
if r.Supports(p.Scheme()) {
|
||||
return
|
||||
}
|
||||
_ = r.Register(p)
|
||||
}
|
||||
|
||||
httpClient, err := http.NewClient(http.DefaultConfig())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registerIfMissing(httpClient)
|
||||
registerIfMissing(ftp.NewProtocol())
|
||||
registerIfMissing(file.NewProtocol())
|
||||
registerIfMissing(dataurl.NewProtocol())
|
||||
registerIfMissing(gopher.NewProtocol())
|
||||
registerIfMissing(gemini.NewProtocol())
|
||||
registerIfMissing(sftp.NewProtocol())
|
||||
registerIfMissing(webdav.NewProtocol())
|
||||
return httpClient, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
)
|
||||
|
||||
// Registry manages all registered protocols
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
protocols map[string]core.Protocol
|
||||
}
|
||||
|
||||
// NewRegistry creates a new registry
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{
|
||||
protocols: make(map[string]core.Protocol),
|
||||
}
|
||||
}
|
||||
|
||||
// Register registers a protocol under its primary scheme
|
||||
func (r *Registry) Register(p core.Protocol) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
scheme := p.Scheme()
|
||||
if scheme == "" {
|
||||
return fmt.Errorf("protocol scheme cannot be empty")
|
||||
}
|
||||
|
||||
if _, exists := r.protocols[scheme]; exists {
|
||||
return fmt.Errorf("protocol already registered: %s", scheme)
|
||||
}
|
||||
|
||||
r.protocols[scheme] = p
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeScheme normalizes a scheme for lookup (https -> http, webdavs -> webdav, etc.)
|
||||
func normalizeScheme(scheme string) string {
|
||||
scheme = strings.ToLower(scheme)
|
||||
switch scheme {
|
||||
case "https":
|
||||
return "http"
|
||||
case "webdavs":
|
||||
return "webdav"
|
||||
case "ftps":
|
||||
return "ftp"
|
||||
default:
|
||||
return scheme
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets a protocol for a given URL with scheme normalization
|
||||
func (r *Registry) Get(u *url.URL) (core.Protocol, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
if u == nil {
|
||||
return nil, fmt.Errorf("url cannot be nil")
|
||||
}
|
||||
|
||||
scheme := normalizeScheme(u.Scheme)
|
||||
p, exists := r.protocols[scheme]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme)
|
||||
}
|
||||
|
||||
// Additional validation via CanHandle
|
||||
if !p.CanHandle(u) {
|
||||
return nil, fmt.Errorf("protocol %s cannot handle url: %s", scheme, u.String())
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// List returns a list of all registered protocols
|
||||
func (r *Registry) List() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
schemes := make([]string, 0, len(r.protocols))
|
||||
for scheme := range r.protocols {
|
||||
schemes = append(schemes, scheme)
|
||||
}
|
||||
|
||||
sort.Strings(schemes)
|
||||
return schemes
|
||||
}
|
||||
|
||||
// Supports determines whether a protocol is supported (with normalization)
|
||||
func (r *Registry) Supports(scheme string) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
_, exists := r.protocols[normalizeScheme(scheme)]
|
||||
return exists
|
||||
}
|
||||
|
||||
// Capabilities returns the capabilities of all protocols
|
||||
func (r *Registry) Capabilities() map[string][]string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
caps := make(map[string][]string)
|
||||
for scheme, p := range r.protocols {
|
||||
caps[scheme] = p.Capabilities()
|
||||
}
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
// GlobalRegistry is the global registry instance
|
||||
var GlobalRegistry = NewRegistry()
|
||||
|
||||
// Register is a helper for registering in the global registry
|
||||
func Register(p core.Protocol) error {
|
||||
return GlobalRegistry.Register(p)
|
||||
}
|
||||
|
||||
// Get is a helper for getting a protocol from the global registry
|
||||
func Get(u *url.URL) (core.Protocol, error) {
|
||||
return GlobalRegistry.Get(u)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, error) {
|
||||
if insecure {
|
||||
return ssh.InsecureIgnoreHostKey(), nil
|
||||
}
|
||||
|
||||
if knownHostsPath == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("known_hosts: %w", err)
|
||||
}
|
||||
knownHostsPath = filepath.Join(home, ".ssh", "known_hosts")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(knownHostsPath); err != nil {
|
||||
return nil, fmt.Errorf("known_hosts file not found: %s (use --ssh-insecure to skip host key check)", knownHostsPath)
|
||||
}
|
||||
|
||||
return knownhosts.New(knownHostsPath)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHostKeyCallbackInsecure(t *testing.T) {
|
||||
cb, err := hostKeyCallback("", true)
|
||||
if err != nil {
|
||||
t.Fatalf("hostKeyCallback: %v", err)
|
||||
}
|
||||
if cb == nil {
|
||||
t.Fatal("expected callback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostKeyCallbackMissingFile(t *testing.T) {
|
||||
_, err := hostKeyCallback("/nonexistent/known_hosts", false)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing known_hosts")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
// TestSaveSFTPResume verifies the partial-progress sidecar for the
|
||||
// SFTP single-file path. The SFTP parallel workers don't read attr.Size
|
||||
// in the download loop, so Total is recorded as 0 (only Downloaded
|
||||
// is known reliably). A future enhancement could plumb totalSize
|
||||
// through sftpDownloadFile if more accurate accounting is needed.
|
||||
func TestSaveSFTPResume(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
out := filepath.Join(tmp, "file.bin")
|
||||
|
||||
u, err := url.Parse("sftp://user@host:22/file.bin")
|
||||
if err != nil {
|
||||
t.Fatalf("parse URL: %v", err)
|
||||
}
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: out,
|
||||
}
|
||||
|
||||
saveSFTPResume(req, 12345)
|
||||
|
||||
meta, err := output.LoadResumeMetadata(out)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadResumeMetadata: %v", err)
|
||||
}
|
||||
if meta == nil {
|
||||
t.Fatal("expected resume metadata, got nil")
|
||||
}
|
||||
if meta.Downloaded != 12345 {
|
||||
t.Errorf("Downloaded = %d, want 12345", meta.Downloaded)
|
||||
}
|
||||
if meta.URL != "sftp://user@host:22/file.bin" {
|
||||
t.Errorf("URL = %q", meta.URL)
|
||||
}
|
||||
// Total is 0 because the parallel worker doesn't read attr.Size;
|
||||
// the resume code path uses this as "size unknown" and works
|
||||
// regardless of Total.
|
||||
if meta.Total != 0 {
|
||||
t.Errorf("Total = %d, want 0 (size unknown for SFTP parallel workers)", meta.Total)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,623 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// startTestServer starts a minimal SSH+SFTP server for testing.
|
||||
func startTestServer(t *testing.T) (addr string, cleanup func()) {
|
||||
t.Helper()
|
||||
|
||||
config := &ssh.ServerConfig{
|
||||
NoClientAuth: true,
|
||||
}
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
signer, err := ssh.NewSignerFromKey(privateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("new signer: %v", err)
|
||||
}
|
||||
config.AddHostKey(signer)
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
// Accept connections in a loop so the parallel-download tests
|
||||
// (which open N SFTP sessions) can run. Each connection is
|
||||
// served in its own goroutine, so the N pool connections can be
|
||||
// alive concurrently as long as the test goroutine keeps
|
||||
// accepting.
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
go func(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
sshConn, chans, reqs, err := ssh.NewServerConn(conn, config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer sshConn.Close()
|
||||
go ssh.DiscardRequests(reqs)
|
||||
|
||||
for newChannel := range chans {
|
||||
if newChannel.ChannelType() != "session" {
|
||||
newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
|
||||
continue
|
||||
}
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func(ch ssh.Channel, reqs <-chan *ssh.Request) {
|
||||
for req := range reqs {
|
||||
ok := false
|
||||
if req.Type == "subsystem" && len(req.Payload) > 4 {
|
||||
l := binary.BigEndian.Uint32(req.Payload[:4])
|
||||
name := string(req.Payload[4 : 4+l])
|
||||
if name == "sftp" {
|
||||
ok = true
|
||||
if req.WantReply {
|
||||
req.Reply(ok, nil)
|
||||
}
|
||||
handleSFTPSession(ch)
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.WantReply {
|
||||
req.Reply(ok, nil)
|
||||
}
|
||||
}
|
||||
ch.Close()
|
||||
}(channel, requests)
|
||||
}
|
||||
}(conn)
|
||||
}
|
||||
}()
|
||||
|
||||
return listener.Addr().String(), func() {
|
||||
listener.Close()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// handleSFTPSession handles a minimal SFTP v3 session.
|
||||
func handleSFTPSession(ch ssh.Channel) {
|
||||
defer ch.Close()
|
||||
|
||||
// Expect INIT
|
||||
pkt, err := readTestPacket(ch)
|
||||
if err != nil || len(pkt) < 5 || pkt[0] != sshFxpInit {
|
||||
return
|
||||
}
|
||||
// Respond VERSION
|
||||
var resp bytes.Buffer
|
||||
resp.WriteByte(sshFxpVersion)
|
||||
writeUint32(&resp, 3)
|
||||
writeTestPacket(ch, resp.Bytes())
|
||||
|
||||
handles := make(map[string]string)
|
||||
dirHandles := make(map[string][]string)
|
||||
dirHandlePaths := make(map[string]string)
|
||||
handleCounter := 0
|
||||
|
||||
for {
|
||||
pkt, err := readTestPacket(ch)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(pkt) < 1 {
|
||||
continue
|
||||
}
|
||||
reqID, _ := readUint32(pkt, 1)
|
||||
|
||||
switch pkt[0] {
|
||||
case sshFxpRealpath:
|
||||
path, _ := readString(pkt, 5)
|
||||
if path == "." {
|
||||
path = "/home/test"
|
||||
}
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpName)
|
||||
writeUint32(&r, reqID)
|
||||
writeUint32(&r, 1)
|
||||
writeString(&r, path)
|
||||
writeString(&r, path)
|
||||
writeUint32(&r, 0) // attrs
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
case sshFxpStat:
|
||||
path, _ := readString(pkt, 5)
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpAttrs)
|
||||
writeUint32(&r, reqID)
|
||||
fs := testFSLookup(path)
|
||||
if fs != nil && fs.isDir {
|
||||
// mode flag only
|
||||
writeUint32(&r, 0x4)
|
||||
writeUint32(&r, 0x4000) // S_IFDIR
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
continue
|
||||
}
|
||||
if fs != nil && !fs.isDir {
|
||||
// size flag only
|
||||
writeUint32(&r, 0x1)
|
||||
writeUint64(&r, uint64(len(fs.content)))
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
continue
|
||||
}
|
||||
// legacy single-file mock behaviour
|
||||
writeUint32(&r, 0x1) // size flag
|
||||
size := uint64(len(testFileContent))
|
||||
if testOverrideAttrSize != nil {
|
||||
size = *testOverrideAttrSize
|
||||
}
|
||||
writeUint64(&r, size)
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
case sshFxpOpendir:
|
||||
path, _ := readString(pkt, 5)
|
||||
fs := testFSLookup(path)
|
||||
if fs == nil || !fs.isDir {
|
||||
status(ch, reqID, sshFxNoSuchFile, "no such directory")
|
||||
continue
|
||||
}
|
||||
handleCounter++
|
||||
h := fmt.Sprintf("dhandle%d", handleCounter)
|
||||
dirHandles[h] = fs.children
|
||||
dirHandlePaths[h] = path
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpHandle)
|
||||
writeUint32(&r, reqID)
|
||||
writeString(&r, h)
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
case sshFxpReaddir:
|
||||
handle, _ := readString(pkt, 5)
|
||||
children, ok := dirHandles[handle]
|
||||
if !ok {
|
||||
status(ch, reqID, sshFxFailure, "no such dir handle")
|
||||
continue
|
||||
}
|
||||
parentPath := dirHandlePaths[handle]
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpName)
|
||||
writeUint32(&r, reqID)
|
||||
writeUint32(&r, uint32(len(children)))
|
||||
for _, name := range children {
|
||||
writeString(&r, name)
|
||||
writeString(&r, name)
|
||||
childPath := parentPath + "/" + name
|
||||
childFS := testFSLookup(childPath)
|
||||
isDir := childFS != nil && childFS.isDir
|
||||
// attrs: size flag (0x1) and mode flag (0x4) so the
|
||||
// client can both classify the entry (dir vs file) and
|
||||
// read the file size for parallel accounting.
|
||||
if isDir {
|
||||
writeUint32(&r, 0x4)
|
||||
writeUint32(&r, 0x4000) // S_IFDIR
|
||||
} else {
|
||||
writeUint32(&r, 0x5) // size + mode
|
||||
var sz uint64
|
||||
if childFS != nil {
|
||||
sz = uint64(len(childFS.content))
|
||||
} else {
|
||||
sz = uint64(len(testFileContent))
|
||||
}
|
||||
writeUint64(&r, sz)
|
||||
writeUint32(&r, 0x8000) // S_IFREG
|
||||
}
|
||||
}
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
case sshFxpOpen:
|
||||
path, off := readString(pkt, 5)
|
||||
pflags, _ := readUint32(pkt, off)
|
||||
handleCounter++
|
||||
h := fmt.Sprintf("handle%d", handleCounter)
|
||||
handles[h] = path
|
||||
var r bytes.Buffer
|
||||
if pflags&sshFxfRead != 0 {
|
||||
r.WriteByte(sshFxpHandle)
|
||||
writeUint32(&r, reqID)
|
||||
writeString(&r, h)
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
} else if pflags&sshFxfWrite != 0 {
|
||||
r.WriteByte(sshFxpHandle)
|
||||
writeUint32(&r, reqID)
|
||||
writeString(&r, h)
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
} else {
|
||||
status(ch, reqID, sshFxFailure, "unsupported flags")
|
||||
}
|
||||
|
||||
case sshFxpRead:
|
||||
handle, off := readString(pkt, 5)
|
||||
offset, off := readUint64(pkt, off)
|
||||
length, _ := readUint32(pkt, off)
|
||||
_ = length
|
||||
path := handles[handle]
|
||||
if path == "" {
|
||||
status(ch, reqID, sshFxNoSuchFile, "no such handle")
|
||||
continue
|
||||
}
|
||||
var data []byte
|
||||
if fs := testFSLookup(path); fs != nil && !fs.isDir {
|
||||
data = fs.content
|
||||
} else {
|
||||
data = testFileContent
|
||||
}
|
||||
if int(offset) >= len(data) {
|
||||
status(ch, reqID, sshFxEOF, "")
|
||||
continue
|
||||
}
|
||||
end := int(offset) + int(length)
|
||||
if end > len(data) {
|
||||
end = len(data)
|
||||
}
|
||||
chunk := data[offset:end]
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpData)
|
||||
writeUint32(&r, reqID)
|
||||
writeUint32(&r, uint32(len(chunk)))
|
||||
r.Write(chunk)
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
case sshFxpClose:
|
||||
handle, _ := readString(pkt, 5)
|
||||
delete(handles, handle)
|
||||
delete(dirHandles, handle)
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpStatus)
|
||||
writeUint32(&r, reqID)
|
||||
writeUint32(&r, sshFxOk)
|
||||
writeString(&r, "")
|
||||
writeString(&r, "")
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
|
||||
default:
|
||||
status(ch, reqID, sshFxOpUnsupported, "unsupported")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testFSLookup is a tiny helper that hides the nil-map check so the
|
||||
// mock switch arms above can stay readable.
|
||||
func testFSLookup(path string) *testFSEntry {
|
||||
if testFS == nil {
|
||||
return nil
|
||||
}
|
||||
return testFS[path]
|
||||
}
|
||||
|
||||
func status(ch ssh.Channel, reqID, code uint32, msg string) {
|
||||
var r bytes.Buffer
|
||||
r.WriteByte(sshFxpStatus)
|
||||
writeUint32(&r, reqID)
|
||||
writeUint32(&r, code)
|
||||
writeString(&r, msg)
|
||||
writeString(&r, "")
|
||||
writeTestPacket(ch, r.Bytes())
|
||||
}
|
||||
|
||||
func readTestPacket(r io.Reader) ([]byte, error) {
|
||||
var lenBuf [4]byte
|
||||
if _, err := io.ReadFull(r, lenBuf[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
length := binary.BigEndian.Uint32(lenBuf[:])
|
||||
if length > 256*1024 {
|
||||
return nil, fmt.Errorf("packet too large")
|
||||
}
|
||||
pkt := make([]byte, length)
|
||||
if _, err := io.ReadFull(r, pkt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pkt, nil
|
||||
}
|
||||
|
||||
func writeTestPacket(w io.Writer, pkt []byte) error {
|
||||
var lenBuf [4]byte
|
||||
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(pkt)))
|
||||
if _, err := w.Write(lenBuf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.Write(pkt)
|
||||
return err
|
||||
}
|
||||
|
||||
var testFileContent = []byte("hello sftp world")
|
||||
|
||||
// testOverrideAttrSize, when non-nil, replaces the value reported by the
|
||||
// mock server's sshFxpStat handler. Tests use it to simulate a server
|
||||
// that returns a wrong (typically too large) size attribute so we can
|
||||
// verify the client does not blindly trust attr.Size for
|
||||
// BytesDownloaded accounting.
|
||||
var testOverrideAttrSize *uint64
|
||||
|
||||
// testFSEntry describes a single in-memory file or directory used by
|
||||
// the recursive-download tests. isDir==true entries carry a children
|
||||
// slice; isDir==false entries carry the file content.
|
||||
type testFSEntry struct {
|
||||
isDir bool
|
||||
content []byte
|
||||
children []string
|
||||
}
|
||||
|
||||
// testFS is an optional in-memory filesystem consulted by the mock
|
||||
// server before falling back to the default testFileContent behaviour.
|
||||
// Tests that exercise recursive downloads set it up and clear it via
|
||||
// t.Cleanup. Keys are absolute paths; paths NOT present in the map
|
||||
// keep the legacy single-file mock semantics so existing
|
||||
// non-recursive tests continue to pass.
|
||||
var testFS map[string]*testFSEntry
|
||||
|
||||
func TestDownload(t *testing.T) {
|
||||
addr, cleanup := startTestServer(t)
|
||||
defer cleanup()
|
||||
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/testfile")
|
||||
outDir := t.TempDir()
|
||||
outputPath := filepath.Join(outDir, "out")
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outputPath,
|
||||
Ctx: context.Background(),
|
||||
SSHInsecure: true,
|
||||
}
|
||||
|
||||
result, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("download failed: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data, testFileContent) {
|
||||
t.Errorf("got %q, want %q", data, testFileContent)
|
||||
}
|
||||
if result.BytesDownloaded != int64(len(testFileContent)) {
|
||||
t.Errorf("bytes=%d, want %d", result.BytesDownloaded, len(testFileContent))
|
||||
}
|
||||
if result.Protocol != "sftp" {
|
||||
t.Errorf("protocol=%q, want sftp", result.Protocol)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDownloadWithStaleAttrSize is a regression guard for the BACKLOG
|
||||
// entry "SFTP BytesDownloaded unconditionally overwritten". The previous
|
||||
// implementation set result.BytesDownloaded = int64(attr.Size) when
|
||||
// attr.Size > 0, ignoring the locally-tracked `total` counter. If the
|
||||
// server reported a stale (typically too large) size, the result would
|
||||
// over-report the bytes actually transferred — which is exactly what
|
||||
// this test exercises.
|
||||
func TestDownloadWithStaleAttrSize(t *testing.T) {
|
||||
// Simulate a server that reports 10 KiB even though the file is
|
||||
// only len(testFileContent) bytes long. After the fix, the result
|
||||
// must reflect the actual bytes received, not the stale attribute.
|
||||
wrong := uint64(10 * 1024)
|
||||
testOverrideAttrSize = &wrong
|
||||
t.Cleanup(func() { testOverrideAttrSize = nil })
|
||||
|
||||
addr, cleanup := startTestServer(t)
|
||||
defer cleanup()
|
||||
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/testfile")
|
||||
outDir := t.TempDir()
|
||||
outputPath := filepath.Join(outDir, "out")
|
||||
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outputPath,
|
||||
Ctx: context.Background(),
|
||||
SSHInsecure: true,
|
||||
}
|
||||
|
||||
result, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("download failed: %v", err)
|
||||
}
|
||||
|
||||
// The downloaded file must still hold the real (small) payload.
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data, testFileContent) {
|
||||
t.Errorf("got %q, want %q", data, testFileContent)
|
||||
}
|
||||
|
||||
// And BytesDownloaded must reflect the actual bytes received, not
|
||||
// the stale attr.Size the server returned.
|
||||
if result.BytesDownloaded != int64(len(testFileContent)) {
|
||||
t.Errorf("BytesDownloaded = %d, want %d (real payload size, not stale attr.Size=%d)",
|
||||
result.BytesDownloaded, len(testFileContent), wrong)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHeader(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
writeUint32(&buf, 0x05) // size + mode flags
|
||||
writeUint64(&buf, 60)
|
||||
writeUint32(&buf, 0644)
|
||||
|
||||
attr, off := readAttrs(buf.Bytes(), 0)
|
||||
if attr.Size != 60 {
|
||||
t.Errorf("size=%d, want 60", attr.Size)
|
||||
}
|
||||
if off != 16 {
|
||||
t.Errorf("off=%d, want 16", off)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEOF(t *testing.T) {
|
||||
if !isEOF(fmt.Errorf("sftp status %d", sshFxEOF)) {
|
||||
t.Error("expected true")
|
||||
}
|
||||
if isEOF(fmt.Errorf("sftp status %d", sshFxFailure)) {
|
||||
t.Error("expected false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialTimeout(t *testing.T) {
|
||||
u, _ := url.Parse("sftp://user@127.0.0.1:1/")
|
||||
_, err := dial(context.Background(), u, "", true)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFTPRecursiveDownloadParallel(t *testing.T) {
|
||||
addr, cleanup := startTestServer(t)
|
||||
defer cleanup()
|
||||
|
||||
setupRecursiveFS := func(t *testing.T) {
|
||||
t.Helper()
|
||||
testFS = map[string]*testFSEntry{
|
||||
"/home/test/dir": {isDir: true, children: []string{"a.txt", "b.txt", "c.txt", "sub"}},
|
||||
"/home/test/dir/a.txt": {isDir: false, content: []byte("alpha")},
|
||||
"/home/test/dir/b.txt": {isDir: false, content: []byte("bravo-bravo")},
|
||||
"/home/test/dir/c.txt": {isDir: false, content: []byte("charlie")},
|
||||
"/home/test/dir/sub": {isDir: true, children: []string{"x.txt", "y.txt"}},
|
||||
"/home/test/dir/sub/x.txt": {isDir: false, content: []byte("x-ray")},
|
||||
"/home/test/dir/sub/y.txt": {isDir: false, content: []byte("yankee")},
|
||||
}
|
||||
t.Cleanup(func() { testFS = nil })
|
||||
}
|
||||
|
||||
t.Run("parallel=3 downloads all files", func(t *testing.T) {
|
||||
setupRecursiveFS(t)
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
|
||||
outDir := t.TempDir()
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outDir,
|
||||
Recursive: true,
|
||||
RecursiveParallel: 3,
|
||||
SSHInsecure: true,
|
||||
}
|
||||
result, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("recursive parallel download failed: %v", err)
|
||||
}
|
||||
if result.ChunksCount != 5 {
|
||||
t.Errorf("ChunksCount = %d, want 5 (3 top-level files + 2 in subdir)", result.ChunksCount)
|
||||
}
|
||||
// 5 + 11 + 7 + 5 + 6 = 34 bytes total content
|
||||
const wantBytes = int64(5 + 11 + 7 + 5 + 6)
|
||||
if result.BytesDownloaded != wantBytes {
|
||||
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
|
||||
}
|
||||
// Verify all 5 files actually exist with correct content.
|
||||
wantFiles := map[string]string{
|
||||
"a.txt": "alpha",
|
||||
"b.txt": "bravo-bravo",
|
||||
"c.txt": "charlie",
|
||||
"sub/x.txt": "x-ray",
|
||||
"sub/y.txt": "yankee",
|
||||
}
|
||||
for name, want := range wantFiles {
|
||||
got, err := os.ReadFile(filepath.Join(outDir, name))
|
||||
if err != nil {
|
||||
t.Errorf("file %s: %v", name, err)
|
||||
continue
|
||||
}
|
||||
if string(got) != want {
|
||||
t.Errorf("file %s = %q, want %q", name, string(got), want)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parallel=0 falls back to sequential", func(t *testing.T) {
|
||||
setupRecursiveFS(t)
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
|
||||
outDir := t.TempDir()
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outDir,
|
||||
Recursive: true,
|
||||
RecursiveParallel: 0,
|
||||
SSHInsecure: true,
|
||||
}
|
||||
result, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("recursive sequential download failed: %v", err)
|
||||
}
|
||||
if result.ChunksCount != 5 {
|
||||
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parallel=1 falls back to sequential", func(t *testing.T) {
|
||||
setupRecursiveFS(t)
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
|
||||
outDir := t.TempDir()
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outDir,
|
||||
Recursive: true,
|
||||
RecursiveParallel: 1,
|
||||
SSHInsecure: true,
|
||||
}
|
||||
result, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("download with parallel=1 failed: %v", err)
|
||||
}
|
||||
if result.ChunksCount != 5 {
|
||||
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("subdir propagation with parallel", func(t *testing.T) {
|
||||
setupRecursiveFS(t)
|
||||
u, _ := url.Parse("sftp://user@" + addr + "/home/test/dir")
|
||||
outDir := t.TempDir()
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: outDir,
|
||||
Recursive: true,
|
||||
RecursiveParallel: 4,
|
||||
SSHInsecure: true,
|
||||
}
|
||||
_, err := download(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("download failed: %v", err)
|
||||
}
|
||||
// Both subdir files should be present and correct.
|
||||
for _, name := range []string{"sub/x.txt", "sub/y.txt"} {
|
||||
path := filepath.Join(outDir, name)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Errorf("expected file %s to exist: %v", name, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,61 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package webdav
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
)
|
||||
|
||||
// TestSaveWebDAVResume verifies the partial-progress sidecar written
|
||||
// by streamWebDAVBody on context cancellation. The ETag and
|
||||
// Last-Modified headers are preserved so the next --resume attempt
|
||||
// can send a matching If-Range and either continue or restart
|
||||
// cleanly depending on the server's view of the file.
|
||||
func TestSaveWebDAVResume(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
out := filepath.Join(tmp, "file.bin")
|
||||
|
||||
u, _ := url.Parse("webdav://host/file.bin")
|
||||
req := &core.DownloadRequest{
|
||||
URL: u,
|
||||
Output: out,
|
||||
Resume: true,
|
||||
}
|
||||
resumed := true
|
||||
|
||||
saveWebDAVResume(req, 8192, 1<<20, `"etag-abc"`, "Mon, 01 Jan 2026 00:00:00 GMT", resumed)
|
||||
|
||||
meta, err := output.LoadResumeMetadata(out)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadResumeMetadata: %v", err)
|
||||
}
|
||||
if meta == nil {
|
||||
t.Fatal("expected resume metadata, got nil")
|
||||
}
|
||||
if meta.Downloaded != 8192 {
|
||||
t.Errorf("Downloaded = %d, want 8192", meta.Downloaded)
|
||||
}
|
||||
if meta.Total != 1<<20 {
|
||||
t.Errorf("Total = %d, want %d", meta.Total, 1<<20)
|
||||
}
|
||||
if meta.ETag != `"etag-abc"` {
|
||||
t.Errorf("ETag = %q", meta.ETag)
|
||||
}
|
||||
if meta.LastModified != "Mon, 01 Jan 2026 00:00:00 GMT" {
|
||||
t.Errorf("LastModified = %q", meta.LastModified)
|
||||
}
|
||||
|
||||
// Resume=false must short-circuit the save.
|
||||
out2 := filepath.Join(tmp, "noresume.bin")
|
||||
req.Resume = false
|
||||
saveWebDAVResume(req, 4096, 1000, "", "", false)
|
||||
if _, err := output.LoadResumeMetadata(out2); err != nil {
|
||||
t.Fatalf("LoadResumeMetadata (noresume): %v", err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user