//go:build linux || freebsd
// +build linux freebsd
package webdav
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
"codeberg.org/petrbalvin/goget/internal/protocol"
httpproto "codeberg.org/petrbalvin/goget/internal/protocol/http"
)
func TestWebDAVProtocol(t *testing.T) {
if !protocol.GlobalRegistry.Supports("http") {
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
if err != nil {
t.Fatalf("failed to create http client: %v", err)
}
_ = protocol.Register(httpClient)
}
// Create a mock WebDAV server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" {
depth := r.Header.Get("Depth")
path := r.URL.Path
if depth == "0" {
if path == "/dir" || path == "/dir/" {
// Directory response
xmlResp := `
/dir/
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
if path == "/file.txt" {
// File response
xmlResp := `
/file.txt
12
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
}
if depth == "1" {
if path == "/dir" || path == "/dir/" {
// Directory contents response
xmlResp := `
/dir/
HTTP/1.1 200 OK
/dir/file1.txt
13
HTTP/1.1 200 OK
/dir/file2.txt
13
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
}
w.WriteHeader(http.StatusNotFound)
return
}
if r.Method == "GET" {
if r.URL.Path == "/file.txt" {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("file content"))
return
}
if r.URL.Path == "/dir/file1.txt" {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("file1 content"))
return
}
if r.URL.Path == "/dir/file2.txt" {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("file2 content"))
return
}
}
if r.Method == "PUT" {
if r.URL.Path == "/upload.txt" {
w.WriteHeader(http.StatusCreated)
return
}
}
w.WriteHeader(http.StatusMethodNotAllowed)
}))
defer server.Close()
// Parse server URL to construct test URLs
parsedServerURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("failed to parse mock server URL: %v", err)
}
p := NewProtocol()
// 1. Test CanHandle
t.Run("CanHandle", func(t *testing.T) {
u1, _ := url.Parse("webdav://example.com/file")
u2, _ := url.Parse("webdavs://example.com/file")
u3, _ := url.Parse("http://example.com/file")
if !p.CanHandle(u1) {
t.Error("CanHandle(webdav://) = false, want true")
}
if !p.CanHandle(u2) {
t.Error("CanHandle(webdavs://) = false, want true")
}
if p.CanHandle(u3) {
t.Error("CanHandle(http://) = true, want false")
}
})
// 2. Test File Download
t.Run("Download File", func(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "webdav-test-*")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
fileURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/file.txt",
}
outputPath := filepath.Join(tmpDir, "downloaded_file.txt")
req := &core.DownloadRequest{
URL: fileURL,
Output: outputPath,
Recursive: false,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("download failed: %v", err)
}
if result.BytesDownloaded != 12 {
t.Errorf("BytesDownloaded = %d, want 12", result.BytesDownloaded)
}
data, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read output file: %v", err)
}
if string(data) != "file content" {
t.Errorf("file content = %q, want %q", string(data), "file content")
}
})
// 3. Test Directory Detection and error on non-recursive download of directory
t.Run("Directory Non-Recursive Error", func(t *testing.T) {
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
req := &core.DownloadRequest{
URL: dirURL,
Output: "",
Recursive: false,
Timeout: 5 * time.Second,
}
_, err := p.Download(context.Background(), req)
if err == nil {
t.Fatal("expected error downloading directory non-recursively, got nil")
}
if !strings.Contains(err.Error(), "remote path is a directory") {
t.Errorf("unexpected error: %v", err)
}
})
// 4. Test Recursive Download
t.Run("Download Directory Recursive", func(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "webdav-test-*")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
outputPath := filepath.Join(tmpDir, "local_dir")
req := &core.DownloadRequest{
URL: dirURL,
Output: outputPath,
Recursive: true,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("recursive download failed: %v", err)
}
// Two files: file1.txt (13 bytes) + file2.txt (13 bytes) = 26 bytes
if result.BytesDownloaded != 26 {
t.Errorf("BytesDownloaded = %d, want 26", result.BytesDownloaded)
}
if result.ChunksCount != 2 {
t.Errorf("ChunksCount = %d, want 2", result.ChunksCount)
}
// Verify files exist in directory
f1, err := os.ReadFile(filepath.Join(outputPath, "file1.txt"))
if err != nil {
t.Errorf("file1.txt read failed: %v", err)
} else if string(f1) != "file1 content" {
t.Errorf("file1 content = %q, want %q", string(f1), "file1 content")
}
f2, err := os.ReadFile(filepath.Join(outputPath, "file2.txt"))
if err != nil {
t.Errorf("file2.txt read failed: %v", err)
} else if string(f2) != "file2 content" {
t.Errorf("file2 content = %q, want %q", string(f2), "file2 content")
}
})
// 5. Test Upload
t.Run("Upload File", func(t *testing.T) {
tmpFile, err := os.CreateTemp("", "webdav-upload-*")
if err != nil {
t.Fatalf("failed to create temp file for upload: %v", err)
}
defer os.Remove(tmpFile.Name())
content := "upload test content"
if _, err := tmpFile.Write([]byte(content)); err != nil {
t.Fatalf("failed to write upload file content: %v", err)
}
tmpFile.Close()
uploadURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/upload.txt",
}
req := &core.UploadRequest{
URL: uploadURL,
Input: tmpFile.Name(),
Timeout: 5 * time.Second,
}
result, err := p.Upload(context.Background(), req)
if err != nil {
t.Fatalf("upload failed: %v", err)
}
if result.BytesUploaded != int64(len(content)) {
t.Errorf("BytesUploaded = %d, want %d", result.BytesUploaded, len(content))
}
if result.Protocol != "webdav" {
t.Errorf("Protocol = %q, want %q", result.Protocol, "webdav")
}
})
}
func TestToHTTPURL(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{"webdav to http", "webdav://example.com/file", "http://example.com/file"},
{"webdavs to https", "webdavs://example.com/file", "https://example.com/file"},
{"already http", "http://example.com/file", "http://example.com/file"},
{"already https", "https://example.com/file", "https://example.com/file"},
{"webdav with port", "webdav://example.com:8080/file", "http://example.com:8080/file"},
{"webdavs with port", "webdavs://example.com:443/file", "https://example.com:443/file"},
{"webdav with user", "webdav://user:pass@example.com/file", "http://user:pass@example.com/file"},
{"empty path", "webdav://example.com", "http://example.com"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u, err := url.Parse(tt.input)
if err != nil {
t.Fatalf("failed to parse input url: %v", err)
}
result := toHTTPURL(u)
if result.String() != tt.want {
t.Errorf("toHTTPURL(%q) = %q, want %q", tt.input, result.String(), tt.want)
}
})
}
}
func TestFilteredRecursiveDownload(t *testing.T) {
// Register HTTP protocol handler.
if !protocol.GlobalRegistry.Supports("http") {
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
if err != nil {
t.Fatalf("failed to create http client: %v", err)
}
_ = protocol.Register(httpClient)
}
// Create mock WebDAV server with multiple entries.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
// Return unfiltered listing.
xmlResp := `
/dir/
HTTP/1.1 200 OK
/dir/a.pdf
100
HTTP/1.1 200 OK
/dir/b.txt
200
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
if r.Method == "GET" {
w.Write([]byte("file content"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
parsedServerURL, _ := url.Parse(server.URL)
p := NewProtocol()
t.Run("accept filter", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-filtered-*")
defer os.RemoveAll(tmpDir)
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
Timeout: 5 * time.Second,
AcceptPatterns: []string{"*.pdf"},
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("filtered download failed: %v", err)
}
// Only a.pdf should be downloaded (100 bytes).
if result.ChunksCount != 1 {
t.Errorf("ChunksCount = %d, want 1 (only *.pdf matched)", result.ChunksCount)
}
})
t.Run("reject filter", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-reject-*")
defer os.RemoveAll(tmpDir)
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
Timeout: 5 * time.Second,
RejectPatterns: []string{"*.txt"},
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("reject download failed: %v", err)
}
// Only a.pdf should be downloaded (b.txt rejected).
if result.ChunksCount != 1 {
t.Errorf("ChunksCount = %d, want 1 (*.txt rejected)", result.ChunksCount)
}
})
}
func TestAuthRequired(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, ok := r.BasicAuth()
if !ok || user != "admin" || pass != "secret" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.Method == "PROPFIND" {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
xmlResp := `
/file.txt
12
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
if r.Method == "GET" {
w.Write([]byte("file content"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
parsedServerURL, _ := url.Parse(server.URL)
p := NewProtocol()
t.Run("auth success", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-auth-*")
defer os.RemoveAll(tmpDir)
fileURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/file.txt",
User: url.UserPassword("admin", "secret"),
}
outputPath := filepath.Join(tmpDir, "file.txt")
req := &core.DownloadRequest{
URL: fileURL,
Output: outputPath,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("auth download failed: %v", err)
}
if result.BytesDownloaded != 12 {
t.Errorf("BytesDownloaded = %d, want 12", result.BytesDownloaded)
}
})
t.Run("auth failure", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-auth-fail-*")
defer os.RemoveAll(tmpDir)
fileURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/file.txt",
User: url.UserPassword("wrong", "creds"),
}
outputPath := filepath.Join(tmpDir, "file.txt")
req := &core.DownloadRequest{
URL: fileURL,
Output: outputPath,
Timeout: 5 * time.Second,
}
_, err := p.Download(context.Background(), req)
if err == nil {
t.Fatal("expected auth error, got nil")
}
if !strings.Contains(err.Error(), string(core.ErrAuth)) {
t.Errorf("expected AUTH error, got: %v", err)
}
})
}
// TestProtocolHTTPTransportIsShared verifies that repeated calls to
// getHTTPClient return clients that all wrap the same *http.Transport.
// Connection pooling depends on this: a fresh transport per call would
// disable idle-connection reuse, HTTP/2 multiplexing, and TLS session
// resumption.
func TestProtocolHTTPTransportIsShared(t *testing.T) {
p := NewProtocol()
// First call triggers the lazy init.
tr1 := p.getHTTPTransport()
if tr1 == nil {
t.Fatal("getHTTPTransport returned nil on first call")
}
// Subsequent calls must return the very same pointer.
for i := 0; i < 5; i++ {
tr := p.getHTTPTransport()
if tr != tr1 {
t.Errorf("getHTTPTransport call %d returned a fresh transport; want pointer reuse", i+2)
}
}
// And every client wrapper must point at that same transport.
for i := 0; i < 3; i++ {
client := p.getHTTPClient(15 * time.Second)
if client.Transport != tr1 {
t.Errorf("getHTTPClient call %d wrapped a different transport; want shared", i+1)
}
if client.Timeout != 15*time.Second {
t.Errorf("getHTTPClient call %d timeout = %v, want 15s", i+1, client.Timeout)
}
}
}
// TestGetHTTPClientTimeoutDefault verifies that a zero/negative timeout is
// upgraded to a 30-minute default, matching the previous on-site fallback
// in downloadFile and doUpload.
func TestGetHTTPClientTimeoutDefault(t *testing.T) {
p := NewProtocol()
tr := p.getHTTPTransport()
cases := []struct {
name string
timeout time.Duration
want time.Duration
}{
{"zero becomes default", 0, 30 * time.Minute},
{"negative becomes default", -1 * time.Second, 30 * time.Minute},
{"explicit positive is preserved", 5 * time.Second, 5 * time.Second},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
client := p.getHTTPClient(tc.timeout)
if client.Timeout != tc.want {
t.Errorf("Timeout = %v, want %v", client.Timeout, tc.want)
}
if client.Transport != tr {
t.Errorf("Transport is not the shared one")
}
})
}
}
// TestProtocolHTTPTransportConcurrentInit makes sure that concurrent first
// callers see exactly one transport built (no double-init under the mutex).
// Run with -race to catch any data race in the lazy-init path.
func TestProtocolHTTPTransportConcurrentInit(t *testing.T) {
p := NewProtocol()
const goroutines = 16
results := make([]*http.Transport, goroutines)
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
i := i
go func() {
defer wg.Done()
results[i] = p.getHTTPTransport()
}()
}
wg.Wait()
first := results[0]
for i, tr := range results {
if tr != first {
t.Errorf("goroutine %d saw a different transport; lazy init is not safe", i)
}
}
}
func TestRecursiveDownloadParallel(t *testing.T) {
if !protocol.GlobalRegistry.Supports("http") {
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
if err != nil {
t.Fatalf("failed to create http client: %v", err)
}
_ = protocol.Register(httpClient)
}
// Mock WebDAV server with 5 files in a single directory.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
xmlResp := `
/dir/
HTTP/1.1 200 OK
/dir/a.txt
10
HTTP/1.1 200 OK
/dir/b.txt
20
HTTP/1.1 200 OK
/dir/c.txt
30
HTTP/1.1 200 OK
/dir/d.txt
40
HTTP/1.1 200 OK
/dir/e.txt
50
HTTP/1.1 200 OK
`
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(xmlResp))
return
}
if r.Method == "GET" {
w.Write([]byte("file content"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
parsedServerURL, _ := url.Parse(server.URL)
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
t.Run("parallel=3 downloads all files", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-parallel-*")
defer os.RemoveAll(tmpDir)
p := NewProtocol()
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 3,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("parallel download failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5 (all files)", result.ChunksCount)
}
// 10+20+30+40+50 = 150 bytes of metadata-reported sizes, but our
// mock server returns the same 12-byte body for every file, so
// the actual bytes downloaded is 5 * 12 = 60.
const expectedBytes = int64(5 * len("file content"))
if result.BytesDownloaded != expectedBytes {
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, expectedBytes)
}
// Verify all files actually exist on disk.
for _, name := range []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt"} {
path := filepath.Join(tmpDir, name)
data, err := os.ReadFile(path)
if err != nil {
t.Errorf("file %s not downloaded: %v", name, err)
continue
}
if string(data) != "file content" {
t.Errorf("file %s has wrong content: %q", name, string(data))
}
}
})
t.Run("parallel=1 falls back to sequential path", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-seq1-*")
defer os.RemoveAll(tmpDir)
p := NewProtocol()
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 1,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("download with parallel=1 failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
}
})
t.Run("parallel=0 falls back to sequential path", func(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "webdav-seq0-*")
defer os.RemoveAll(tmpDir)
p := NewProtocol()
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 0,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("download with parallel=0 failed: %v", err)
}
if result.ChunksCount != 5 {
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
}
})
t.Run("parallel=3 propagates to subdirectory recursion", func(t *testing.T) {
// Mock server with one subdirectory containing 2 files.
subServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" && (r.URL.Path == "/root" || r.URL.Path == "/root/") {
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/root/
HTTP/1.1 200 OK
/root/sub/
HTTP/1.1 200 OK
`))
return
}
if r.Method == "PROPFIND" && (r.URL.Path == "/root/sub" || r.URL.Path == "/root/sub/") {
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/root/sub/
HTTP/1.1 200 OK
/root/sub/x.txt
5
HTTP/1.1 200 OK
/root/sub/y.txt
5
HTTP/1.1 200 OK
`))
return
}
if r.Method == "GET" {
w.Write([]byte("hello"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer subServer.Close()
parsedSubURL, _ := url.Parse(subServer.URL)
rootURL := &url.URL{
Scheme: "webdav",
Host: parsedSubURL.Host,
Path: "/root/",
}
tmpDir, _ := os.MkdirTemp("", "webdav-subdir-*")
defer os.RemoveAll(tmpDir)
p := NewProtocol()
req := &core.DownloadRequest{
URL: rootURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 3,
MaxDepth: 1,
Timeout: 5 * time.Second,
}
result, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("recursive subdir download failed: %v", err)
}
if result.ChunksCount != 2 {
t.Errorf("ChunksCount = %d, want 2 (files in subdirectory)", result.ChunksCount)
}
// Verify the subdirectory and its files were created.
for _, name := range []string{"sub/x.txt", "sub/y.txt"} {
path := filepath.Join(tmpDir, name)
if _, err := os.Stat(path); err != nil {
t.Errorf("expected file %s to exist: %v", name, err)
}
}
})
t.Run("context cancellation stops parallel workers", func(t *testing.T) {
// Mock server that blocks GETs until ctx is cancelled, so we can
// observe the parallel pool responding to cancellation.
block := make(chan struct{})
cancelServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" && (r.URL.Path == "/slow" || r.URL.Path == "/slow/") {
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/slow/
HTTP/1.1 200 OK
/slow/a.txt
10
HTTP/1.1 200 OK
/slow/b.txt
10
HTTP/1.1 200 OK
/slow/c.txt
10
HTTP/1.1 200 OK
`))
return
}
if r.Method == "GET" {
// Block until the test closes `block` (i.e. the request
// will only return after the test cancels the context).
select {
case <-block:
w.Write([]byte("data"))
case <-r.Context().Done():
return
}
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer cancelServer.Close()
defer close(block)
parsedCancelURL, _ := url.Parse(cancelServer.URL)
slowURL := &url.URL{
Scheme: "webdav",
Host: parsedCancelURL.Host,
Path: "/slow/",
}
tmpDir, _ := os.MkdirTemp("", "webdav-cancel-*")
defer os.RemoveAll(tmpDir)
ctx, cancel := context.WithCancel(context.Background())
p := NewProtocol()
req := &core.DownloadRequest{
URL: slowURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 2,
Timeout: 10 * time.Second,
Ctx: ctx,
}
// Cancel after a short delay; the workers should bail out
// instead of waiting for the blocked GETs.
go func() {
time.Sleep(150 * time.Millisecond)
cancel()
}()
done := make(chan struct{})
go func() {
_, _ = p.Download(ctx, req)
close(done)
}()
select {
case <-done:
// Good — the download returned (likely with errors for the
// cancelled files, but it did return promptly).
case <-time.After(3 * time.Second):
t.Fatal("parallel download did not honour context cancellation within 3s")
}
})
}
// captureStderr redirects os.Stderr for the duration of fn and returns
// whatever was written. Used by the dry-run test to assert on the
// human-readable list of entries that would be fetched.
func captureStderr(t *testing.T, fn func()) string {
t.Helper()
orig := os.Stderr
r, w, err := os.Pipe()
if err != nil {
t.Fatalf("pipe: %v", err)
}
os.Stderr = w
defer func() { os.Stderr = orig }()
done := make(chan struct{})
var buf bytes.Buffer
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
fn()
_ = w.Close()
<-done
_ = r.Close()
return buf.String()
}
func TestRecursiveDownloadDryRun(t *testing.T) {
if !protocol.GlobalRegistry.Supports("http") {
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
if err != nil {
t.Fatalf("failed to create http client: %v", err)
}
_ = protocol.Register(httpClient)
}
// Mock server with the same layout the other recursive tests use.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/dir/
HTTP/1.1 200 OK
/dir/a.txt
100
HTTP/1.1 200 OK
/dir/b.txt
200
HTTP/1.1 200 OK
/dir/c.txt
300
HTTP/1.1 200 OK
/dir/sub/
HTTP/1.1 200 OK
`))
return
}
if r.Method == "PROPFIND" && (r.URL.Path == "/dir/sub" || r.URL.Path == "/dir/sub/") {
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/dir/sub/
HTTP/1.1 200 OK
/dir/sub/x.txt
42
HTTP/1.1 200 OK
`))
return
}
// GETs should not be invoked during dry-run; the test asserts
// this by checking the output dir is empty afterwards.
if r.Method == "GET" {
t.Errorf("unexpected GET during dry-run: %s", r.URL.Path)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
parsedServerURL, _ := url.Parse(server.URL)
dirURL := &url.URL{
Scheme: "webdav",
Host: parsedServerURL.Host,
Path: "/dir/",
}
// runDryRun wraps a dry-run invocation and returns both the result
// and the captured stderr so the assertions stay readable.
runDryRun := func(t *testing.T, parallel int) (*core.DownloadResult, string) {
t.Helper()
tmpDir := t.TempDir()
p := NewProtocol()
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: parallel,
DryRun: true,
MaxDepth: 5,
Timeout: 5 * time.Second,
}
var result *core.DownloadResult
stderr := captureStderr(t, func() {
var err error
result, err = p.Download(context.Background(), req)
if err != nil {
t.Fatalf("dry-run failed: %v", err)
}
})
// Verify that no files or local directories were created.
entries, err := os.ReadDir(tmpDir)
if err != nil {
t.Fatalf("read tmpDir: %v", err)
}
if len(entries) != 0 {
var names []string
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("dry-run wrote files to disk: %v", names)
}
return result, stderr
}
t.Run("sequential dry-run lists all matched entries", func(t *testing.T) {
result, stderr := runDryRun(t, 0)
if result.ChunksCount != 4 {
t.Errorf("ChunksCount = %d, want 4 (3 top-level files + 1 subdir file)", result.ChunksCount)
}
// 100 + 200 + 300 + 42 = 642 bytes (sum of getcontentlength).
const wantBytes = int64(100 + 200 + 300 + 42)
if result.BytesDownloaded != wantBytes {
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
}
// Stderr should mention every matched file and the subdir.
for _, want := range []string{
"would download: webdav://" + parsedServerURL.Host + "/dir/a.txt (100 bytes)",
"would download: webdav://" + parsedServerURL.Host + "/dir/b.txt (200 bytes)",
"would download: webdav://" + parsedServerURL.Host + "/dir/c.txt (300 bytes)",
"would recurse into: webdav://" + parsedServerURL.Host + "/dir/sub/",
"would download: webdav://" + parsedServerURL.Host + "/dir/sub/x.txt (42 bytes)",
} {
if !strings.Contains(stderr, want) {
t.Errorf("stderr missing %q\nfull output:\n%s", want, stderr)
}
}
})
t.Run("parallel dry-run lists all matched entries", func(t *testing.T) {
result, stderr := runDryRun(t, 4)
if result.ChunksCount != 4 {
t.Errorf("ChunksCount = %d, want 4", result.ChunksCount)
}
const wantBytes = int64(100 + 200 + 300 + 42)
if result.BytesDownloaded != wantBytes {
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
}
// Order is non-deterministic with a worker pool, so just check
// the set of expected lines is present.
for _, want := range []string{
"/dir/a.txt (100 bytes)",
"/dir/b.txt (200 bytes)",
"/dir/c.txt (300 bytes)",
"/dir/sub/",
"/dir/sub/x.txt (42 bytes)",
} {
if !strings.Contains(stderr, want) {
t.Errorf("stderr missing %q\nfull output:\n%s", want, stderr)
}
}
})
t.Run("dry-run with reject filter only lists matched files", func(t *testing.T) {
tmpDir := t.TempDir()
p := NewProtocol()
req := &core.DownloadRequest{
URL: dirURL,
Output: tmpDir,
Recursive: true,
RecursiveParallel: 0,
DryRun: true,
MaxDepth: 5,
Timeout: 5 * time.Second,
RejectPatterns: []string{"b.txt", "c.txt"},
}
stderr := captureStderr(t, func() {
_, err := p.Download(context.Background(), req)
if err != nil {
t.Fatalf("dry-run with reject failed: %v", err)
}
})
if !strings.Contains(stderr, "/dir/a.txt (100 bytes)") {
t.Errorf("expected a.txt in dry-run list:\n%s", stderr)
}
if strings.Contains(stderr, "/dir/b.txt") {
t.Errorf("b.txt should have been rejected but appeared:\n%s", stderr)
}
if strings.Contains(stderr, "/dir/c.txt") {
t.Errorf("c.txt should have been rejected but appeared:\n%s", stderr)
}
// Filter applies per-directory: sub/ is unaffected so sub/x.txt
// is still listed (it doesn't match b.txt or c.txt).
if !strings.Contains(stderr, "/dir/sub/x.txt (42 bytes)") {
t.Errorf("expected sub/x.txt in dry-run list (filter doesn't apply across dirs):\n%s", stderr)
}
})
}
// TestWebDAVSingleGracefulShutdown covers the graceful-shutdown contract
// for single-file WebDAV downloads: when the context is cancelled
// mid-transfer, the worker must persist a `.goget.meta` sidecar with
// the partial byte count so `--resume` can pick up where the user left
// off. On a clean completion the sidecar is removed.
func TestWebDAVSingleGracefulShutdown(t *testing.T) {
if !protocol.GlobalRegistry.Supports("http") {
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
if err != nil {
t.Fatalf("failed to create http client: %v", err)
}
_ = protocol.Register(httpClient)
}
// Mock server: responds to GET with a stream that blocks until the
// request context is cancelled. This lets the test cancel from the
// outside and observe how much was downloaded before the cancel.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "PROPFIND" {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/file.bin
1048576
HTTP/1.1 200 OK
`))
return
}
if r.Method == "GET" {
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("ETag", `"etag-graceful"`)
w.Header().Set("Last-Modified", "Mon, 01 Jan 2026 00:00:00 GMT")
w.WriteHeader(http.StatusOK)
flusher, _ := w.(http.Flusher)
// Write a few KB synchronously so the client has something
// to write to disk, then block until the request is
// cancelled. The body is large enough that the client
// will not read to EOF naturally.
chunk := bytes.Repeat([]byte("a"), 4096)
for i := 0; i < 4; i++ {
if _, err := w.Write(chunk); err != nil {
return
}
if flusher != nil {
flusher.Flush()
}
}
// Drain or block until the client cancels.
<-r.Context().Done()
}
}))
defer server.Close()
parsedURL, _ := url.Parse(server.URL)
fileURL := &url.URL{
Scheme: "webdav",
Host: parsedURL.Host,
Path: "/file.bin",
}
t.Run("ctx cancel saves resume metadata", func(t *testing.T) {
outDir := t.TempDir()
outPath := filepath.Join(outDir, "file.bin")
ctx, cancel := context.WithCancel(context.Background())
p := NewProtocol()
req := &core.DownloadRequest{
URL: fileURL,
Output: outPath,
Resume: true,
Timeout: 5 * time.Second,
Ctx: ctx,
Recursive: false,
}
done := make(chan struct{})
go func() {
_, _ = p.Download(ctx, req)
close(done)
}()
// Give the worker a moment to start writing chunks, then
// cancel. The mock holds the connection open until cancelled.
time.Sleep(150 * time.Millisecond)
cancel()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("download did not honour context cancellation within 3s")
}
// The mock wrote 4 × 4096 = 16384 bytes before blocking, so
// the resume metadata should record roughly that many bytes
// (give or take the transport buffer). The point of the test
// is the sidecar exists and is non-empty; exact byte counts
// are racy.
meta, err := output.LoadResumeMetadata(outPath)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata after cancellation, got nil")
}
if meta.URL != req.URL.String() {
t.Errorf("metadata URL = %q, want %q", meta.URL, req.URL.String())
}
if meta.ETag != `"etag-graceful"` {
t.Errorf("metadata ETag = %q, want %q", meta.ETag, `"etag-graceful"`)
}
if meta.Downloaded <= 0 {
t.Errorf("metadata Downloaded = %d, want > 0", meta.Downloaded)
}
})
t.Run("successful download removes resume metadata", func(t *testing.T) {
// Pre-create a stale resume sidecar to verify the success
// path cleans it up. The mock for this subtest returns the
// full body immediately.
outDir := t.TempDir()
outPath := filepath.Join(outDir, "file.bin")
stale := output.NewResumeMetadata("stale://url", "stale-etag", "state-mod", 999, 999)
if err := stale.Save(outPath); err != nil {
t.Fatalf("pre-save: %v", err)
}
if _, err := output.LoadResumeMetadata(outPath); err != nil {
t.Fatalf("LoadResumeMetadata after pre-save: %v", err)
}
// Spin up a fresh mock that returns the whole file in one shot.
fullServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "PROPFIND" {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
w.WriteHeader(http.StatusMultiStatus)
w.Write([]byte(`
/file.bin
16
HTTP/1.1 200 OK
`))
return
}
if r.Method == "GET" {
w.Header().Set("Content-Type", "application/octet-stream")
w.WriteHeader(http.StatusOK)
w.Write([]byte("0123456789abcdef"))
}
}))
defer fullServer.Close()
fullParsed, _ := url.Parse(fullServer.URL)
fullURL := &url.URL{
Scheme: "webdav",
Host: fullParsed.Host,
Path: "/file.bin",
}
p := NewProtocol()
req := &core.DownloadRequest{
URL: fullURL,
Output: outPath,
Resume: true,
Timeout: 5 * time.Second,
Recursive: false,
}
if _, err := p.Download(context.Background(), req); err != nil {
t.Fatalf("download failed: %v", err)
}
// The pre-seeded stale sidecar should be gone after a
// successful download.
meta, err := output.LoadResumeMetadata(outPath)
if err != nil {
t.Fatalf("LoadResumeMetadata after success: %v", err)
}
if meta != nil {
t.Errorf("expected resume metadata to be cleared, got %+v", meta)
}
})
}