1551 lines
42 KiB
Go
1551 lines
42 KiB
Go
//go:build linux || freebsd
|
||
// +build linux freebsd
|
||
|
||
package webdav
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"net/url"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"codeberg.org/petrbalvin/goget/internal/core"
|
||
"codeberg.org/petrbalvin/goget/internal/output"
|
||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||
httpproto "codeberg.org/petrbalvin/goget/internal/protocol/http"
|
||
)
|
||
|
||
func TestWebDAVProtocol(t *testing.T) {
|
||
if !protocol.GlobalRegistry.Supports("http") {
|
||
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
|
||
if err != nil {
|
||
t.Fatalf("failed to create http client: %v", err)
|
||
}
|
||
_ = protocol.Register(httpClient)
|
||
}
|
||
|
||
// Create a mock WebDAV server
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" {
|
||
depth := r.Header.Get("Depth")
|
||
path := r.URL.Path
|
||
|
||
if depth == "0" {
|
||
if path == "/dir" || path == "/dir/" {
|
||
// Directory response
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
|
||
if path == "/file.txt" {
|
||
// File response
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/file.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>12</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
}
|
||
|
||
if depth == "1" {
|
||
if path == "/dir" || path == "/dir/" {
|
||
// Directory contents response
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/file1.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>13</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/file2.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>13</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
if r.URL.Path == "/file.txt" {
|
||
w.Header().Set("Content-Type", "text/plain")
|
||
w.Write([]byte("file content"))
|
||
return
|
||
}
|
||
if r.URL.Path == "/dir/file1.txt" {
|
||
w.Header().Set("Content-Type", "text/plain")
|
||
w.Write([]byte("file1 content"))
|
||
return
|
||
}
|
||
if r.URL.Path == "/dir/file2.txt" {
|
||
w.Header().Set("Content-Type", "text/plain")
|
||
w.Write([]byte("file2 content"))
|
||
return
|
||
}
|
||
}
|
||
|
||
if r.Method == "PUT" {
|
||
if r.URL.Path == "/upload.txt" {
|
||
w.WriteHeader(http.StatusCreated)
|
||
return
|
||
}
|
||
}
|
||
|
||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||
}))
|
||
defer server.Close()
|
||
|
||
// Parse server URL to construct test URLs
|
||
parsedServerURL, err := url.Parse(server.URL)
|
||
if err != nil {
|
||
t.Fatalf("failed to parse mock server URL: %v", err)
|
||
}
|
||
|
||
p := NewProtocol()
|
||
|
||
// 1. Test CanHandle
|
||
t.Run("CanHandle", func(t *testing.T) {
|
||
u1, _ := url.Parse("webdav://example.com/file")
|
||
u2, _ := url.Parse("webdavs://example.com/file")
|
||
u3, _ := url.Parse("http://example.com/file")
|
||
|
||
if !p.CanHandle(u1) {
|
||
t.Error("CanHandle(webdav://) = false, want true")
|
||
}
|
||
if !p.CanHandle(u2) {
|
||
t.Error("CanHandle(webdavs://) = false, want true")
|
||
}
|
||
if p.CanHandle(u3) {
|
||
t.Error("CanHandle(http://) = true, want false")
|
||
}
|
||
})
|
||
|
||
// 2. Test File Download
|
||
t.Run("Download File", func(t *testing.T) {
|
||
tmpDir, err := os.MkdirTemp("", "webdav-test-*")
|
||
if err != nil {
|
||
t.Fatalf("failed to create temp dir: %v", err)
|
||
}
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
fileURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/file.txt",
|
||
}
|
||
outputPath := filepath.Join(tmpDir, "downloaded_file.txt")
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: fileURL,
|
||
Output: outputPath,
|
||
Recursive: false,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("download failed: %v", err)
|
||
}
|
||
|
||
if result.BytesDownloaded != 12 {
|
||
t.Errorf("BytesDownloaded = %d, want 12", result.BytesDownloaded)
|
||
}
|
||
|
||
data, err := os.ReadFile(outputPath)
|
||
if err != nil {
|
||
t.Fatalf("failed to read output file: %v", err)
|
||
}
|
||
|
||
if string(data) != "file content" {
|
||
t.Errorf("file content = %q, want %q", string(data), "file content")
|
||
}
|
||
})
|
||
|
||
// 3. Test Directory Detection and error on non-recursive download of directory
|
||
t.Run("Directory Non-Recursive Error", func(t *testing.T) {
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: "",
|
||
Recursive: false,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
_, err := p.Download(context.Background(), req)
|
||
if err == nil {
|
||
t.Fatal("expected error downloading directory non-recursively, got nil")
|
||
}
|
||
|
||
if !strings.Contains(err.Error(), "remote path is a directory") {
|
||
t.Errorf("unexpected error: %v", err)
|
||
}
|
||
})
|
||
|
||
// 4. Test Recursive Download
|
||
t.Run("Download Directory Recursive", func(t *testing.T) {
|
||
tmpDir, err := os.MkdirTemp("", "webdav-test-*")
|
||
if err != nil {
|
||
t.Fatalf("failed to create temp dir: %v", err)
|
||
}
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
outputPath := filepath.Join(tmpDir, "local_dir")
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: outputPath,
|
||
Recursive: true,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("recursive download failed: %v", err)
|
||
}
|
||
|
||
// Two files: file1.txt (13 bytes) + file2.txt (13 bytes) = 26 bytes
|
||
if result.BytesDownloaded != 26 {
|
||
t.Errorf("BytesDownloaded = %d, want 26", result.BytesDownloaded)
|
||
}
|
||
|
||
if result.ChunksCount != 2 {
|
||
t.Errorf("ChunksCount = %d, want 2", result.ChunksCount)
|
||
}
|
||
|
||
// Verify files exist in directory
|
||
f1, err := os.ReadFile(filepath.Join(outputPath, "file1.txt"))
|
||
if err != nil {
|
||
t.Errorf("file1.txt read failed: %v", err)
|
||
} else if string(f1) != "file1 content" {
|
||
t.Errorf("file1 content = %q, want %q", string(f1), "file1 content")
|
||
}
|
||
|
||
f2, err := os.ReadFile(filepath.Join(outputPath, "file2.txt"))
|
||
if err != nil {
|
||
t.Errorf("file2.txt read failed: %v", err)
|
||
} else if string(f2) != "file2 content" {
|
||
t.Errorf("file2 content = %q, want %q", string(f2), "file2 content")
|
||
}
|
||
})
|
||
|
||
// 5. Test Upload
|
||
t.Run("Upload File", func(t *testing.T) {
|
||
tmpFile, err := os.CreateTemp("", "webdav-upload-*")
|
||
if err != nil {
|
||
t.Fatalf("failed to create temp file for upload: %v", err)
|
||
}
|
||
defer os.Remove(tmpFile.Name())
|
||
|
||
content := "upload test content"
|
||
if _, err := tmpFile.Write([]byte(content)); err != nil {
|
||
t.Fatalf("failed to write upload file content: %v", err)
|
||
}
|
||
tmpFile.Close()
|
||
|
||
uploadURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/upload.txt",
|
||
}
|
||
|
||
req := &core.UploadRequest{
|
||
URL: uploadURL,
|
||
Input: tmpFile.Name(),
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Upload(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("upload failed: %v", err)
|
||
}
|
||
|
||
if result.BytesUploaded != int64(len(content)) {
|
||
t.Errorf("BytesUploaded = %d, want %d", result.BytesUploaded, len(content))
|
||
}
|
||
|
||
if result.Protocol != "webdav" {
|
||
t.Errorf("Protocol = %q, want %q", result.Protocol, "webdav")
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestToHTTPURL(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
input string
|
||
want string
|
||
}{
|
||
{"webdav to http", "webdav://example.com/file", "http://example.com/file"},
|
||
{"webdavs to https", "webdavs://example.com/file", "https://example.com/file"},
|
||
{"already http", "http://example.com/file", "http://example.com/file"},
|
||
{"already https", "https://example.com/file", "https://example.com/file"},
|
||
{"webdav with port", "webdav://example.com:8080/file", "http://example.com:8080/file"},
|
||
{"webdavs with port", "webdavs://example.com:443/file", "https://example.com:443/file"},
|
||
{"webdav with user", "webdav://user:pass@example.com/file", "http://user:pass@example.com/file"},
|
||
{"empty path", "webdav://example.com", "http://example.com"},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
u, err := url.Parse(tt.input)
|
||
if err != nil {
|
||
t.Fatalf("failed to parse input url: %v", err)
|
||
}
|
||
result := toHTTPURL(u)
|
||
if result.String() != tt.want {
|
||
t.Errorf("toHTTPURL(%q) = %q, want %q", tt.input, result.String(), tt.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestFilteredRecursiveDownload(t *testing.T) {
|
||
// Register HTTP protocol handler.
|
||
if !protocol.GlobalRegistry.Supports("http") {
|
||
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
|
||
if err != nil {
|
||
t.Fatalf("failed to create http client: %v", err)
|
||
}
|
||
_ = protocol.Register(httpClient)
|
||
}
|
||
|
||
// Create mock WebDAV server with multiple entries.
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
|
||
// Return unfiltered listing.
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/a.pdf</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>100</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/b.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>200</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
w.Write([]byte("file content"))
|
||
return
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer server.Close()
|
||
|
||
parsedServerURL, _ := url.Parse(server.URL)
|
||
p := NewProtocol()
|
||
|
||
t.Run("accept filter", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-filtered-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
Timeout: 5 * time.Second,
|
||
AcceptPatterns: []string{"*.pdf"},
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("filtered download failed: %v", err)
|
||
}
|
||
|
||
// Only a.pdf should be downloaded (100 bytes).
|
||
if result.ChunksCount != 1 {
|
||
t.Errorf("ChunksCount = %d, want 1 (only *.pdf matched)", result.ChunksCount)
|
||
}
|
||
})
|
||
|
||
t.Run("reject filter", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-reject-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
Timeout: 5 * time.Second,
|
||
RejectPatterns: []string{"*.txt"},
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("reject download failed: %v", err)
|
||
}
|
||
|
||
// Only a.pdf should be downloaded (b.txt rejected).
|
||
if result.ChunksCount != 1 {
|
||
t.Errorf("ChunksCount = %d, want 1 (*.txt rejected)", result.ChunksCount)
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestAuthRequired(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
user, pass, ok := r.BasicAuth()
|
||
if !ok || user != "admin" || pass != "secret" {
|
||
w.WriteHeader(http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
if r.Method == "PROPFIND" {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/file.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>12</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
w.Write([]byte("file content"))
|
||
return
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer server.Close()
|
||
|
||
parsedServerURL, _ := url.Parse(server.URL)
|
||
p := NewProtocol()
|
||
|
||
t.Run("auth success", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-auth-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
fileURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/file.txt",
|
||
User: url.UserPassword("admin", "secret"),
|
||
}
|
||
outputPath := filepath.Join(tmpDir, "file.txt")
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: fileURL,
|
||
Output: outputPath,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("auth download failed: %v", err)
|
||
}
|
||
|
||
if result.BytesDownloaded != 12 {
|
||
t.Errorf("BytesDownloaded = %d, want 12", result.BytesDownloaded)
|
||
}
|
||
})
|
||
|
||
t.Run("auth failure", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-auth-fail-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
fileURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/file.txt",
|
||
User: url.UserPassword("wrong", "creds"),
|
||
}
|
||
outputPath := filepath.Join(tmpDir, "file.txt")
|
||
|
||
req := &core.DownloadRequest{
|
||
URL: fileURL,
|
||
Output: outputPath,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
_, err := p.Download(context.Background(), req)
|
||
if err == nil {
|
||
t.Fatal("expected auth error, got nil")
|
||
}
|
||
if !strings.Contains(err.Error(), string(core.ErrAuth)) {
|
||
t.Errorf("expected AUTH error, got: %v", err)
|
||
}
|
||
})
|
||
}
|
||
|
||
// TestProtocolHTTPTransportIsShared verifies that repeated calls to
|
||
// getHTTPClient return clients that all wrap the same *http.Transport.
|
||
// Connection pooling depends on this: a fresh transport per call would
|
||
// disable idle-connection reuse, HTTP/2 multiplexing, and TLS session
|
||
// resumption.
|
||
func TestProtocolHTTPTransportIsShared(t *testing.T) {
|
||
p := NewProtocol()
|
||
|
||
// First call triggers the lazy init.
|
||
tr1 := p.getHTTPTransport()
|
||
if tr1 == nil {
|
||
t.Fatal("getHTTPTransport returned nil on first call")
|
||
}
|
||
|
||
// Subsequent calls must return the very same pointer.
|
||
for i := 0; i < 5; i++ {
|
||
tr := p.getHTTPTransport()
|
||
if tr != tr1 {
|
||
t.Errorf("getHTTPTransport call %d returned a fresh transport; want pointer reuse", i+2)
|
||
}
|
||
}
|
||
|
||
// And every client wrapper must point at that same transport.
|
||
for i := 0; i < 3; i++ {
|
||
client := p.getHTTPClient(15 * time.Second)
|
||
if client.Transport != tr1 {
|
||
t.Errorf("getHTTPClient call %d wrapped a different transport; want shared", i+1)
|
||
}
|
||
if client.Timeout != 15*time.Second {
|
||
t.Errorf("getHTTPClient call %d timeout = %v, want 15s", i+1, client.Timeout)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestGetHTTPClientTimeoutDefault verifies that a zero/negative timeout is
|
||
// upgraded to a 30-minute default, matching the previous on-site fallback
|
||
// in downloadFile and doUpload.
|
||
func TestGetHTTPClientTimeoutDefault(t *testing.T) {
|
||
p := NewProtocol()
|
||
tr := p.getHTTPTransport()
|
||
|
||
cases := []struct {
|
||
name string
|
||
timeout time.Duration
|
||
want time.Duration
|
||
}{
|
||
{"zero becomes default", 0, 30 * time.Minute},
|
||
{"negative becomes default", -1 * time.Second, 30 * time.Minute},
|
||
{"explicit positive is preserved", 5 * time.Second, 5 * time.Second},
|
||
}
|
||
for _, tc := range cases {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
client := p.getHTTPClient(tc.timeout)
|
||
if client.Timeout != tc.want {
|
||
t.Errorf("Timeout = %v, want %v", client.Timeout, tc.want)
|
||
}
|
||
if client.Transport != tr {
|
||
t.Errorf("Transport is not the shared one")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestProtocolHTTPTransportConcurrentInit makes sure that concurrent first
|
||
// callers see exactly one transport built (no double-init under the mutex).
|
||
// Run with -race to catch any data race in the lazy-init path.
|
||
func TestProtocolHTTPTransportConcurrentInit(t *testing.T) {
|
||
p := NewProtocol()
|
||
|
||
const goroutines = 16
|
||
results := make([]*http.Transport, goroutines)
|
||
var wg sync.WaitGroup
|
||
wg.Add(goroutines)
|
||
for i := 0; i < goroutines; i++ {
|
||
i := i
|
||
go func() {
|
||
defer wg.Done()
|
||
results[i] = p.getHTTPTransport()
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
|
||
first := results[0]
|
||
for i, tr := range results {
|
||
if tr != first {
|
||
t.Errorf("goroutine %d saw a different transport; lazy init is not safe", i)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestRecursiveDownloadParallel(t *testing.T) {
|
||
if !protocol.GlobalRegistry.Supports("http") {
|
||
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
|
||
if err != nil {
|
||
t.Fatalf("failed to create http client: %v", err)
|
||
}
|
||
_ = protocol.Register(httpClient)
|
||
}
|
||
|
||
// Mock WebDAV server with 5 files in a single directory.
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
|
||
xmlResp := `<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/a.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>10</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/b.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>20</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/c.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>30</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/d.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>40</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/e.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>50</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(xmlResp))
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
w.Write([]byte("file content"))
|
||
return
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer server.Close()
|
||
|
||
parsedServerURL, _ := url.Parse(server.URL)
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
|
||
t.Run("parallel=3 downloads all files", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-parallel-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 3,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("parallel download failed: %v", err)
|
||
}
|
||
|
||
if result.ChunksCount != 5 {
|
||
t.Errorf("ChunksCount = %d, want 5 (all files)", result.ChunksCount)
|
||
}
|
||
// 10+20+30+40+50 = 150 bytes of metadata-reported sizes, but our
|
||
// mock server returns the same 12-byte body for every file, so
|
||
// the actual bytes downloaded is 5 * 12 = 60.
|
||
const expectedBytes = int64(5 * len("file content"))
|
||
if result.BytesDownloaded != expectedBytes {
|
||
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, expectedBytes)
|
||
}
|
||
|
||
// Verify all files actually exist on disk.
|
||
for _, name := range []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt"} {
|
||
path := filepath.Join(tmpDir, name)
|
||
data, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Errorf("file %s not downloaded: %v", name, err)
|
||
continue
|
||
}
|
||
if string(data) != "file content" {
|
||
t.Errorf("file %s has wrong content: %q", name, string(data))
|
||
}
|
||
}
|
||
})
|
||
|
||
t.Run("parallel=1 falls back to sequential path", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-seq1-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 1,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("download with parallel=1 failed: %v", err)
|
||
}
|
||
if result.ChunksCount != 5 {
|
||
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
|
||
}
|
||
})
|
||
|
||
t.Run("parallel=0 falls back to sequential path", func(t *testing.T) {
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-seq0-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 0,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("download with parallel=0 failed: %v", err)
|
||
}
|
||
if result.ChunksCount != 5 {
|
||
t.Errorf("ChunksCount = %d, want 5", result.ChunksCount)
|
||
}
|
||
})
|
||
|
||
t.Run("parallel=3 propagates to subdirectory recursion", func(t *testing.T) {
|
||
// Mock server with one subdirectory containing 2 files.
|
||
subServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/root" || r.URL.Path == "/root/") {
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/root/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/root/sub/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/root/sub" || r.URL.Path == "/root/sub/") {
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/root/sub/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/root/sub/x.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>5</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/root/sub/y.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>5</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
w.Write([]byte("hello"))
|
||
return
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer subServer.Close()
|
||
|
||
parsedSubURL, _ := url.Parse(subServer.URL)
|
||
rootURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedSubURL.Host,
|
||
Path: "/root/",
|
||
}
|
||
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-subdir-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: rootURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 3,
|
||
MaxDepth: 1,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
|
||
result, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("recursive subdir download failed: %v", err)
|
||
}
|
||
if result.ChunksCount != 2 {
|
||
t.Errorf("ChunksCount = %d, want 2 (files in subdirectory)", result.ChunksCount)
|
||
}
|
||
|
||
// Verify the subdirectory and its files were created.
|
||
for _, name := range []string{"sub/x.txt", "sub/y.txt"} {
|
||
path := filepath.Join(tmpDir, name)
|
||
if _, err := os.Stat(path); err != nil {
|
||
t.Errorf("expected file %s to exist: %v", name, err)
|
||
}
|
||
}
|
||
})
|
||
|
||
t.Run("context cancellation stops parallel workers", func(t *testing.T) {
|
||
// Mock server that blocks GETs until ctx is cancelled, so we can
|
||
// observe the parallel pool responding to cancellation.
|
||
block := make(chan struct{})
|
||
cancelServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/slow" || r.URL.Path == "/slow/") {
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/slow/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/slow/a.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>10</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/slow/b.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>10</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/slow/c.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>10</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
|
||
if r.Method == "GET" {
|
||
// Block until the test closes `block` (i.e. the request
|
||
// will only return after the test cancels the context).
|
||
select {
|
||
case <-block:
|
||
w.Write([]byte("data"))
|
||
case <-r.Context().Done():
|
||
return
|
||
}
|
||
return
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer cancelServer.Close()
|
||
defer close(block)
|
||
|
||
parsedCancelURL, _ := url.Parse(cancelServer.URL)
|
||
slowURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedCancelURL.Host,
|
||
Path: "/slow/",
|
||
}
|
||
|
||
tmpDir, _ := os.MkdirTemp("", "webdav-cancel-*")
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: slowURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 2,
|
||
Timeout: 10 * time.Second,
|
||
Ctx: ctx,
|
||
}
|
||
|
||
// Cancel after a short delay; the workers should bail out
|
||
// instead of waiting for the blocked GETs.
|
||
go func() {
|
||
time.Sleep(150 * time.Millisecond)
|
||
cancel()
|
||
}()
|
||
|
||
done := make(chan struct{})
|
||
go func() {
|
||
_, _ = p.Download(ctx, req)
|
||
close(done)
|
||
}()
|
||
|
||
select {
|
||
case <-done:
|
||
// Good — the download returned (likely with errors for the
|
||
// cancelled files, but it did return promptly).
|
||
case <-time.After(3 * time.Second):
|
||
t.Fatal("parallel download did not honour context cancellation within 3s")
|
||
}
|
||
})
|
||
}
|
||
|
||
// captureStderr redirects os.Stderr for the duration of fn and returns
|
||
// whatever was written. Used by the dry-run test to assert on the
|
||
// human-readable list of entries that would be fetched.
|
||
func captureStderr(t *testing.T, fn func()) string {
|
||
t.Helper()
|
||
orig := os.Stderr
|
||
r, w, err := os.Pipe()
|
||
if err != nil {
|
||
t.Fatalf("pipe: %v", err)
|
||
}
|
||
os.Stderr = w
|
||
defer func() { os.Stderr = orig }()
|
||
|
||
done := make(chan struct{})
|
||
var buf bytes.Buffer
|
||
go func() {
|
||
_, _ = io.Copy(&buf, r)
|
||
close(done)
|
||
}()
|
||
fn()
|
||
_ = w.Close()
|
||
<-done
|
||
_ = r.Close()
|
||
return buf.String()
|
||
}
|
||
|
||
func TestRecursiveDownloadDryRun(t *testing.T) {
|
||
if !protocol.GlobalRegistry.Supports("http") {
|
||
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
|
||
if err != nil {
|
||
t.Fatalf("failed to create http client: %v", err)
|
||
}
|
||
_ = protocol.Register(httpClient)
|
||
}
|
||
|
||
// Mock server with the same layout the other recursive tests use.
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/dir" || r.URL.Path == "/dir/") {
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/a.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>100</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/b.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>200</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/c.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>300</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/sub/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
if r.Method == "PROPFIND" && (r.URL.Path == "/dir/sub" || r.URL.Path == "/dir/sub/") {
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/dir/sub/</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype><D:collection/></D:resourcetype>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
<D:response>
|
||
<D:href>/dir/sub/x.txt</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>42</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
// GETs should not be invoked during dry-run; the test asserts
|
||
// this by checking the output dir is empty afterwards.
|
||
if r.Method == "GET" {
|
||
t.Errorf("unexpected GET during dry-run: %s", r.URL.Path)
|
||
w.WriteHeader(http.StatusInternalServerError)
|
||
return
|
||
}
|
||
w.WriteHeader(http.StatusNotFound)
|
||
}))
|
||
defer server.Close()
|
||
|
||
parsedServerURL, _ := url.Parse(server.URL)
|
||
dirURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedServerURL.Host,
|
||
Path: "/dir/",
|
||
}
|
||
|
||
// runDryRun wraps a dry-run invocation and returns both the result
|
||
// and the captured stderr so the assertions stay readable.
|
||
runDryRun := func(t *testing.T, parallel int) (*core.DownloadResult, string) {
|
||
t.Helper()
|
||
tmpDir := t.TempDir()
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: parallel,
|
||
DryRun: true,
|
||
MaxDepth: 5,
|
||
Timeout: 5 * time.Second,
|
||
}
|
||
var result *core.DownloadResult
|
||
stderr := captureStderr(t, func() {
|
||
var err error
|
||
result, err = p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("dry-run failed: %v", err)
|
||
}
|
||
})
|
||
// Verify that no files or local directories were created.
|
||
entries, err := os.ReadDir(tmpDir)
|
||
if err != nil {
|
||
t.Fatalf("read tmpDir: %v", err)
|
||
}
|
||
if len(entries) != 0 {
|
||
var names []string
|
||
for _, e := range entries {
|
||
names = append(names, e.Name())
|
||
}
|
||
t.Errorf("dry-run wrote files to disk: %v", names)
|
||
}
|
||
return result, stderr
|
||
}
|
||
|
||
t.Run("sequential dry-run lists all matched entries", func(t *testing.T) {
|
||
result, stderr := runDryRun(t, 0)
|
||
if result.ChunksCount != 4 {
|
||
t.Errorf("ChunksCount = %d, want 4 (3 top-level files + 1 subdir file)", result.ChunksCount)
|
||
}
|
||
// 100 + 200 + 300 + 42 = 642 bytes (sum of getcontentlength).
|
||
const wantBytes = int64(100 + 200 + 300 + 42)
|
||
if result.BytesDownloaded != wantBytes {
|
||
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
|
||
}
|
||
|
||
// Stderr should mention every matched file and the subdir.
|
||
for _, want := range []string{
|
||
"would download: webdav://" + parsedServerURL.Host + "/dir/a.txt (100 bytes)",
|
||
"would download: webdav://" + parsedServerURL.Host + "/dir/b.txt (200 bytes)",
|
||
"would download: webdav://" + parsedServerURL.Host + "/dir/c.txt (300 bytes)",
|
||
"would recurse into: webdav://" + parsedServerURL.Host + "/dir/sub/",
|
||
"would download: webdav://" + parsedServerURL.Host + "/dir/sub/x.txt (42 bytes)",
|
||
} {
|
||
if !strings.Contains(stderr, want) {
|
||
t.Errorf("stderr missing %q\nfull output:\n%s", want, stderr)
|
||
}
|
||
}
|
||
})
|
||
|
||
t.Run("parallel dry-run lists all matched entries", func(t *testing.T) {
|
||
result, stderr := runDryRun(t, 4)
|
||
if result.ChunksCount != 4 {
|
||
t.Errorf("ChunksCount = %d, want 4", result.ChunksCount)
|
||
}
|
||
const wantBytes = int64(100 + 200 + 300 + 42)
|
||
if result.BytesDownloaded != wantBytes {
|
||
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, wantBytes)
|
||
}
|
||
// Order is non-deterministic with a worker pool, so just check
|
||
// the set of expected lines is present.
|
||
for _, want := range []string{
|
||
"/dir/a.txt (100 bytes)",
|
||
"/dir/b.txt (200 bytes)",
|
||
"/dir/c.txt (300 bytes)",
|
||
"/dir/sub/",
|
||
"/dir/sub/x.txt (42 bytes)",
|
||
} {
|
||
if !strings.Contains(stderr, want) {
|
||
t.Errorf("stderr missing %q\nfull output:\n%s", want, stderr)
|
||
}
|
||
}
|
||
})
|
||
|
||
t.Run("dry-run with reject filter only lists matched files", func(t *testing.T) {
|
||
tmpDir := t.TempDir()
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: dirURL,
|
||
Output: tmpDir,
|
||
Recursive: true,
|
||
RecursiveParallel: 0,
|
||
DryRun: true,
|
||
MaxDepth: 5,
|
||
Timeout: 5 * time.Second,
|
||
RejectPatterns: []string{"b.txt", "c.txt"},
|
||
}
|
||
stderr := captureStderr(t, func() {
|
||
_, err := p.Download(context.Background(), req)
|
||
if err != nil {
|
||
t.Fatalf("dry-run with reject failed: %v", err)
|
||
}
|
||
})
|
||
if !strings.Contains(stderr, "/dir/a.txt (100 bytes)") {
|
||
t.Errorf("expected a.txt in dry-run list:\n%s", stderr)
|
||
}
|
||
if strings.Contains(stderr, "/dir/b.txt") {
|
||
t.Errorf("b.txt should have been rejected but appeared:\n%s", stderr)
|
||
}
|
||
if strings.Contains(stderr, "/dir/c.txt") {
|
||
t.Errorf("c.txt should have been rejected but appeared:\n%s", stderr)
|
||
}
|
||
// Filter applies per-directory: sub/ is unaffected so sub/x.txt
|
||
// is still listed (it doesn't match b.txt or c.txt).
|
||
if !strings.Contains(stderr, "/dir/sub/x.txt (42 bytes)") {
|
||
t.Errorf("expected sub/x.txt in dry-run list (filter doesn't apply across dirs):\n%s", stderr)
|
||
}
|
||
})
|
||
}
|
||
|
||
// TestWebDAVSingleGracefulShutdown covers the graceful-shutdown contract
|
||
// for single-file WebDAV downloads: when the context is cancelled
|
||
// mid-transfer, the worker must persist a `.goget.meta` sidecar with
|
||
// the partial byte count so `--resume` can pick up where the user left
|
||
// off. On a clean completion the sidecar is removed.
|
||
func TestWebDAVSingleGracefulShutdown(t *testing.T) {
|
||
if !protocol.GlobalRegistry.Supports("http") {
|
||
httpClient, err := httpproto.NewClient(httpproto.DefaultConfig())
|
||
if err != nil {
|
||
t.Fatalf("failed to create http client: %v", err)
|
||
}
|
||
_ = protocol.Register(httpClient)
|
||
}
|
||
|
||
// Mock server: responds to GET with a stream that blocks until the
|
||
// request context is cancelled. This lets the test cancel from the
|
||
// outside and observe how much was downloaded before the cancel.
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method == "PROPFIND" {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/file.bin</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>1048576</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
if r.Method == "GET" {
|
||
w.Header().Set("Content-Type", "application/octet-stream")
|
||
w.Header().Set("ETag", `"etag-graceful"`)
|
||
w.Header().Set("Last-Modified", "Mon, 01 Jan 2026 00:00:00 GMT")
|
||
w.WriteHeader(http.StatusOK)
|
||
flusher, _ := w.(http.Flusher)
|
||
// Write a few KB synchronously so the client has something
|
||
// to write to disk, then block until the request is
|
||
// cancelled. The body is large enough that the client
|
||
// will not read to EOF naturally.
|
||
chunk := bytes.Repeat([]byte("a"), 4096)
|
||
for i := 0; i < 4; i++ {
|
||
if _, err := w.Write(chunk); err != nil {
|
||
return
|
||
}
|
||
if flusher != nil {
|
||
flusher.Flush()
|
||
}
|
||
}
|
||
// Drain or block until the client cancels.
|
||
<-r.Context().Done()
|
||
}
|
||
}))
|
||
defer server.Close()
|
||
|
||
parsedURL, _ := url.Parse(server.URL)
|
||
fileURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: parsedURL.Host,
|
||
Path: "/file.bin",
|
||
}
|
||
|
||
t.Run("ctx cancel saves resume metadata", func(t *testing.T) {
|
||
outDir := t.TempDir()
|
||
outPath := filepath.Join(outDir, "file.bin")
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: fileURL,
|
||
Output: outPath,
|
||
Resume: true,
|
||
Timeout: 5 * time.Second,
|
||
Ctx: ctx,
|
||
Recursive: false,
|
||
}
|
||
|
||
done := make(chan struct{})
|
||
go func() {
|
||
_, _ = p.Download(ctx, req)
|
||
close(done)
|
||
}()
|
||
|
||
// Give the worker a moment to start writing chunks, then
|
||
// cancel. The mock holds the connection open until cancelled.
|
||
time.Sleep(150 * time.Millisecond)
|
||
cancel()
|
||
|
||
select {
|
||
case <-done:
|
||
case <-time.After(3 * time.Second):
|
||
t.Fatal("download did not honour context cancellation within 3s")
|
||
}
|
||
|
||
// The mock wrote 4 × 4096 = 16384 bytes before blocking, so
|
||
// the resume metadata should record roughly that many bytes
|
||
// (give or take the transport buffer). The point of the test
|
||
// is the sidecar exists and is non-empty; exact byte counts
|
||
// are racy.
|
||
meta, err := output.LoadResumeMetadata(outPath)
|
||
if err != nil {
|
||
t.Fatalf("LoadResumeMetadata: %v", err)
|
||
}
|
||
if meta == nil {
|
||
t.Fatal("expected resume metadata after cancellation, got nil")
|
||
}
|
||
if meta.URL != req.URL.String() {
|
||
t.Errorf("metadata URL = %q, want %q", meta.URL, req.URL.String())
|
||
}
|
||
if meta.ETag != `"etag-graceful"` {
|
||
t.Errorf("metadata ETag = %q, want %q", meta.ETag, `"etag-graceful"`)
|
||
}
|
||
if meta.Downloaded <= 0 {
|
||
t.Errorf("metadata Downloaded = %d, want > 0", meta.Downloaded)
|
||
}
|
||
})
|
||
|
||
t.Run("successful download removes resume metadata", func(t *testing.T) {
|
||
// Pre-create a stale resume sidecar to verify the success
|
||
// path cleans it up. The mock for this subtest returns the
|
||
// full body immediately.
|
||
outDir := t.TempDir()
|
||
outPath := filepath.Join(outDir, "file.bin")
|
||
stale := output.NewResumeMetadata("stale://url", "stale-etag", "state-mod", 999, 999)
|
||
if err := stale.Save(outPath); err != nil {
|
||
t.Fatalf("pre-save: %v", err)
|
||
}
|
||
if _, err := output.LoadResumeMetadata(outPath); err != nil {
|
||
t.Fatalf("LoadResumeMetadata after pre-save: %v", err)
|
||
}
|
||
|
||
// Spin up a fresh mock that returns the whole file in one shot.
|
||
fullServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method == "PROPFIND" {
|
||
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
|
||
w.WriteHeader(http.StatusMultiStatus)
|
||
w.Write([]byte(`<?xml version="1.0"?>
|
||
<D:multistatus xmlns:D="DAV:">
|
||
<D:response>
|
||
<D:href>/file.bin</D:href>
|
||
<D:propstat>
|
||
<D:prop>
|
||
<D:resourcetype/>
|
||
<D:getcontentlength>16</D:getcontentlength>
|
||
</D:prop>
|
||
<D:status>HTTP/1.1 200 OK</D:status>
|
||
</D:propstat>
|
||
</D:response>
|
||
</D:multistatus>`))
|
||
return
|
||
}
|
||
if r.Method == "GET" {
|
||
w.Header().Set("Content-Type", "application/octet-stream")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte("0123456789abcdef"))
|
||
}
|
||
}))
|
||
defer fullServer.Close()
|
||
fullParsed, _ := url.Parse(fullServer.URL)
|
||
fullURL := &url.URL{
|
||
Scheme: "webdav",
|
||
Host: fullParsed.Host,
|
||
Path: "/file.bin",
|
||
}
|
||
|
||
p := NewProtocol()
|
||
req := &core.DownloadRequest{
|
||
URL: fullURL,
|
||
Output: outPath,
|
||
Resume: true,
|
||
Timeout: 5 * time.Second,
|
||
Recursive: false,
|
||
}
|
||
if _, err := p.Download(context.Background(), req); err != nil {
|
||
t.Fatalf("download failed: %v", err)
|
||
}
|
||
|
||
// The pre-seeded stale sidecar should be gone after a
|
||
// successful download.
|
||
meta, err := output.LoadResumeMetadata(outPath)
|
||
if err != nil {
|
||
t.Fatalf("LoadResumeMetadata after success: %v", err)
|
||
}
|
||
if meta != nil {
|
||
t.Errorf("expected resume metadata to be cleared, got %+v", meta)
|
||
}
|
||
})
|
||
}
|