//go:build linux || freebsd // +build linux freebsd // Package webdav implements WebDAV (Web Distributed Authoring and Versioning) downloads and uploads. package webdav import ( "context" "crypto/tls" "encoding/xml" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "sync" "time" "codeberg.org/petrbalvin/goget/internal/core" "codeberg.org/petrbalvin/goget/internal/output" "codeberg.org/petrbalvin/goget/internal/pool" "codeberg.org/petrbalvin/goget/internal/protocol" "codeberg.org/petrbalvin/goget/internal/transport" ) // Multistatus represents a WebDAV PROPFIND response. type Multistatus struct { XMLName xml.Name `xml:"multistatus"` Responses []Response `xml:"response"` } // Response represents a single WebDAV resource response. type Response struct { Href string `xml:"href"` Propstats []Propstat `xml:"propstat"` } // Propstat represents properties status. type Propstat struct { Prop Prop `xml:"prop"` Status string `xml:"status"` } // Prop represents the WebDAV resource properties. type Prop struct { ResourceType ResourceType `xml:"resourcetype"` ContentLength string `xml:"getcontentlength"` GetLastModified string `xml:"getlastmodified"` } // ResourceType represents the resource type. type ResourceType struct { Collection *struct{} `xml:"collection"` } // isCollection checks if the resource is a directory/collection. func (r *Response) isCollection() bool { for _, ps := range r.Propstats { if strings.Contains(ps.Status, "200") { if ps.Prop.ResourceType.Collection != nil { return true } } } return false } // getContentLength extracts the content length of the resource. func (r *Response) getContentLength() int64 { for _, ps := range r.Propstats { if strings.Contains(ps.Status, "200") && ps.Prop.ContentLength != "" { if s, err := strconv.ParseInt(ps.Prop.ContentLength, 10, 64); err == nil { return s } } } return -1 } // getContentType extracts the content type of the resource. func (r *Response) getContentType() string { for _, ps := range r.Propstats { if strings.Contains(ps.Status, "200") && ps.Prop.ResourceType.Collection == nil { return "" // Could extract getcontenttype if needed — depends on WebDAV server response } } return "" } // getLastModified extracts the last modified date of the resource. func (r *Response) getLastModified() string { for _, ps := range r.Propstats { if strings.Contains(ps.Status, "200") && ps.Prop.GetLastModified != "" { return ps.Prop.GetLastModified } } return "" } // WebDAVEntry represents a parsed directory listing entry. type WebDAVEntry struct { URL *url.URL IsDir bool Size int64 LastModified string } // Protocol implements WebDAV downloads and uploads. type Protocol struct { protocol.BaseProtocol tlsConfig *transport.TLSConfig insecureSkip bool proxyURL string // httpTransport is a process-wide shared transport for all WebDAV // operations. Built lazily on first use; see getHTTPTransport. Sharing // the transport enables HTTP connection pooling (keep-alive, TLS // session resumption, HTTP/2 multiplexing) across the PROPFIND/GET/PUT // calls of a recursive download. httpTransport *http.Transport transportMu sync.Mutex } // NewProtocol creates a new WebDAV protocol handler. func NewProtocol() *Protocol { return &Protocol{ BaseProtocol: *protocol.NewBaseProtocol(protocol.ProtocolInfo{ Name: "webdav", Scheme: "webdav", DefaultPort: 443, Operations: []string{"download", "upload"}, Features: []string{"download", "upload", "resume", "recursive"}, }), } } // ConfigureTLS configures TLS options for WebDAV connections. func (p *Protocol) ConfigureTLS(tlsCfg *transport.TLSConfig, insecure bool) { p.tlsConfig = tlsCfg p.insecureSkip = insecure } // ConfigureProxy configures proxy for WebDAV HTTP connections. func (p *Protocol) ConfigureProxy(proxyURL string) { p.proxyURL = proxyURL } // ShowInfo displays WebDAV resource metadata (PROPPATCH info). func (p *Protocol) ShowInfo(ctx context.Context, u *url.URL) error { httpURL := toHTTPURL(u) httpReq, err := http.NewRequestWithContext(ctx, "PROPFIND", httpURL.String(), nil) if err != nil { return fmt.Errorf("failed to create PROPFIND request: %w", err) } httpReq.Header.Set("Depth", "0") httpReq.Header.Set("Content-Type", "text/xml; charset=utf-8") httpReq.Header.Set("User-Agent", "Goget/"+core.Version) if u.User != nil { username := u.User.Username() password, _ := u.User.Password() httpReq.SetBasicAuth(username, password) } client := p.getHTTPClient(15 * time.Second) resp, err := client.Do(httpReq) if err != nil { return fmt.Errorf("PROPFIND request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusMultiStatus { return fmt.Errorf("server returned status %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("failed to read response: %w", err) } var multistatus Multistatus if err := xml.Unmarshal(body, &multistatus); err != nil { return fmt.Errorf("failed to parse response: %w", err) } if len(multistatus.Responses) == 0 { return fmt.Errorf("no resources found") } r := multistatus.Responses[0] fmt.Printf("URL: %s\n", u.String()) fmt.Printf("HREF: %s\n", r.Href) fmt.Printf("Is Directory: %v\n", r.isCollection()) fmt.Printf("Size: %d\n", r.getContentLength()) fmt.Printf("LastModified: %s\n", r.getLastModified()) return nil } // Scheme returns the primary scheme of the protocol. func (p *Protocol) Scheme() string { return "webdav" } // CanHandle checks if the protocol can handle the given URL. func (p *Protocol) CanHandle(u *url.URL) bool { if u == nil { return false } scheme := strings.ToLower(u.Scheme) return scheme == "webdav" || scheme == "webdavs" } // toHTTPURL rewrites webdav(s) schemes to http(s). func toHTTPURL(u *url.URL) *url.URL { newURL := *u scheme := strings.ToLower(newURL.Scheme) if scheme == "webdav" { newURL.Scheme = "http" } else if scheme == "webdavs" { newURL.Scheme = "https" } return &newURL } // getHTTPTransport returns the protocol's shared http.Transport, building // it on first use. The transport is reused across every WebDAV request // (PROPFIND/GET/PUT/MKCOL/PROPPATCH) so that idle TCP connections, TLS // sessions, and HTTP/2 streams are pooled — turning a 50-file recursive // download from 50 fresh TCP+TLS handshakes into a single handshake plus // 49 reuse-or-keepalive calls. // // The lazy init is guarded by transportMu so that concurrent callers // (the recursive download's worker pool calls Download in parallel) only // build the transport once. func (p *Protocol) getHTTPTransport() *http.Transport { p.transportMu.Lock() defer p.transportMu.Unlock() if p.httpTransport != nil { return p.httpTransport } tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: p.insecureSkip}, //nolint:gosec DisableCompression: true, Proxy: http.ProxyFromEnvironment, MaxIdleConns: 100, MaxIdleConnsPerHost: 10, IdleConnTimeout: 90 * time.Second, } // Apply custom proxy if configured. SOCKS5 must be implemented as a // custom DialContext — Go's stdlib http.ProxyURL only supports HTTP // CONNECT proxies. if p.proxyURL != "" { proxyCfg, err := transport.ParseProxyConfig(p.proxyURL) if err == nil { proxyTransport, err := transport.NewProxyTransport(proxyCfg, tr) if err == nil { tr = proxyTransport } } } p.httpTransport = tr return p.httpTransport } // getHTTPClient returns an http.Client that wraps the protocol's pooled // http.Transport. A fresh wrapper is cheap and lets each caller specify // its own Timeout — Timeout lives on http.Client, while connection // reuse lives on the shared http.Transport. // // A zero or negative timeout is upgraded to a 30-minute default so that // downloadFile and doUpload, which only set req.Timeout, still get a // sensible upper bound on long transfers. func (p *Protocol) getHTTPClient(timeout time.Duration) *http.Client { if timeout <= 0 { timeout = 30 * time.Minute } return &http.Client{ Transport: p.getHTTPTransport(), Timeout: timeout, } } // isCollection performs a Depth: 0 PROPFIND to check if the target is a collection. func (p *Protocol) isCollection(ctx context.Context, u *url.URL, req *core.DownloadRequest) (bool, int64, error) { httpURL := toHTTPURL(u) httpReq, err := http.NewRequestWithContext(ctx, "PROPFIND", httpURL.String(), nil) if err != nil { return false, -1, err } httpReq.Header.Set("Depth", "0") httpReq.Header.Set("Content-Type", "text/xml; charset=utf-8") // Apply request headers. for k, v := range req.Headers { httpReq.Header.Set(k, v) } userAgent := "Goget/" + core.Version if req.Headers != nil && req.Headers["User-Agent"] != "" { userAgent = req.Headers["User-Agent"] } httpReq.Header.Set("User-Agent", userAgent) if u.User != nil { username := u.User.Username() password, _ := u.User.Password() httpReq.SetBasicAuth(username, password) } client := p.getHTTPClient(15 * time.Second) resp, err := client.Do(httpReq) if err != nil { return false, -1, err } defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK, http.StatusMultiStatus: // Continue to parse response body. case http.StatusLocked: return false, -1, &core.GogetError{ Type: core.ErrProtocol, Message: "webdav resource is locked — try again after unlock", URL: u.String(), } case http.StatusUnauthorized: return false, -1, &core.GogetError{ Type: core.ErrAuth, Message: "webdav server requires authentication", URL: u.String(), } case http.StatusForbidden: return false, -1, &core.GogetError{ Type: core.ErrProtocol, Message: "webdav server forbidden access", URL: u.String(), } default: // Fallback to assuming it's a file (non-collection). if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] PROPFIND returned status %d, assuming non-collection\n", resp.StatusCode) } return false, -1, nil } body, err := io.ReadAll(resp.Body) if err != nil { return false, -1, err } var multistatus Multistatus if err := xml.Unmarshal(body, &multistatus); err != nil { return false, -1, nil } if len(multistatus.Responses) > 0 { first := multistatus.Responses[0] return first.isCollection(), first.getContentLength(), nil } return false, -1, nil } // listDirectory performs a PROPFIND to list directory contents. // depth can be "0", "1", or "infinity". func (p *Protocol) listDirectory(ctx context.Context, u *url.URL, req *core.DownloadRequest, depth string) ([]WebDAVEntry, error) { httpURL := toHTTPURL(u) httpReq, err := http.NewRequestWithContext(ctx, "PROPFIND", httpURL.String(), nil) if err != nil { return nil, err } httpReq.Header.Set("Depth", depth) httpReq.Header.Set("Content-Type", "text/xml; charset=utf-8") for k, v := range req.Headers { httpReq.Header.Set(k, v) } userAgent := "Goget/" + core.Version if req.Headers != nil && req.Headers["User-Agent"] != "" { userAgent = req.Headers["User-Agent"] } httpReq.Header.Set("User-Agent", userAgent) if u.User != nil { username := u.User.Username() password, _ := u.User.Password() httpReq.SetBasicAuth(username, password) } client := p.getHTTPClient(15 * time.Second) resp, err := client.Do(httpReq) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusMultiStatus { return nil, fmt.Errorf("webdav server returned status %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } var multistatus Multistatus if err := xml.Unmarshal(body, &multistatus); err != nil { return nil, fmt.Errorf("failed to parse webdav xml: %w", err) } var entries []WebDAVEntry dirPath := strings.TrimSuffix(u.Path, "/") userAuth := u.User for _, r := range multistatus.Responses { resolvedHref := r.Href // Resolve href to an absolute URL. var itemURL *url.URL if strings.HasPrefix(resolvedHref, "http://") || strings.HasPrefix(resolvedHref, "https://") { parsed, err := url.Parse(resolvedHref) if err != nil { continue } parsed.Scheme = u.Scheme // Use caller's scheme. itemURL = parsed } else { basePathURL := &url.URL{ Scheme: u.Scheme, Host: u.Host, User: userAuth, } parsed, err := basePathURL.Parse(resolvedHref) if err != nil { continue } itemURL = parsed } itemPath := strings.TrimSuffix(itemURL.Path, "/") if itemPath == dirPath { continue } name := filepath.Base(itemURL.Path) // Apply accept patterns (glob-style). if len(req.AcceptPatterns) > 0 { matched := false for _, pat := range req.AcceptPatterns { if matched, _ = filepath.Match(pat, name); matched { break } } if !matched { continue } } // Apply reject patterns (glob-style). if len(req.RejectPatterns) > 0 { matched := false for _, pat := range req.RejectPatterns { if matched, _ = filepath.Match(pat, name); matched { break } } if matched { continue } } entries = append(entries, WebDAVEntry{ URL: itemURL, IsDir: r.isCollection(), Size: r.getContentLength(), LastModified: r.getLastModified(), }) } return entries, nil } // filterEntries applies accept/reject filters and returns only matching entries. func filterEntries(entries []WebDAVEntry, req *core.DownloadRequest) []WebDAVEntry { var result []WebDAVEntry for _, entry := range entries { name := filepath.Base(entry.URL.Path) // Accept pattern: entry must match at least one pattern. if len(req.AcceptPatterns) > 0 { matched := false for _, pat := range req.AcceptPatterns { if m, _ := filepath.Match(pat, name); m { matched = true break } } if !matched { continue } } // Reject pattern: entry must not match any reject pattern. if len(req.RejectPatterns) > 0 { matched := false for _, pat := range req.RejectPatterns { if m, _ := filepath.Match(pat, name); m { matched = true break } } if matched { continue } } result = append(result, entry) } return result } // setFileTimestamp sets the file's modification time from WebDAV last-modified. func setFileTimestamp(path string, lastModified string) { if lastModified == "" { return } modTime, err := time.Parse(http.TimeFormat, lastModified) if err != nil { return } if err := os.Chtimes(path, time.Now(), modTime); err != nil { // Non-fatal — log only if verbose. } } // downloadFile downloads a single file with optional resume support. func (p *Protocol) downloadFile(ctx context.Context, req *core.DownloadRequest, entrySize int64, lastModified string) (*core.DownloadResult, error) { startTime := time.Now() httpURL := toHTTPURL(req.URL) // Max file size check. if entrySize > 0 && req.MaxFileSize > 0 && entrySize > req.MaxFileSize { return nil, core.NewNetworkError( fmt.Sprintf("file too large: %d bytes (max %d)", entrySize, req.MaxFileSize), nil, req.URL.String()) } resumed := false startOffset := int64(0) // Check partial download for resume. if req.Resume && req.Output != "" && req.Output != "-" { if fi, statErr := os.Stat(req.Output); statErr == nil { localSize := fi.Size() if entrySize > 0 && localSize > 0 && localSize < entrySize { startOffset = localSize resumed = true if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Resuming from byte %d (local: %d, remote: %d)\n", startOffset, localSize, entrySize) } } } } httpReq, err := http.NewRequestWithContext(ctx, "GET", httpURL.String(), nil) if err != nil { return nil, core.NewNetworkError("failed to create request", err, req.URL.String()) } httpReq.Header.Set("User-Agent", "Goget/"+core.Version) // Apply custom headers and auth. for k, v := range req.Headers { httpReq.Header.Set(k, v) } if req.URL.User != nil { username := req.URL.User.Username() password, _ := req.URL.User.Password() httpReq.SetBasicAuth(username, password) } // Range resume header. if startOffset > 0 { httpReq.Header.Set("Range", fmt.Sprintf("bytes=%d-", startOffset)) if lastModified != "" { httpReq.Header.Set("If-Range", lastModified) } } client := p.getHTTPClient(req.Timeout) resp, err := client.Do(httpReq) if err != nil { return nil, core.NewNetworkError("download request failed", err, req.URL.String()) } defer resp.Body.Close() switch resp.StatusCode { case http.StatusUnauthorized: return nil, &core.GogetError{ Type: core.ErrAuth, Message: "webdav download requires authentication", URL: req.URL.String(), } case http.StatusLocked: return nil, &core.GogetError{ Type: core.ErrProtocol, Message: "webdav resource is locked", URL: req.URL.String(), } default: acceptCode := http.StatusOK if startOffset > 0 { acceptCode = http.StatusPartialContent } if resp.StatusCode != acceptCode && resp.StatusCode != http.StatusOK { return nil, core.NewProtocolError( fmt.Sprintf("webdav download failed with status %d(%s)", resp.StatusCode, resp.Status), nil, req.URL.String()) } } // Open output file. var outFile *os.File if req.Output != "" && req.Output != "-" { if err := os.MkdirAll(filepath.Dir(req.Output), 0755); err != nil { return nil, core.NewFileError("failed to create output directory", err) } flag := os.O_CREATE | os.O_WRONLY if startOffset > 0 { flag |= os.O_APPEND } else { flag |= os.O_TRUNC } outFile, err = os.OpenFile(req.Output, flag, 0644) if err != nil { return nil, core.NewFileError("failed to open output file", err) } defer outFile.Close() } var writer io.Writer if outFile != nil { writer = outFile } else if req.Writer != nil { writer = req.Writer } else { writer = io.Discard } bytesDownloaded, err := streamWebDAVBody(ctx, writer, resp.Body, req, startOffset, resp.Header.Get("ETag"), resp.Header.Get("Last-Modified"), entrySize, &resumed) if err != nil { // On ctx cancellation, streamWebDAVBody returns nil; the error // is the context error and partial progress was already saved // to ResumeMetadata by the streamer. return nil, err } duration := time.Since(startTime) return &core.DownloadResult{ BytesDownloaded: bytesDownloaded, Duration: duration, Protocol: "webdav", OutputPath: req.Output, Resumed: resumed, ChunksCount: 1, ContentType: resp.Header.Get("Content-Type"), }, nil } // streamWebDAVBody copies the HTTP response body to dst and counts the // bytes transferred. On ctx.Done() it saves a ResumeMetadata sidecar // (so the user can resume after Ctrl+C) and returns the context error; // on a write error it returns a NewNetworkError wrapping the cause. // The caller is responsible for deleting the metadata on success. func streamWebDAVBody( ctx context.Context, dst io.Writer, src io.Reader, req *core.DownloadRequest, startOffset int64, etag, lastModified string, totalSize int64, resumed *bool, ) (int64, error) { // 32 KiB matches the buffer size used by io.Copy and the existing // HTTP downloader; large enough to amortise Write syscalls, small // enough to detect ctx cancellation promptly. buf := make([]byte, 32*1024) var written int64 for { // Read first, then check the context. A premature EOF caused by // the server tearing down the connection (a typical reaction to // client-side cancellation) is indistinguishable from a clean // EOF without looking at ctx.Err() after the read returns. n, readErr := src.Read(buf) if n > 0 { if _, writeErr := dst.Write(buf[:n]); writeErr != nil { return written, core.NewNetworkError("failed to write response body", writeErr, req.URL.String()) } written += int64(n) } if readErr == io.EOF { // Real end of stream, or a server-side close triggered by // our cancellation? Only ctx.Err() can tell. if ctx.Err() != nil { saveWebDAVResume(req, startOffset+written, totalSize, etag, lastModified, *resumed) return written, ctx.Err() } break } if readErr != nil { if ctx.Err() != nil { // Cancellation often surfaces as a transport error on // the underlying read; treat it as a cancellation so // the resume sidecar is written. saveWebDAVResume(req, startOffset+written, totalSize, etag, lastModified, *resumed) return written, ctx.Err() } return written, core.NewNetworkError("failed to read response body", readErr, req.URL.String()) } // Healthy read: also poll for cancellation between reads so a // pause in the server's response is detected promptly. if ctx.Err() != nil { saveWebDAVResume(req, startOffset+written, totalSize, etag, lastModified, *resumed) return written, ctx.Err() } } // Successful download — drop any stale resume metadata. if req.Output != "" && req.Output != "-" { _ = output.DeleteResumeMetadata(req.Output) } return written, nil } // saveWebDAVResume persists the current download progress so the user // can resume after an interruption. No-op if the destination is stdout // or if --resume wasn't requested. func saveWebDAVResume(req *core.DownloadRequest, downloaded, total int64, etag, lastModified string, wasResumed bool) { if !req.Resume || req.Output == "" || req.Output == "-" || downloaded <= 0 { return } _ = wasResumed // reserved for future "true resume" reporting meta := output.NewResumeMetadata(req.URL.String(), etag, lastModified, downloaded, total) if err := meta.Save(req.Output); err != nil && req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to save resume metadata: %v\n", err) } } // Download downloads a file or directory from a WebDAV server. func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) { if !p.CanHandle(req.URL) { return nil, core.NewProtocolError("url not supported by webdav protocol", nil, req.URL.String()) } isDir, size, err := p.isCollection(ctx, req.URL, req) if err != nil && req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] PROPFIND failed: %v, trying default download\n", err) } if isDir { if !req.Recursive { return nil, core.NewProtocolError("remote path is a directory, use --recursive to download", nil, req.URL.String()) } return p.downloadRecursive(ctx, req) } // Use native WebDAV file download with resume support. return p.downloadFile(ctx, req, size, "") } // downloadRecursive recursively downloads all files and subdirectories. // // Dispatches to the sequential implementation (preserved behaviour) when // req.RecursiveParallel <= 1, or to a worker-pool implementation when // req.RecursiveParallel > 1. Subdirectory recursion is propagated to // downloadRecursive itself, so the same worker limit applies at every // level of the tree. // dryRunVerb returns the verb used in warning messages depending on // whether the request is a dry-run or an actual download. Keeps the // warning text grammatical without scattering the same ternary around. func dryRunVerb(dryRun bool) string { if dryRun { return "dry-run" } return "download" } func (p *Protocol) downloadRecursive(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) { if req.RecursiveParallel <= 1 { return p.downloadRecursiveSequential(ctx, req) } return p.downloadRecursiveParallel(ctx, req) } // downloadRecursiveSequential is the original behaviour: one entry at a // time, no goroutines. Used when req.RecursiveParallel <= 1. func (p *Protocol) downloadRecursiveSequential(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) { startTime := time.Now() outputDir := req.Output if outputDir == "" { outputDir = filepath.Base(req.URL.Path) if outputDir == "" || outputDir == "/" { outputDir = "download" } } var bytesDownloaded int64 var chunksCount int // Use infinity depth to get all entries in one PROPFIND call. depth := "infinity" if req.MaxDepth <= 1 { depth = "1" } entries, err := p.listDirectory(ctx, req.URL, req, depth) if err != nil { return nil, core.NewProtocolError(fmt.Sprintf("failed to list directory: %v", err), nil, req.URL.String()) } if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] listing %d entries in %s (MaxDepth=%d, depth=%s)\n", len(entries), req.URL.Path, req.MaxDepth, depth) } filteredEntries := filterEntries(entries, req) if req.Verbose && len(filteredEntries) != len(entries) { fmt.Fprintf(os.Stderr, "[webdav] filtered %d → %d entries (patterns applied)\n", len(entries), len(filteredEntries)) } for _, entry := range filteredEntries { select { case <-ctx.Done(): return nil, ctx.Err() default: } // Respect MaxDepth to avoid infinite recursion. if entry.IsDir && req.MaxDepth == 0 { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] MaxDepth reached, skipping directory %s\n", entry.URL.String()) } continue } itemName := filepath.Base(entry.URL.Path) itemOutputPath := filepath.Join(outputDir, itemName) // Dry-run short-circuits the actual download. The matched entry // is printed (size for files, "would recurse" for subdirs) and // subdirectories are still recursed into so the operator can see // the full blast radius before committing. No files or local // directories are created on disk. if req.DryRun { if entry.IsDir { fmt.Fprintf(os.Stderr, "[webdav] [dry-run] would recurse into: %s/\n", entry.URL.String()) subReq := *req subReq.URL = entry.URL subReq.Output = itemOutputPath subReq.MaxDepth = req.MaxDepth - 1 subReq.RecursiveParallel = req.RecursiveParallel subReq.DryRun = true subResult, err := p.downloadRecursive(ctx, &subReq) if err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to dry-run sub-directory %s: %v\n", entry.URL.String(), err) } continue } bytesDownloaded += subResult.BytesDownloaded chunksCount += subResult.ChunksCount } else { fmt.Fprintf(os.Stderr, "[webdav] [dry-run] would download: %s (%d bytes)\n", entry.URL.String(), entry.Size) bytesDownloaded += entry.Size chunksCount++ } continue } // Ensure parent directory exists for file output. if err := os.MkdirAll(filepath.Dir(itemOutputPath), 0755); err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] failed to create directory %s: %v\n", filepath.Dir(itemOutputPath), err) } continue } if entry.IsDir { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] recurse into directory %s (remaining depth=%d)\n", entry.URL.String(), req.MaxDepth-1) } subReq := &core.DownloadRequest{ URL: entry.URL, Output: itemOutputPath, Resume: req.Resume, Timeout: req.Timeout, Verbose: req.Verbose, DebugTransport: req.DebugTransport, Headers: req.Headers, Proxy: req.Proxy, AutoDecompress: req.AutoDecompress, ProgressCallback: req.ProgressCallback, Recursive: true, RecursiveParallel: req.RecursiveParallel, MaxDepth: req.MaxDepth - 1, AcceptPatterns: req.AcceptPatterns, RejectPatterns: req.RejectPatterns, Ctx: req.Ctx, } subResult, err := p.downloadRecursive(ctx, subReq) if err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to download sub-directory %s: %v\n", entry.URL.String(), err) } continue } bytesDownloaded += subResult.BytesDownloaded chunksCount += subResult.ChunksCount } else { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Downloading file %s -> %s (depth=%d)\n", entry.URL.String(), itemOutputPath, req.MaxDepth) } subReq := &core.DownloadRequest{ URL: entry.URL, Output: itemOutputPath, Resume: req.Resume, Timeout: req.Timeout, Verbose: req.Verbose, DebugTransport: req.DebugTransport, Headers: req.Headers, Proxy: req.Proxy, AutoDecompress: req.AutoDecompress, ProgressCallback: req.ProgressCallback, Recursive: false, MaxDepth: req.MaxDepth - 1, Ctx: req.Ctx, } subResult, err := p.downloadFile(ctx, subReq, entry.Size, entry.LastModified) if err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to download file %s: %v\n", entry.URL.String(), err) } continue } // Apply keep-timestamps if requested. if req.KeepTimestamps { setFileTimestamp(itemOutputPath, entry.LastModified) } bytesDownloaded += subResult.BytesDownloaded chunksCount++ } } duration := time.Since(startTime) return &core.DownloadResult{ BytesDownloaded: bytesDownloaded, Duration: duration, ChunksCount: chunksCount, Protocol: "webdav", }, nil } // webdavTask represents a unit of work for the concurrent recursive // download: a file or subdirectory entry with its pre-built request. type webdavTask struct { entry WebDAVEntry subReq *core.DownloadRequest outPath string isDir bool } // downloadRecursiveParallel is the worker-pool implementation used when // req.RecursiveParallel > 1. Top-level entries are processed concurrently, // bounded by the pool. Subdirectory recursion is delegated back to // downloadRecursive, so the same RecursiveParallel limit applies at every // level of the tree. Stats are aggregated under a mutex. func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) { startTime := time.Now() outputDir := req.Output if outputDir == "" { outputDir = filepath.Base(req.URL.Path) if outputDir == "" || outputDir == "/" { outputDir = "download" } } // Use infinity depth to get all entries in one PROPFIND call. depth := "infinity" if req.MaxDepth <= 1 { depth = "1" } entries, err := p.listDirectory(ctx, req.URL, req, depth) if err != nil { return nil, core.NewProtocolError(fmt.Sprintf("failed to list directory: %v", err), nil, req.URL.String()) } if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] listing %d entries in %s (MaxDepth=%d, depth=%s, parallel=%d)\n", len(entries), req.URL.Path, req.MaxDepth, depth, req.RecursiveParallel) } filteredEntries := filterEntries(entries, req) if req.Verbose && len(filteredEntries) != len(entries) { fmt.Fprintf(os.Stderr, "[webdav] filtered %d → %d entries (patterns applied)\n", len(entries), len(filteredEntries)) } // Aggregated stats under a mutex. Subdirectory recursion adds to // these counters, so they reflect the entire subtree processed by // this call. var ( statsMu sync.Mutex totalBytes int64 totalChunks int ) processedDirs := 1 wp := pool.New(ctx, req.RecursiveParallel, func(taskCtx context.Context, t webdavTask) { if taskCtx.Err() != nil { return } if t.isDir { subResult, err := p.downloadRecursive(taskCtx, t.subReq) if err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to %s sub-directory %s: %v\n", dryRunVerb(req.DryRun), t.entry.URL.String(), err) } return } statsMu.Lock() totalBytes += subResult.BytesDownloaded totalChunks += subResult.ChunksCount statsMu.Unlock() return } subResult, err := p.downloadFile(taskCtx, t.subReq, t.entry.Size, t.entry.LastModified) if err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to download file %s: %v\n", t.entry.URL.String(), err) } return } // Apply keep-timestamps if requested. if req.KeepTimestamps { setFileTimestamp(t.outPath, t.entry.LastModified) } statsMu.Lock() totalBytes += subResult.BytesDownloaded totalChunks++ statsMu.Unlock() }) for _, entry := range filteredEntries { // Respect MaxDepth to avoid infinite recursion. if entry.IsDir && req.MaxDepth == 0 { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] MaxDepth reached, skipping directory %s\n", entry.URL.String()) } continue } itemName := filepath.Base(entry.URL.Path) itemOutputPath := filepath.Join(outputDir, itemName) // Dry-run short-circuits the actual download. The matched entry // is printed (size for files, "would recurse" for subdirs) and // subdirectories are still recursed into so the operator can see // the full blast radius before committing. No files or local // directories are created on disk. if req.DryRun { if entry.IsDir { fmt.Fprintf(os.Stderr, "[webdav] [dry-run] would recurse into: %s/\n", entry.URL.String()) subReq := *req subReq.URL = entry.URL subReq.Output = itemOutputPath subReq.MaxDepth = req.MaxDepth - 1 subReq.RecursiveParallel = req.RecursiveParallel subReq.DryRun = true wp.Submit(webdavTask{entry: entry, subReq: &subReq, outPath: itemOutputPath, isDir: true}) } else { fmt.Fprintf(os.Stderr, "[webdav] [dry-run] would download: %s (%d bytes)\n", entry.URL.String(), entry.Size) statsMu.Lock() totalBytes += entry.Size totalChunks++ statsMu.Unlock() } continue } // Ensure parent directory exists for file output. if err := os.MkdirAll(filepath.Dir(itemOutputPath), 0755); err != nil { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] failed to create directory %s: %v\n", filepath.Dir(itemOutputPath), err) } continue } if entry.IsDir { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] recurse into directory %s (remaining depth=%d)\n", entry.URL.String(), req.MaxDepth-1) } subReq := &core.DownloadRequest{ URL: entry.URL, Output: itemOutputPath, Resume: req.Resume, Timeout: req.Timeout, Verbose: req.Verbose, DebugTransport: req.DebugTransport, Headers: req.Headers, Proxy: req.Proxy, AutoDecompress: req.AutoDecompress, ProgressCallback: req.ProgressCallback, Recursive: true, RecursiveParallel: req.RecursiveParallel, MaxDepth: req.MaxDepth - 1, AcceptPatterns: req.AcceptPatterns, RejectPatterns: req.RejectPatterns, Ctx: req.Ctx, } wp.Submit(webdavTask{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: true}) } else { if req.Verbose { fmt.Fprintf(os.Stderr, "[webdav] Downloading file %s -> %s (depth=%d)\n", entry.URL.String(), itemOutputPath, req.MaxDepth) } subReq := &core.DownloadRequest{ URL: entry.URL, Output: itemOutputPath, Resume: req.Resume, Timeout: req.Timeout, Verbose: req.Verbose, DebugTransport: req.DebugTransport, Headers: req.Headers, Proxy: req.Proxy, AutoDecompress: req.AutoDecompress, ProgressCallback: req.ProgressCallback, Recursive: false, MaxDepth: req.MaxDepth - 1, Ctx: req.Ctx, } wp.Submit(webdavTask{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: false}) } } wp.Wait() _ = processedDirs // reserved for future per-directory accounting duration := time.Since(startTime) return &core.DownloadResult{ BytesDownloaded: totalBytes, Duration: duration, ChunksCount: totalChunks, Protocol: "webdav", }, nil } // mkcol creates a WebDAV collection (directory). func (p *Protocol) mkcol(ctx context.Context, httpURL *url.URL, req *core.UploadRequest) error { mkReq, err := http.NewRequestWithContext(ctx, "MKCOL", httpURL.String(), nil) if err != nil { return err } if req.URL.User != nil { username := req.URL.User.Username() password, _ := req.URL.User.Password() mkReq.SetBasicAuth(username, password) } client := p.getHTTPClient(15 * time.Second) resp, err := client.Do(mkReq) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { return fmt.Errorf("MKCOL failed with status: %s", resp.Status) } return nil } // Upload uploads a file to a WebDAV server using HTTP PUT method. func (p *Protocol) Upload(ctx context.Context, req *core.UploadRequest) (*core.UploadResult, error) { httpURL := toHTTPURL(req.URL) file, err := os.Open(req.Input) if err != nil { return nil, core.NewFileError("failed to open input file", err) } defer file.Close() fileInfo, err := file.Stat() if err != nil { return nil, core.NewFileError("failed to stat input file", err) } startTime := time.Now() // Attempt PUT. result, err := p.doUpload(ctx, httpURL, file, fileInfo, req) if err != nil { // If PUT fails with 405 (Method Not Allowed), try MKCOL on parent. if strings.Contains(err.Error(), "405") || strings.Contains(err.Error(), "409") { parentURL := &url.URL{ Scheme: httpURL.Scheme, Host: httpURL.Host, User: httpURL.User, Path: filepath.Dir(httpURL.Path), } if mkErr := p.mkcol(ctx, parentURL, req); mkErr == nil { // Retry PUT after creating collection. file.Seek(0, io.SeekStart) return p.doUpload(ctx, httpURL, file, fileInfo, req) } } return nil, err } duration := time.Since(startTime) result.Duration = duration return result, nil } // doUpload performs the actual HTTP PUT request. func (p *Protocol) doUpload(ctx context.Context, httpURL *url.URL, file *os.File, fileInfo os.FileInfo, req *core.UploadRequest) (*core.UploadResult, error) { httpReq, err := http.NewRequestWithContext(ctx, "PUT", httpURL.String(), file) if err != nil { return nil, fmt.Errorf("failed to create PUT request: %w", err) } httpReq.Header.Set("Content-Type", "application/octet-stream") httpReq.Header.Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) for k, v := range req.Headers { httpReq.Header.Set(k, v) } if req.URL.User != nil { username := req.URL.User.Username() password, _ := req.URL.User.Password() httpReq.SetBasicAuth(username, password) } client := p.getHTTPClient(req.Timeout) resp, err := client.Do(httpReq) if err != nil { return nil, fmt.Errorf("PUT request failed: %w", err) } defer resp.Body.Close() switch resp.StatusCode { case http.StatusLocked: return nil, &core.GogetError{ Type: core.ErrProtocol, Message: "webdav upload failed: resource is locked — try again after unlock", URL: req.URL.String(), } case http.StatusOK, http.StatusCreated, http.StatusNoContent: // Success. default: return nil, core.NewProtocolError( fmt.Sprintf("webdav upload failed with status %d(%s)", resp.StatusCode, resp.Status), nil, req.URL.String()) } return &core.UploadResult{ BytesUploaded: fileInfo.Size(), Protocol: "webdav", ResultURL: req.URL.String(), }, nil }