petrbalvin/goget
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: add unified worker pool for recursive downloads

Browse Source
This commit is contained in:
petrbalvin
2026-06-29 15:30:31 +02:00
parent 1672698936
commit 11aed3771d
6 changed files with 563 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/protocol/webdav/protocol.go
+57 -79
View File
@@ -21,6 +21,7 @@ import (
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
"codeberg.org/petrbalvin/goget/internal/pool"
"codeberg.org/petrbalvin/goget/internal/protocol"
"codeberg.org/petrbalvin/goget/internal/transport"
)
@@ -982,9 +983,18 @@ func (p *Protocol) downloadRecursiveSequential(ctx context.Context, req *core.Do
}, nil
}
// webdavTask represents a unit of work for the concurrent recursive
// download: a file or subdirectory entry with its pre-built request.
type webdavTask struct {
entry WebDAVEntry
subReq *core.DownloadRequest
outPath string
isDir bool
}
// downloadRecursiveParallel is the worker-pool implementation used when
// req.RecursiveParallel > 1. Top-level entries are processed concurrently,
// bounded by a semaphore. Subdirectory recursion is delegated back to
// bounded by the pool. Subdirectory recursion is delegated back to
// downloadRecursive, so the same RecursiveParallel limit applies at every
// level of the tree. Stats are aggregated under a mutex.
func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
@@ -1021,24 +1031,52 @@ func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.Down
// these counters, so they reflect the entire subtree processed by
// this call.
var (
statsMu sync.Mutex
totalBytes int64
totalChunks int
processedDirs int
statsMu sync.Mutex
totalBytes int64
totalChunks int
)
processedDirs++
processedDirs := 1
// Pre-build sub-requests and output paths so the goroutine body has
// no per-entry allocation cost.
type task struct {
entry WebDAVEntry
subReq *core.DownloadRequest
outPath string
isDir bool
}
wp := pool.New(ctx, req.RecursiveParallel, func(taskCtx context.Context, t webdavTask) {
if taskCtx.Err() != nil {
return
}
if t.isDir {
subResult, err := p.downloadRecursive(taskCtx, t.subReq)
if err != nil {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to %s sub-directory %s: %v\n", dryRunVerb(req.DryRun), t.entry.URL.String(), err)
}
return
}
statsMu.Lock()
totalBytes += subResult.BytesDownloaded
totalChunks += subResult.ChunksCount
statsMu.Unlock()
return
}
subResult, err := p.downloadFile(taskCtx, t.subReq, t.entry.Size, t.entry.LastModified)
if err != nil {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to download file %s: %v\n", t.entry.URL.String(), err)
}
return
}
// Apply keep-timestamps if requested.
if req.KeepTimestamps {
setFileTimestamp(t.outPath, t.entry.LastModified)
}
statsMu.Lock()
totalBytes += subResult.BytesDownloaded
totalChunks++
statsMu.Unlock()
})
tasks := make([]task, 0, len(filteredEntries))
for _, entry := range filteredEntries {
// Respect MaxDepth to avoid infinite recursion.
if entry.IsDir && req.MaxDepth == 0 {
@@ -1065,7 +1103,7 @@ func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.Down
subReq.MaxDepth = req.MaxDepth - 1
subReq.RecursiveParallel = req.RecursiveParallel
subReq.DryRun = true
tasks = append(tasks, task{entry: entry, subReq: &subReq, outPath: itemOutputPath, isDir: true})
wp.Submit(webdavTask{entry: entry, subReq: &subReq, outPath: itemOutputPath, isDir: true})
} else {
fmt.Fprintf(os.Stderr, "[webdav] [dry-run] would download: %s (%d bytes)\n", entry.URL.String(), entry.Size)
statsMu.Lock()
@@ -1106,7 +1144,7 @@ func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.Down
RejectPatterns: req.RejectPatterns,
Ctx: req.Ctx,
}
tasks = append(tasks, task{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: true})
wp.Submit(webdavTask{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: true})
} else {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[webdav] Downloading file %s -> %s (depth=%d)\n", entry.URL.String(), itemOutputPath, req.MaxDepth)
@@ -1126,71 +1164,11 @@ func (p *Protocol) downloadRecursiveParallel(ctx context.Context, req *core.Down
MaxDepth: req.MaxDepth - 1,
Ctx: req.Ctx,
}
tasks = append(tasks, task{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: false})
wp.Submit(webdavTask{entry: entry, subReq: subReq, outPath: itemOutputPath, isDir: false})
}
}
// Worker pool: bounded by req.RecursiveParallel. We use a semaphore
// channel + WaitGroup so total in-flight workers never exceed the
// requested count, regardless of how many tasks are enqueued.
sem := make(chan struct{}, req.RecursiveParallel)
var wg sync.WaitGroup
for _, t := range tasks {
// Honour context cancellation before launching a new task.
if ctx.Err() != nil {
break
}
wg.Add(1)
sem <- struct{}{}
go func(t task) {
defer wg.Done()
defer func() { <-sem }()
// If the context was cancelled while we were waiting for
// the semaphore, just exit cleanly.
if ctx.Err() != nil {
return
}
if t.isDir {
subResult, err := p.downloadRecursive(ctx, t.subReq)
if err != nil {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to %s sub-directory %s: %v\n", dryRunVerb(req.DryRun), t.entry.URL.String(), err)
}
return
}
statsMu.Lock()
totalBytes += subResult.BytesDownloaded
totalChunks += subResult.ChunksCount
statsMu.Unlock()
return
}
subResult, err := p.downloadFile(ctx, t.subReq, t.entry.Size, t.entry.LastModified)
if err != nil {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[webdav] Warning: failed to download file %s: %v\n", t.entry.URL.String(), err)
}
return
}
// Apply keep-timestamps if requested.
if req.KeepTimestamps {
setFileTimestamp(t.outPath, t.entry.LastModified)
}
statsMu.Lock()
totalBytes += subResult.BytesDownloaded
totalChunks++
statsMu.Unlock()
}(t)
}
wg.Wait()
wp.Wait()
_ = processedDirs // reserved for future per-directory accounting