feat: add unified worker pool for recursive downloads

internal/protocol/ftp/client.go

@@ -16,10 +16,10 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"codeberg.org/petrbalvin/goget/internal/output"
+	"codeberg.org/petrbalvin/goget/internal/pool"
 )
 
 type Client struct {
@@ -698,6 +698,76 @@ func (p *clientPool) close() {
 	}
 }
 
+// ftpTask represents a unit of work for the concurrent recursive
+// download: either a directory to list or a file to download.
+type ftpTask struct {
+	remotePath string
+	localPath  string
+	isDir      bool
+	name       string
+}
+
+// ftpTaskHandler processes one ftpTask. For directories it lists the
+// contents and submits child tasks for each entry. For files it
+// acquires a client from the pool and downloads.
+func ftpTaskHandler(cpool *clientPool, progressFunc func(file string, current, total int64)) func(ctx context.Context, t ftpTask) {
+	return func(ctx context.Context, t ftpTask) {
+		if ctx.Err() != nil {
+			return
+		}
+
+		if err := os.MkdirAll(t.localPath, 0755); err != nil {
+			pool.Get[ftpTask](ctx).AddError(fmt.Errorf("failed to create local dir %s: %w", t.localPath, err))
+			return
+		}
+
+		if !t.isDir {
+			c := cpool.acquire()
+			defer cpool.release(c)
+			if progressFunc != nil {
+				progressFunc(t.name, 0, 0)
+			}
+			var fileProgress func(current, total int64)
+			if progressFunc != nil {
+				fileProgress = func(current, total int64) {
+					progressFunc(t.name, current, total)
+				}
+			}
+			if err := c.DownloadFile(ctx, t.remotePath, t.localPath, 0, fileProgress); err != nil {
+				pool.Get[ftpTask](ctx).AddError(fmt.Errorf("download %s: %w", t.remotePath, err))
+			}
+			return
+		}
+
+		c := cpool.acquire()
+		entries, err := c.ListRemoteDir(t.remotePath)
+		cpool.release(c)
+		if err != nil {
+			pool.Get[ftpTask](ctx).AddError(fmt.Errorf("list %s: %w", t.remotePath, err))
+			return
+		}
+
+		p := pool.Get[ftpTask](ctx)
+		for _, entry := range entries {
+			fields := strings.Fields(entry)
+			if len(fields) < 9 {
+				continue
+			}
+			name := fields[len(fields)-1]
+			if name == "." || name == ".." {
+				continue
+			}
+			childIsDir := fields[0][0] == 'd'
+			p.Submit(ftpTask{
+				remotePath: filepath.Join(t.remotePath, name),
+				localPath:  filepath.Join(t.localPath, name),
+				isDir:      childIsDir,
+				name:       name,
+			})
+		}
+	}
+}
+
 // DownloadDirectoryConcurrent downloads a remote directory to localPath
 // using a pool of N FTP connections. All operations (LIST, RETR) use
 // absolute paths so concurrent workers never share or mutate CWD state
@@ -708,110 +778,21 @@ func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout tim
 	if parallel < 1 {
 		parallel = 1
 	}
-	pool, err := newClientPool(rawURL, timeout, parallel)
+	cpool, err := newClientPool(rawURL, timeout, parallel)
 	if err != nil {
 		return err
 	}
-	defer pool.close()
+	defer cpool.close()
 
-	sem := make(chan struct{}, parallel)
-	var wg sync.WaitGroup
-	var firstErr error
-	var errMu sync.Mutex
+	p := pool.New(ctx, parallel, ftpTaskHandler(cpool, progressFunc))
+	p.Submit(ftpTask{
+		remotePath: remotePath,
+		localPath:  localPath,
+		isDir:      true,
+	})
 
-	setErr := func(err error) {
-		errMu.Lock()
-		if firstErr == nil {
-			firstErr = err
-		}
-		errMu.Unlock()
-	}
-
-	processEntry(ctx, pool, sem, &wg, setErr, remotePath, localPath, progressFunc)
-
-	wg.Wait()
-	return firstErr
-}
-
-// processEntry processes one remote path: lists it, then dispatches
-// its child entries (file → download, subdir → recursive processEntry)
-// through the shared semaphore-bounded goroutine pool. A single
-// sem is threaded through the recursion so the total in-flight
-// goroutines never exceed `parallel`, regardless of directory depth.
-func processEntry(ctx context.Context, pool *clientPool, sem chan struct{}, wg *sync.WaitGroup, setErr func(error), remotePath, localPath string, progressFunc func(file string, current, total int64)) {
-	defer wg.Done()
-	defer func() { <-sem }()
-
-	if ctx.Err() != nil {
-		return
-	}
-
-	if err := os.MkdirAll(localPath, 0755); err != nil {
-		setErr(fmt.Errorf("failed to create local dir %s: %w", localPath, err))
-		return
-	}
-
-	c := pool.acquire()
-	entries, err := c.ListRemoteDir(remotePath)
-	pool.release(c)
-	if err != nil {
-		setErr(fmt.Errorf("list %s: %w", remotePath, err))
-		return
-	}
-
-	for _, entry := range entries {
-		fields := strings.Fields(entry)
-		if len(fields) < 9 {
-			continue
-		}
-		name := fields[len(fields)-1]
-		if name == "." || name == ".." {
-			continue
-		}
-		isDir := fields[0][0] == 'd'
-		remoteFilePath := filepath.Join(remotePath, name)
-		localFilePath := filepath.Join(localPath, name)
-
-		if isDir {
-			wg.Add(1)
-			select {
-			case sem <- struct{}{}:
-				go processEntry(ctx, pool, sem, wg, setErr, remoteFilePath, localFilePath, progressFunc)
-			case <-ctx.Done():
-				return
-			}
-		} else {
-			wg.Add(1)
-			select {
-			case sem <- struct{}{}:
-				go func(remoteFilePath, localFilePath, name string) {
-					defer wg.Done()
-					defer func() { <-sem }()
-					if ctx.Err() != nil {
-						return
-					}
-					c := pool.acquire()
-					defer pool.release(c)
-					if progressFunc != nil {
-						progressFunc(name, 0, 0)
-					}
-					// DownloadFile's progress callback has a different
-					// signature than our directory-level progressFunc, so
-					// we wrap it here to keep the existing per-file
-					// progress behaviour.
-					var fileProgress func(current, total int64)
-					if progressFunc != nil {
-						fileProgress = func(current, total int64) {
-							progressFunc(name, current, total)
-						}
-					}
-					if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, fileProgress); err != nil {
-						setErr(fmt.Errorf("download %s: %w", remoteFilePath, err))
-					}
-				}(remoteFilePath, localFilePath, name)
-			case <-ctx.Done():
-				return
-			}
-		}
+	if errs := p.Wait(); len(errs) > 0 {
+		return errs[0]
 	}
+	return nil
 }