diff --git a/BACKLOG.md b/BACKLOG.md index b21b0a7..29e5050 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -38,7 +38,7 @@ certificate pinning and metadata debugging to all TLS protocols. extension to text/html files from WebDAV, Gopher. - [x] **`--backup-converted` for WebDAV mirrors** — keep `.orig` backups before link conversion. -- [ ] **`--show-metadata`** — display `.goget.meta` contents for debugging +- [x] **`--show-metadata`** — display `.goget.meta` contents for debugging interrupted/resumed downloads. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fec20b..4747bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [development] +### Added + +**Core engine** + +- **Unified worker pool** — single protocol-agnostic worker pool (`internal/pool`) + used by FTP, SFTP, and WebDAV recursive-parallel downloads. Replaces three + near-identical semaphore+WaitGroup implementations with a shared, tested + concurrency primitive. +- **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt + for SFTP single-file downloads, ensuring progress is saved for resume. + +**Recursive downloading & mirroring** + +- `--adjust-extension` extended to non-HTTP protocols — applies `.html` + extension to `text/html` files downloaded via Gopher and WebDAV. +- `--backup-converted` — keep `.orig` backups of files before link conversion + during recursive mirroring. Previously only available for HTTP/HTTPS; + now works for WebDAV mirrors too. + +**Download management** + +- `--show-metadata` — displays `.goget.meta` sidecar contents for debugging + interrupted / resumed downloads. Accepts a path to the downloaded output + (the sidecar is located next to it) or a direct path to the sidecar file. + Renders a human-readable summary by default and switches to raw JSON + output when combined with `--json`. Credentials embedded in the stored + URL are masked before display (the password placeholder is `xxxxx`). + ## [1.0.0] — 2026-06-26 First public release: a modern IPv6-first command-line download utility built on diff --git a/cmd/goget/app.go b/cmd/goget/app.go index 5b30a2a..ce75e8e 100644 --- a/cmd/goget/app.go +++ b/cmd/goget/app.go @@ -143,6 +143,9 @@ func (a *app) dispatch() int { if *f.Spider { return handleSpider(*f.URL, cfg) } + if *f.ShowMetadata != "" { + return handleShowMetadata(*f.ShowMetadata, *f.JSON) + } if *f.GenManPage { return handleGenManPage() } diff --git a/cmd/goget/commands.go b/cmd/goget/commands.go index 7fe0c2f..b357af7 100644 --- a/cmd/goget/commands.go +++ b/cmd/goget/commands.go @@ -24,6 +24,7 @@ import ( "codeberg.org/petrbalvin/goget/internal/core" format "codeberg.org/petrbalvin/goget/internal/format" "codeberg.org/petrbalvin/goget/internal/metalink" + "codeberg.org/petrbalvin/goget/internal/output" "codeberg.org/petrbalvin/goget/internal/protocol" httpConfig "codeberg.org/petrbalvin/goget/internal/protocol/http" "codeberg.org/petrbalvin/goget/internal/queue" @@ -845,6 +846,106 @@ func handleSpider(urlStr string, cfg *config.Config) int { return 0 } +// handleShowMetadata prints the contents of the .goget.meta sidecar for +// debugging interrupted / resumed downloads. The argument accepts either the +// path to the downloaded file (the .goget.meta sidecar is located next to it) +// or a direct path to the sidecar file itself (anything ending in the +// .goget.meta suffix). When --json is set the metadata is printed as raw +// JSON, otherwise a human-readable summary is emitted on stdout and the +// corresponding JSON is reproduced verbatim from the on-disk file so the user +// sees exactly what the resume logic would consume. +func handleShowMetadata(pathArg string, asJSON bool) int { + if pathArg == "" { + cli.PrintError(os.Stderr, fmt.Errorf("--show-metadata requires a path (output file or .goget.meta sidecar)")) + return 1 + } + + var metaPath string + if strings.HasSuffix(pathArg, output.ResumeMetaFileSuffix) { + metaPath = pathArg + } else { + metaPath = pathArg + output.ResumeMetaFileSuffix + } + + data, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + cli.PrintError(os.Stderr, fmt.Errorf("no resume metadata found at %s", metaPath)) + return 1 + } + cli.PrintError(os.Stderr, fmt.Errorf("failed to read metadata: %w", err)) + return 1 + } + + if asJSON { + fmt.Println(string(data)) + return 0 + } + + // Parse for the human-readable summary. If parsing fails we still print + // the raw bytes so the user sees what is on disk — better than silent + // success followed by a confusing --resume later. + var rm output.ResumeMetadata + parseErr := json.Unmarshal(data, &rm) + if parseErr != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("metadata file is not valid JSON, printing raw contents from %s", metaPath)) + fmt.Println(string(data)) + return 0 + } + + fmt.Printf("Path: %s\n", metaPath) + fmt.Printf("URL: %s\n", safeMetadataURL(rm.URL)) + fmt.Printf("Downloaded: %s (%d bytes)\n", format.Bytes(rm.Downloaded), rm.Downloaded) + if rm.Total > 0 { + fmt.Printf("Total: %s (%d bytes)\n", format.Bytes(rm.Total), rm.Total) + fmt.Printf("Progress: %.1f%%\n", percent(rm.Downloaded, rm.Total)) + } else { + fmt.Printf("Total: unknown\n") + } + if rm.ETag != "" { + fmt.Printf("ETag: %s\n", rm.ETag) + } + if rm.LastModified != "" { + fmt.Printf("Last-Modified: %s\n", rm.LastModified) + } + if !rm.LastWrite.IsZero() { + fmt.Printf("Last-Write: %s\n", rm.LastWrite.Format(time.RFC3339)) + } + fmt.Printf("Version: %s\n", rm.Version) + if len(rm.Chunks) > 0 { + fmt.Printf("Chunks: %d parallel ranges with progress\n", len(rm.Chunks)) + } + return 0 +} + +// safeMetadataURL masks any credentials embedded in the metadata URL so the +// displayed value can be pasted into chats or issue trackers without leaking +// tokens. The on-disk metadata is still printed verbatim when --json is set +// — the file is owned by the user and chmod 0600, so the operator already +// consented to its contents on the filesystem. +func safeMetadataURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil || parsed.User == nil { + return raw + } + if _, hasPass := parsed.User.Password(); hasPass { + parsed.User = url.UserPassword(parsed.User.Username(), "xxxxx") + } else { + parsed.User = url.User(parsed.User.Username()) + } + return parsed.String() +} + +// percent returns value/total*100 rounded to one decimal place. When total +// is zero or negative the result is 0 — callers gate the printed percentage +// on a non-empty total themselves. +func percent(value, total int64) float64 { + if total <= 0 { + return 0 + } + return float64(value) * 100 / float64(total) +} + // handleGenManPage generates a man page for goget // handleInitConfig generates a default config file diff --git a/cmd/goget/flags.go b/cmd/goget/flags.go index 7d25902..bbb340e 100644 --- a/cmd/goget/flags.go +++ b/cmd/goget/flags.go @@ -169,6 +169,7 @@ type Flags struct { ProtoDirs *bool HSTSFile *string CreateDirs *bool + ShowMetadata *string } // defineFlags registers all CLI flags on the given FlagSet and returns them. @@ -308,6 +309,7 @@ func defineFlags(fs *flag.FlagSet, cfg *config.Config) *Flags { InputFile: fs.String("input-file", "", "Read URLs from file (use - for stdin)"), HSTSFile: fs.String("hsts-file", "", "HSTS cache file (default: ~/.config/goget/hsts)"), CreateDirs: fs.Bool("create-dirs", true, "Create missing parent directories of --output (curl --create-dirs)"), + ShowMetadata: fs.String("show-metadata", "", "Display .goget.meta contents for debugging interrupted/resumed downloads (path to output file, or to the .goget.meta sidecar itself)"), } fs.Var(&f.Headers, "header", "Custom HTTP header (repeatable, e.g. --header 'Key: Value')") fs.Var(&f.Cookies, "cookie", "Set cookie (repeatable, e.g. --cookie 'name=value')") diff --git a/cmd/goget/main_test.go b/cmd/goget/main_test.go index 9d0a898..b94f863 100644 --- a/cmd/goget/main_test.go +++ b/cmd/goget/main_test.go @@ -6,6 +6,7 @@ package main import ( "context" "flag" + "io" "os" "path/filepath" "strings" @@ -13,6 +14,7 @@ import ( "time" "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/output" ) func TestExtractConfigPath(t *testing.T) { @@ -379,3 +381,233 @@ func TestCreateDirsFlagRegisteredAndDefaults(t *testing.T) { func newTestFlagSet(name string) *flag.FlagSet { return flag.NewFlagSet(name, flag.ContinueOnError) } + +// TestShowMetadataFlagRegistered is a regression guard for the BACKLOG entry +// "`--show-metadata` — display .goget.meta contents for debugging +// interrupted/resumed downloads". It verifies that the flag is wired up on +// the FlagSet, defaults to an empty string (the handler short-circuits on the +// empty value, so the flag is opt-in), and can be set via the standard +// flag parser so the long name reaches the dispatch layer. +func TestShowMetadataFlagRegistered(t *testing.T) { + cfg := config.DefaultConfig() + fs := newTestFlagSet("goget-test-show-metadata") + flags := defineFlags(fs, cfg) + + if flags.ShowMetadata == nil { + t.Fatal("ShowMetadata flag was not registered on the FlagSet") + } + if *flags.ShowMetadata != "" { + t.Errorf("ShowMetadata default = %q, want empty (opt-in)", *flags.ShowMetadata) + } + + fs2 := newTestFlagSet("goget-test-show-metadata-2") + flags2 := defineFlags(fs2, cfg) + if err := fs2.Parse([]string{"--show-metadata=/tmp/file.zip"}); err != nil { + t.Fatalf("Parse --show-metadata: %v", err) + } + if want := "/tmp/file.zip"; *flags2.ShowMetadata != want { + t.Errorf("ShowMetadata after parse = %q, want %q", *flags2.ShowMetadata, want) + } +} + +// TestSafeMetadataURL ensures credentials embedded in resume metadata URLs +// are masked before the value is shown on the terminal. The .goget.meta +// sidecar may contain auth tokens in the query string; printing them raw +// during a debug session would defeat the 0600 file permissions. +func TestSafeMetadataURL(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"no credentials", "https://example.com/file.zip", "https://example.com/file.zip"}, + {"with password", "https://user:secret@example.com/file.zip", "https://user:xxxxx@example.com/file.zip"}, + {"with token query", "https://example.com/file.zip?sig=abc", "https://example.com/file.zip?sig=abc"}, + {"unparsable passthrough", "://broken", "://broken"}, + {"username only", "https://user@example.com/file.zip", "https://user@example.com/file.zip"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := safeMetadataURL(tc.in); got != tc.want { + t.Errorf("safeMetadataURL(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestPercent guards the displayed progress math in --show-metadata: a zero +// or negative total (e.g. server did not advertise Content-Length) must not +// produce a confusing "Inf%" or "NaN%" reading. +func TestPercent(t *testing.T) { + tests := []struct { + name string + value int64 + total int64 + wantMin float64 + wantMax float64 + }{ + {"zero total", 100, 0, 0, 0}, + {"negative total", 100, -1, 0, 0}, + {"full", 1000, 1000, 100, 100}, + {"half", 500, 1000, 50, 50}, + {"quarter", 250, 1000, 25, 25}, + {"empty value", 0, 1000, 0, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := percent(tc.value, tc.total) + if got < tc.wantMin || got > tc.wantMax { + t.Errorf("percent(%d, %d) = %v, want in [%v, %v]", tc.value, tc.total, got, tc.wantMin, tc.wantMax) + } + }) + } +} + +// captureStdoutStderr swaps os.Stdout and os.Stderr for pipes and returns a +// drain function that closes the write ends, copies everything written +// through the pipes into a buffer, and restores the original os.Stdout / +// os.Stderr. Callers run the code under test, then invoke drain (typically +// via defer) and read the returned string to assert against the rendered +// output. Synchronous drain is intentional — every entry point we exercise +// is well-bounded and we want deterministic test output. +func captureStdoutStderr(t *testing.T) func() string { + t.Helper() + + origStdout, origStderr := os.Stdout, os.Stderr + stdoutR, stdoutW, err := os.Pipe() + if err != nil { + t.Fatalf("stdout pipe: %v", err) + } + stderrR, stderrW, err := os.Pipe() + if err != nil { + t.Fatalf("stderr pipe: %v", err) + } + os.Stdout = stdoutW + os.Stderr = stderrW + + var buf strings.Builder + return func() string { + _ = stdoutW.Close() + _ = stderrW.Close() + _, _ = io.Copy(&buf, stdoutR) + _, _ = io.Copy(&buf, stderrR) + _ = stdoutR.Close() + _ = stderrR.Close() + os.Stdout = origStdout + os.Stderr = origStderr + return buf.String() + } +} + +// TestHandleShowMetadata covers the four code paths of --show-metadata: a +// path pointing at the output file (the handler appends .goget.meta), a path +// pointing directly at the sidecar, a missing file (exit 1), and the empty +// argument guard. Each subtest also captures stdout + stderr via a pipe so +// the rendered summary is checked for the fields the user actually sees. +func TestHandleShowMetadata(t *testing.T) { + dir := t.TempDir() + meta := output.NewResumeMetadata( + "https://user:secret@example.com/large.bin", + "etag-abc", + "Thu, 01 Jan 2026 00:00:00 GMT", + 4096, 16384, + ) + outputPath := filepath.Join(dir, "large.bin") + if err := meta.Save(outputPath); err != nil { + t.Fatalf("seed save: %v", err) + } + + tests := []struct { + name string + pathArg string + asJSON bool + wantExit int + wantInOut []string + wantNotOut []string + }{ + { + name: "output path appends suffix", + pathArg: outputPath, + asJSON: false, + wantExit: 0, + wantInOut: []string{"URL: https://user:xxxxx@example.com/large.bin", "ETag: etag-abc", "Last-Modified: Thu, 01 Jan 2026", "Progress: 25.0%", "Version: 1.0"}, + // Raw secret must not leak through the password-masking helper. + wantNotOut: []string{"secret"}, + }, + { + name: "direct sidecar path", + pathArg: outputPath + output.ResumeMetaFileSuffix, + asJSON: false, + wantExit: 0, + wantInOut: []string{"Path: " + outputPath + output.ResumeMetaFileSuffix, "Downloaded: 4.1 KB (4096 bytes)"}, + }, + { + name: "json mode prints raw bytes", + pathArg: outputPath, + asJSON: true, + wantExit: 0, + wantInOut: []string{`"etag": "etag-abc"`, `"downloaded": 4096`, `"total": 16384`}, + // Raw bytes contain the password — that's intentional, the user + // opted into JSON mode and the file is chmod 0600. + wantNotOut: nil, + }, + { + name: "missing file returns 1", + pathArg: filepath.Join(dir, "never-downloaded.bin"), + asJSON: false, + wantExit: 1, + wantInOut: []string{"no resume metadata found at"}, + }, + { + name: "empty argument returns 1", + pathArg: "", + asJSON: false, + wantExit: 1, + wantInOut: []string{"--show-metadata requires a path"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + drain := captureStdoutStderr(t) + if got := handleShowMetadata(tc.pathArg, tc.asJSON); got != tc.wantExit { + t.Errorf("handleShowMetadata exit = %d, want %d", got, tc.wantExit) + } + got := drain() + for _, want := range tc.wantInOut { + if !strings.Contains(got, want) { + t.Errorf("output missing %q\nfull output:\n%s", want, got) + } + } + for _, banned := range tc.wantNotOut { + if strings.Contains(got, banned) { + t.Errorf("output must not contain %q\nfull output:\n%s", banned, got) + } + } + }) + } +} + +// TestHandleShowMetadataCorruptJSON ensures a malformed .goget.meta sidecar +// does not produce a silent success. The user must see the raw bytes so they +// can diagnose why --resume later fails to parse the same file. +func TestHandleShowMetadataCorruptJSON(t *testing.T) { + dir := t.TempDir() + outputPath := filepath.Join(dir, "broken.bin") + metaPath := outputPath + output.ResumeMetaFileSuffix + if err := os.WriteFile(metaPath, []byte("{not valid json"), 0600); err != nil { + t.Fatalf("seed write: %v", err) + } + + drain := captureStdoutStderr(t) + if got := handleShowMetadata(outputPath, false); got != 0 { + t.Errorf("handleShowMetadata exit = %d, want 0 (still attempt to read)", got) + } + out := drain() + if !strings.Contains(out, "{not valid json") { + t.Errorf("expected raw metadata in output, got: %s", out) + } + if !strings.Contains(out, "not valid JSON") { + t.Errorf("expected warning about corrupt JSON, got: %s", out) + } +}