diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fefbc16..e9a5727 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,46 +119,50 @@ chore: update golang.org/x/crypto ``` goget/ ├── cmd/ -│ └── goget/ # CLI entry point +│ └── goget/ # CLI entry point │ ├── main.go # Core run loop, flag setup, orchestrator +│ ├── app.go # Application wiring │ ├── commands.go # Command handlers (download, upload, queue, config, etc.) │ ├── flags.go # CLI flag definitions (struct + defineFlags) │ ├── completion.go # Shell completion generators (bash, zsh, fish) -│ ├── man_page.go # Man page generation +│ ├── man_page.go # Embedded manpage source and --generate-man-page +│ ├── setup.go # One-time CLI setup helpers │ ├── schedule.go # Cron and datetime scheduling │ └── verify.go # Checksum and PGP verification ├── internal/ -│ ├── archive/ # Archive extraction (tar, targz, tarbz2, zip) -│ ├── auth/ # Authentication (basic, digest, OAuth 2.0, netrc) -│ ├── cli/ # CLI utilities (help text, progress bar, color output) -│ ├── compression/ # Decompression (gzip, bzip2, zlib, flate, lzw) -│ ├── config/ # TOML config loading, saving, and merging -│ ├── cookie/ # Netscape-format cookie jar -│ ├── core/ # Core types (DownloadRequest, Result, Protocol interface, errors) -│ ├── crypto/ # Checksum verification, PGP, certificate handling -│ ├── format/ # Human-readable formatting (bytes, speed) -│ ├── hsts/ # HSTS cache (RFC 6797) -│ ├── log/ # Structured logger (text/JSON output, level filtering) -│ ├── linkrewrite/ # HTML link rewriting for offline mirroring -│ ├── metalink/ # Metalink (multi-source downloads) -│ ├── mirror/ # Website mirroring logic -│ ├── output/ # Output writing (atomic writes, resume metadata) -│ ├── protocol/ # Protocol abstraction layer -│ │ ├── http/ # HTTP/HTTPS client, parallel download, upload -│ │ ├── ftp/ # FTP/FTPS client -│ │ ├── sftp/ # SFTP client -│ │ ├── file/ # Local file:// protocol -│ │ ├── dataurl/ # data: URI protocol -│ │ ├── gopher/ # Gopher protocol -│ │ ├── gemini/ # Gemini protocol -│ │ └── webdav/ # WebDAV protocol -│ ├── queue/ # Download queue management -│ ├── recursive/ # Recursive download (HTML parser, URL filter, crawler) -│ ├── warc/ # WARC archiving writer -│ └── transport/ # HTTP transport (dialer, proxy, TLS, retry, rate limiting) +│ ├── archive/ # Archive extraction (tar, targz, tarbz2, zip) +│ ├── auth/ # Authentication (basic, digest, OAuth 2.0, netrc) +│ ├── cli/ # CLI utilities (help text, progress bar, color output) +│ ├── compression/ # Decompression (gzip, bzip2, zlib, flate, lzw) +│ ├── config/ # TOML config loading, saving, and merging +│ ├── cookie/ # Netscape-format cookie jar +│ ├── core/ # Core types (DownloadRequest, Result, Protocol interface, errors) +│ ├── crypto/ # Checksum verification, PGP, certificate handling +│ ├── format/ # Human-readable formatting (bytes, speed) +│ ├── hsts/ # HSTS cache (RFC 6797) +│ ├── linkrewrite/ # HTML link rewriting for offline mirroring +│ ├── log/ # Structured logger (text/JSON output, level filtering) +│ ├── metalink/ # Metalink (multi-source downloads) +│ ├── mirror/ # Website mirroring logic +│ ├── output/ # Output writing (atomic writes, resume metadata) +│ ├── pool/ # Unified recursive worker pool (FTP, SFTP, WebDAV) +│ ├── protocol/ # Protocol abstraction layer +│ │ ├── http/ # HTTP/HTTPS client, parallel download, upload +│ │ ├── ftp/ # FTP/FTPS client +│ │ ├── sftp/ # SFTP client +│ │ ├── file/ # Local file:// protocol +│ │ ├── dataurl/ # data: URI protocol +│ │ ├── gopher/ # Gopher protocol +│ │ ├── gemini/ # Gemini protocol +│ │ ├── webdav/ # WebDAV protocol +│ │ └── register/ # Single registration entry point +│ ├── queue/ # Download queue management +│ ├── recursive/ # Recursive download (HTML parser, URL filter, crawler) +│ ├── transport/ # HTTP transport (dialer, proxy, TLS, retry, rate limiting) +│ └── warc/ # WARC archiving writer ├── pkg/ -│ └── api/ # Public library API (Client, Download, Upload, Queue) -└── docs/ # Architecture and design documentation +│ └── api/ # Public library API (Client, Download, Upload, Queue) +└── docs/ # Architecture and design documentation ``` ## Architecture Overview @@ -189,11 +193,14 @@ graph TD Metalink[internal/metalink/] QueueInt[internal/queue/] Archive[internal/archive/] + Pool[internal/pool/ - Unified worker pool] + Log[internal/log/ - Structured logger] API[pkg/api/ - Public Library] CLI --> Flags CLI --> Config CLI --> Core + CLI --> Log CLI --> Auth CLI --> Crypto CLI --> Recursive @@ -214,6 +221,9 @@ graph TD HTTP --> Cookie HTTP --> HSTS HTTP --> Output + FTP --> Pool + SFTP --> Pool + WebDAV --> Pool API --> Core API --> Registry ``` diff --git a/README.md b/README.md index a390a97..e6f9c0a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,14 @@ goget --rate-limit 1MB/s --url https://example.com/file.zip # Proxy goget --proxy http://proxy.example.com:8080 --url https://example.com/file.zip goget --proxy socks5h://127.0.0.1:1080 --url https://example.com/file.zip + +# Pin server certificate (applies to HTTPS, FTPS, Gemini, WebDAVS) +goget --pinned-cert a1b2c3d4e5f6... --url https://example.com/file +goget --pinned-cert a1b2c3d4e5f6... --url gemini://gemini.example.com/ + +# Pin SSH host key for SFTP (OpenSSH fingerprint from `ssh-keygen -lf`) +goget --pinned-host-key "SHA256:AAAAC3NzaC1lZDI1NTE5AAAAIE..." \ + --url sftp://example.com/file ``` ### Recursive & Mirroring @@ -178,6 +186,10 @@ goget --on-complete "process.sh" --url https://example.com/data.csv # JSON progress for scripting goget --json --no-progress --url https://example.com/file.zip +# Inspect a partially-downloaded transfer's resume metadata +goget --show-metadata ./file.zip +goget --show-metadata ./file.zip.goget.meta --json + # Shell completion goget --completion bash > /etc/bash_completion.d/goget goget --completion zsh > /usr/share/zsh/site-functions/_goget @@ -293,6 +305,8 @@ just uninstall-man# remove installed manpage - [`docs/download-pipeline.md`](docs/download-pipeline.md) — download flow - [`docs/authentication.md`](docs/authentication.md) — auth mechanisms - [`docs/security.md`](docs/security.md) — threat model and controls +- [`docs/migration.md`](docs/migration.md) — curl/wget → goget flag mapping +- [`docs/development.md`](docs/development.md) — dev workflow, releases, CI ## Contributing diff --git a/docs/architecture.md b/docs/architecture.md index 5ce67ef..e5db402 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -6,39 +6,40 @@ ```mermaid graph TD - User[User / Terminal]:::accent6 - Script[Script / CI]:::accent6 - CLI[cmd/goget/ - Main entry]:::accent0 - API[pkg/api/ - Library interface]:::accent3 - Core[internal/core/ - Types, Protocol Interface, Errors]:::accent1 - Registry[internal/protocol/ - Protocol Registry]:::accent1 - HTTP[internal/protocol/http/]:::accent1 - FTP[internal/protocol/ftp/]:::accent1 - SFTP[internal/protocol/sftp/]:::accent1 - FILE[internal/protocol/file/]:::accent1 - DataURL[internal/protocol/dataurl/]:::accent1 - Gopher[internal/protocol/gopher/]:::accent1 - Gemini[internal/protocol/gemini/]:::accent1 - WebDAV[internal/protocol/webdav/]:::accent1 - Transport[internal/transport/ - Dialer, Proxy, TLS, Rate Limit]:::accent7 - Config[internal/config/]:::accent7 - Output[internal/output/ - Atomic write, Resume, WARC]:::accent2 - Auth[internal/auth/ - Basic, Digest, OAuth, netrc]:::accent7 - Crypto[internal/crypto/ - Checksum, PGP, Certs]:::accent7 - Cookie[internal/cookie/ - Netscape jar]:::accent7 - HSTS[internal/hsts/ - RFC 6797 cache]:::accent7 - Archive[internal/archive/ - Extraction]:::accent7 - Recursive[internal/recursive/ - Crawler, Parser, Filter]:::accent2 - Mirror[internal/mirror/]:::accent2 - Metalink[internal/metalink/]:::accent2 - Queue[internal/queue/]:::accent2 - Compression[internal/compression/ - gzip, bzip2, zlib, flate, lzw]:::accent7 - Log[internal/log/ - Structured logging]:::accent7 - WARC[internal/warc/ - Archiving]:::accent7 - Register[internal/protocol/register/ - Single registration]:::accent7 - Format[internal/format/ - Bytes, Speed formatting]:::accent7 - LinkRewrite[internal/linkrewrite/ - HTML link conversion]:::accent7 - CLIUtil[internal/cli/ - Help, Progress bar, Colors]:::accent7 + User[User / Terminal] + Script[Script / CI] + CLI[cmd/goget/ - Main entry] + API[pkg/api/ - Library interface] + Core[internal/core/ - Types, Protocol Interface, Errors] + Registry[internal/protocol/ - Protocol Registry] + HTTP[internal/protocol/http/] + FTP[internal/protocol/ftp/] + SFTP[internal/protocol/sftp/] + FILE[internal/protocol/file/] + DataURL[internal/protocol/dataurl/] + Gopher[internal/protocol/gopher/] + Gemini[internal/protocol/gemini/] + WebDAV[internal/protocol/webdav/] + Transport[internal/transport/ - Dialer, Proxy, TLS, Rate Limit] + Config[internal/config/] + Output[internal/output/ - Atomic write, Resume, WARC] + Auth[internal/auth/ - Basic, Digest, OAuth, netrc] + Crypto[internal/crypto/ - Checksum, PGP, Certs] + Cookie[internal/cookie/ - Netscape jar] + HSTS[internal/hsts/ - RFC 6797 cache] + Archive[internal/archive/ - Extraction] + Recursive[internal/recursive/ - Crawler, Parser, Filter] + Mirror[internal/mirror/] + Metalink[internal/metalink/] + Queue[internal/queue/] + Compression[internal/compression/ - gzip, bzip2, zlib, flate, lzw] + Log[internal/log/ - Structured logging] + WARC[internal/warc/ - Archiving] + Register[internal/protocol/register/ - Single registration] + Format[internal/format/ - Bytes, Speed formatting] + LinkRewrite[internal/linkrewrite/ - HTML link conversion] + CLIUtil[internal/cli/ - Help, Progress bar, Colors] + Pool[internal/pool/ - Unified worker pool] User --> CLI Script --> API @@ -65,7 +66,10 @@ graph TD HTTP --> HSTS HTTP --> Output FTP --> Transport + FTP --> Pool SFTP --> Transport + SFTP --> Pool + WebDAV --> Pool end subgraph "Post-Download" @@ -88,13 +92,6 @@ graph TD Transport --> Auth Transport --> Config - - classDef accent0 fill:#3B82F6,stroke:#2563EB,color:#fff - classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff - classDef accent2 fill:#F59E0B,stroke:#D97706,color:#000 - classDef accent3 fill:#A855F7,stroke:#7C3AED,color:#fff - classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff - classDef accent7 fill:#64748B,stroke:#475569,color:#fff ``` ## Layers @@ -168,6 +165,7 @@ Protocols register themselves by scheme in a global `Registry`. Scheme normaliza | `log` | Structured logger with text and JSON output modes, level filtering, ANSI colors | | `warc` | WARC (Web ARChive) format writing for legal and archival compliance | | `cli` | ANSI color output, progress bar, help text, completion | +| `pool` | Protocol-agnostic recursive worker pool — bounded goroutine execution with recursive task submission. Replaces per-protocol semaphore+WaitGroup code in FTP, SFTP, and WebDAV recursive-parallel downloads | ## Download Flow diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 70b6f64..f47feb6 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -22,6 +22,7 @@ goget --url [OPTIONS] | `--content-disposition` | — | Use filename from server's `Content-Disposition` header | | `--create-dirs` | — | Create missing parent directories of `--output` (default: true). Use `--create-dirs=false` to disable (curl `--create-dirs` / wget behavior). Ignored for stdout output | | `--keep-timestamps` | — | Set file modification time to server's `Last-Modified` value | +| `--show-metadata` | `` | Display `.goget.meta` sidecar contents for debugging interrupted or resumed downloads. `` is either the downloaded output file (sidecar is located next to it) or the sidecar file itself. Renders a human-readable summary; combine with `--json` for raw JSON. Credentials embedded in the stored URL are masked | ## Progress @@ -77,7 +78,7 @@ goget --url [OPTIONS] |---|---|---| | `--accept` | `"*.pdf"` | Accept patterns — filename glob or MIME type (comma-separated) | | `--no-parent` | — | Do not ascend to parent directories during recursive download | -| `--adjust-extension` | — | Append `.html` extension to text/html files without one | +| `--adjust-extension` | — | Append `.html` extension to `text/html` files missing one. Applies across HTTP, Gopher, and WebDAV | | `--wait` | `` | Delay between requests in recursive mode | | `--random-wait` | — | Randomize wait time (0.5x–1.5x of `--wait` value) | | `--page-requisites` | — | Download CSS, JS, and images required to render HTML pages | @@ -105,9 +106,9 @@ goget --url [OPTIONS] | `--reject` | `` | Reject URL patterns (glob) | | `--cut-dirs` | `` | Ignore N directory components when creating local filenames | | `--proto-dirs` | — | Create protocol-prefixed directories (http/, ftp/ etc.) | -| `--adjust-extension` | — | Append `.html` to text/html files missing an extension | +| `--adjust-extension` | — | Append `.html` to `text/html` files missing an extension. Applies across HTTP, Gopher, and WebDAV | | `--timestamping` | — | Only download files newer than the local copy | -| `--backup-converted` | — | Back up original `.orig` files before link conversion | +| `--backup-converted` | — | Back up original `.orig` files before link conversion. Applies to HTTP and WebDAV mirrors | | `--recursive-parallel` | `` | Number of concurrent file downloads in recursive mode across all protocols (HTTP, WebDAV, FTP, SFTP). `0` = sequential (default) | | `--continue` | — | Continue getting partially-downloaded file | diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..ceeeee4 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,253 @@ +# Development + +This guide covers the day-to-day workflow for working on **goget** — building, +testing, releasing, and contributing. + +## Requirements + +- **Go 1.26+** +- **Platform:** Linux (amd64, arm64, riscv64, loong64) or + FreeBSD (amd64, arm64, riscv64) +- **`just`** — task runner (`brew install just` / `apt install just` / + `pacman -S just`) +- **`groff`** *(optional)* — for `just man` manpage validation + +All Go source files carry the build constraint `//go:build linux || freebsd`. +goget does not build or run on macOS or Windows. + +## First-time Setup + +```bash +git clone https://codeberg.org/petrbalvin/goget.git +cd goget +just install # go mod download +just test # verify everything works on your machine +just build # produce bin/goget +``` + +A successful `just build` produces a static `bin/goget` binary with no +runtime dependencies beyond the Go standard library and `golang.org/x/` +packages. + +## Task Reference + +Every task is defined in [`justfile`](../justfile) and is the canonical +way to interact with the project. + +| Task | What it does | +|------|--------------| +| `just` | List all recipes (default) | +| `just install` | `go mod download` | +| `just run` | `go run ./cmd/goget` (development loop) | +| `just build` | `gofmt -l` check + `go vet ./...` + build `bin/goget` | +| `just test` | `go test -race -count=1 ./...` | +| `just uninstall` | Remove `bin/` and `coverage.out` | +| `just man` | Validate `man/goget.1` with `groff` (skips if absent) | +| `just install-man` | Install manpage to `~/.local/share/man/man1/` (gzipped, XDG) | +| `just uninstall-man` | Remove installed manpage | + +`just build` is **strict**: it runs `gofmt -l` (any output is a failure) +plus `go vet ./...`. Both must report zero warnings before the binary +compiles. This is exactly what CI checks; running it locally first +saves a round-trip. + +## Development Loop + +```mermaid +graph TD + Edit[Edit source in cmd/, internal/, pkg/] + Test[just test] + Vet[just build runs gofmt + vet + build] + Man[just man - groff validation] + Commit[Conventional Commits message] + Push[Push to development branch] + CI[Forgejo CI: test.yml] + + Edit --> Test + Test -->|fail| Edit + Test --> Vet + Vet -->|fail| Edit + Vet --> Man + Man -->|fail| Edit + Man --> Commit + Commit --> Push + Push --> CI + CI -->|fail| Edit + CI -->|pass| Merge[Merge / open PR] +``` + +Typical iteration: + +```bash +# Make a focused change. +$EDITOR cmd/goget/flags.go + +# Quick check that compile + format + vet pass. +just build + +# Run the relevant tests with the race detector. +just test + +# Validate any manpage edits. +just man +``` + +## Building from Source + +`just build` does three things in order: + +1. Copies `man/goget.1` into `cmd/goget/man-page-data/` — the manpage is + embedded into the binary via `//go:embed`, and `//go:embed` can only + reference files within the embedding package's directory. +2. Runs `gofmt -l` over `cmd/`, `internal/`, and `pkg/`. Any output is a + formatting violation that breaks the build. +3. Runs `go vet ./...` — must report zero warnings. +4. Compiles `bin/goget` with `-ldflags "-s -w"` (strips DWARF). + +The resulting binary is fully static. `ldd bin/goget` should report +`not a dynamic executable` on Linux. + +## Testing + +```bash +just test # full suite, race detector, count=1 +go test ./internal/protocol/gemini/... # focused +go test -run TestFoo ./internal/core/ # one test +``` + +Conventions: + +- Tests live next to the code as `*_test.go`. +- Use table-driven tests for parsers, validators, and protocol handlers. +- Cover both happy path and error path. +- Test names follow `TestFunctionName_Scenario`. +- Fuzz tests use `testing.F` and live next to the code they exercise. + +The race detector is **always** on in CI. New code that introduces a +data race must be fixed, not silenced. + +## Manpage Authoring + +The single source of truth is `man/goget.1` (troff). Editing this file is +what most users eventually read via `man goget` and what +`--generate-man-page` prints. + +Workflow: + +```bash +$EDITOR man/goget.1 +just man # validates the troff +just build # copies the file into cmd/goget/man-page-data/ +just test # cmd/goget/man_page_test.go asserts every CLI flag + # is documented and the embedded source still parses +``` + +`cmd/goget/man_page_test.go` enforces documentation coverage — adding a +new flag to `defineFlags` without documenting it in `man/goget.1` breaks +the build. + +## Logging + +The project uses `internal/log` for structured output. Prefer `log.Infof`, +`log.Errorf`, etc. over ad-hoc `fmt.Fprintf(os.Stderr, ...)` or +`cli.Print*`. The backlog tracks a migration of the remaining ~140 legacy +callsites to `internal/log`. + +## Branching & Releases + +```mermaid +graph LR + Feature[Feature branches] + Development[development] + Main[main] + Tag[vX.Y.Z tag] + Release[Forgejo release] + + Feature --> Development + Development --> Main + Main --> Tag + Tag --> Release +``` + +- **`development`** — the only branch that receives day-to-day work. + All pull requests target `development`. +- **`main`** — only changes when `development` is merged for a release. + Never commit directly to `main`. +- **Tags** — `vX.Y.Z` tags trigger the release workflow. + +### Cutting a Release + +1. Bump `Version` in [`internal/core/version.go`](../internal/core/version.go). +2. Move the top `## [development]` section in + [`CHANGELOG.md`](../CHANGELOG.md) to `## [X.Y.Z] — YYYY-MM-DD`, + inserting a fresh empty `## [development]` section above it. +3. Ensure CI is green on `development`. +4. Merge `development` → `main`. +5. Tag and push: + ```bash + git tag -a vX.Y.Z -m "goget vX.Y.Z" + git push origin main + git push origin vX.Y.Z + ``` +6. The release workflow builds binaries for all seven platform targets, + extracts the matching `CHANGELOG.md` section, and publishes the + release on Codeberg. + +## CI + +Two Forgejo Actions workflows live under +[`.forgejo/workflows/`](../.forgejo/workflows): + +### `test.yml` + +Triggered on every push to `development` and on every pull request +targeting `development`. + +| Job | What it does | +|-----|--------------| +| `vet` | `gofmt -l` check + `go vet ./...` (no output = pass) | +| `test` | `go test -race -count=1 ./...` + coverage gate | + +Run the same locally before opening a PR: + +```bash +go vet ./... +just test +``` + +### `release.yml` + +Triggered by pushing a `v*` tag. Verifies that the tag matches +`internal/core.Version`, runs the quality gate, builds static binaries +for all seven targets, and publishes the release. + +Requires a `FORGEJO_TOKEN` secret with release-creation permissions on +the runner. + +## Code Style + +- Follow [Effective Go](https://go.dev/doc/effective_go). +- Keep lines under 100 characters. +- `camelCase` for locals, `PascalCase` for exported symbols. +- Prefix every file with a one-line comment describing its purpose. +- Use `//go:build linux || freebsd` on every source file. +- Use long CLI flags only (`--header`, not `-H`). +- Stick to the Go standard library and `golang.org/x/`. External + dependencies require an explicit justification. +- Error messages: English, lowercase, no trailing period, always with + context (e.g. `open config: permission denied`). +- Run `gofmt` before committing. + +## Reporting Bugs + +Open an issue at + with: + +- `goget --version` output. +- Operating system and architecture. +- The full command used and its output. +- Expected vs actual behaviour. + +For **security issues**, email +[opensource@petrbalvin.org](mailto:opensource@petrbalvin.org) instead of +opening a public issue. diff --git a/docs/download-pipeline.md b/docs/download-pipeline.md index 71fcfca..d25f40a 100644 --- a/docs/download-pipeline.md +++ b/docs/download-pipeline.md @@ -6,33 +6,33 @@ This document explains how goget processes a download request from start to fini ```mermaid flowchart TD - Start([User invokes goget]):::accent0 - ParseFlags["Parse CLI flags"]:::accent7 - LoadConfig["Load config + merge overrides"]:::accent7 - ResolveProtocol["Resolve protocol handler\nfrom URL scheme"]:::accent1 - CreateRequest["Build core.DownloadRequest"]:::accent1 - CheckResume{"Resume metadata\nexists?"}:::accent2 - LoadResume["Load .goget.meta\n(resume info + chunk map)"]:::accent2 - SetupOutput["Setup output writer\n(atomic temp file + resume)"]:::accent2 - CreateDirs{"--create-dirs\n(parent missing)?"}:::accent2 - MkdirAll["os.MkdirAll → create\nparent directories"]:::accent2 - CheckParallel{"File > 100 MB\nor --parallel set?"}:::accent1 - SequentialDownload["Sequential download\n(single connection)"]:::accent1 - ParallelDownload["Parallel chunked download\n(multiple Range requests)"]:::accent1 - Decompress{"Auto-decompress\nenabled?"}:::accent7 - Decompression["Decompress response\n(gzip, deflate, bzip2, zlib)"]:::accent7 - VerifyChecksum{"Checksum\nspecified?"}:::accent7 - ChecksumVerify["Verify checksum\n(SHA-256, SHA-512, etc.)"]:::accent7 - PGPVerify{"PGP verify/\ndecrypt?"}:::accent7 - PGPProcess["Verify signature /\ndecrypt file"]:::accent7 - SaveHSTS["Save HSTS cache\n(RFC 6797)"]:::accent7 - AtomicRename["Atomic rename\ntemp → final"]:::accent2 - SigInt{"SIGINT received?"}:::accent4 - SaveSidecar["Save .goget.meta\nfor resume"]:::accent2 - Exit130["Exit 130"]:::accent4 - RunHook{"--on-complete\nset?"}:::accent7 - PostHook["Run post-download command\n(GOGET_OUTPUT, GOGET_SIZE, GOGET_URL)"]:::accent7 - Done([Done]):::accent0 + Start([User invokes goget]) + ParseFlags["Parse CLI flags"] + LoadConfig["Load config + merge overrides"] + ResolveProtocol["Resolve protocol handler\nfrom URL scheme"] + CreateRequest["Build core.DownloadRequest"] + CheckResume{"Resume metadata\nexists?"} + LoadResume["Load .goget.meta\n(resume info + chunk map)"] + SetupOutput["Setup output writer\n(atomic temp file + resume)"] + CreateDirs{"--create-dirs\n(parent missing)?"} + MkdirAll["os.MkdirAll → create\nparent directories"] + CheckParallel{"File > 100 MB\nor --parallel set?"} + SequentialDownload["Sequential download\n(single connection)"] + ParallelDownload["Parallel chunked download\n(multiple Range requests)"] + Decompress{"Auto-decompress\nenabled?"} + Decompression["Decompress response\n(gzip, deflate, bzip2, zlib)"] + VerifyChecksum{"Checksum\nspecified?"} + ChecksumVerify["Verify checksum\n(SHA-256, SHA-512, etc.)"] + PGPVerify{"PGP verify/\ndecrypt?"} + PGPProcess["Verify signature /\ndecrypt file"] + SaveHSTS["Save HSTS cache\n(RFC 6797)"] + AtomicRename["Atomic rename\ntemp → final"] + SigInt{"SIGINT received?"} + SaveSidecar["Save .goget.meta\nfor resume"] + Exit130["Exit 130"] + RunHook{"--on-complete\nset?"} + PostHook["Run post-download command\n(GOGET_OUTPUT, GOGET_SIZE, GOGET_URL)"] + Done([Done]) Start --> ParseFlags --> LoadConfig --> ResolveProtocol ResolveProtocol --> CreateRequest --> CheckResume @@ -58,12 +58,6 @@ flowchart TD SigInt -->|No| RunHook RunHook -->|Yes| PostHook --> Done RunHook -->|No| Done - - classDef accent0 fill:#3B82F6,stroke:#2563EB,color:#fff - classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff - classDef accent2 fill:#F59E0B,stroke:#D97706,color:#000 - classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff - classDef accent7 fill:#64748B,stroke:#475569,color:#fff ``` ## Phase 1: Request Construction @@ -94,17 +88,13 @@ The protocol registry resolves the handler by URL scheme: ```mermaid flowchart LR - URL["https://example.com"]:::accent6 - Parse["Parse scheme\n→ https"]:::accent7 - Normalized["Normalize\nhttps → http"]:::accent7 - Registry["Registry lookup\nprotocols[http]"]:::accent1 - Handler["HTTP protocol\nhandler"]:::accent1 + URL["https://example.com"] + Parse["Parse scheme\n→ https"] + Normalized["Normalize\nhttps → http"] + Registry["Registry lookup\nprotocols[http]"] + Handler["HTTP protocol\nhandler"] URL --> Parse --> Normalized --> Registry --> Handler - - classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff - classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff - classDef accent7 fill:#64748B,stroke:#475569,color:#fff ``` Scheme normalization maps: diff --git a/docs/protocols.md b/docs/protocols.md index 3f0a107..6e0c71c 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -11,10 +11,10 @@ goget supports 9 network protocols through a unified abstraction layer. Each pro | `sftp://` | SFTP | ✓ | ✓ | — | ✓ | `golang.org/x/crypto/ssh` | | `file://` | Local file | — | — | — | ✓ | Go stdlib `os` | | `data:` | Data URI | — | — | — | — | Go stdlib | -| `gopher://` | Gopher | — | — | — | — | Custom implementation | +| `gopher://` | Gopher | ✓ | — | — | — | Custom implementation | | `webdav://` `webdavs://` | WebDAV | ✓ | — | ✓ | ✓ | HTTP (via delegation) | -| `gemini://` | Gemini | — | — | — | — | Custom implementation | +| `gemini://` | Gemini | ✓ | — | — | — | Custom implementation | ## HTTP/HTTPS (`http://`, `https://`) @@ -182,10 +182,14 @@ Implements the Gopher protocol as specified in RFC 1436. - **Type parsing** — Respects Gopher type codes (0 = text, 1 = directory, 9 = binary) - **Directory listing** — For recursive operations - **Text and binary downloads** +- **Resume** — `.goget.meta` sidecar on interrupt for `typeHTML`, + `typeTextFile`, and `typeInformation` responses; `--resume` continues + interrupted Gopher transfers (skips already-written output bytes + — post-transform length because Gopher line metadata is stripped + before writing) ### Limitations -- No resume - No parallel downloads - No encryption (Gopher has no TLS) - No upload @@ -196,6 +200,9 @@ Implements the Gopher protocol as specified in RFC 1436. # Download a file from a Gopher server goget --url gopher://gopher.example.com/9/file.zip +# Resume an interrupted Gopher text download +goget --url gopher://gopher.example.com/0/article --output article.txt --resume + # Browse a Gopher directory goget --url gopher://gopher.example.com/1/ ``` @@ -235,21 +242,16 @@ goget --url gemini://gemini.example.com/ --pinned-cert a1b2c3d4e5f6... ```mermaid flowchart TD - URL[URL input]:::accent6 - Parse["Parse URL"]:::accent7 - Normalize["Normalize scheme\n(https→http, ftps→ftp)"]:::accent7 - Registry["Look up in protocol registry"]:::accent1 - Found{"Protocol found?"}:::accent4 - Error["Unsupported protocol error"]:::accent4 - Handle["Delegate to protocol handler"]:::accent1 + URL[URL input] + Parse["Parse URL"] + Normalize["Normalize scheme\n(https→http, ftps→ftp)"] + Registry["Look up in protocol registry"] + Found{"Protocol found?"} + Error["Unsupported protocol error"] + Handle["Delegate to protocol handler"] URL --> Parse --> Normalize --> Registry Registry --> Found Found -->|Yes| Handle Found -->|No| Error - - classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff - classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff - classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff - classDef accent7 fill:#64748B,stroke:#475569,color:#fff ``` diff --git a/docs/recursive-mirror.md b/docs/recursive-mirror.md index 8fc03e5..4c8d81f 100644 --- a/docs/recursive-mirror.md +++ b/docs/recursive-mirror.md @@ -31,15 +31,15 @@ goget --recursive --accept "image/*,text/html" --url https://example.com/ ```mermaid flowchart TD - HTML[Parse HTML page]:::accent1 - Extract["Extract all links\n(a, img, link, script)"]:::accent1 - FilterByDomain{"Domain match?"}:::accent7 - FilterByPattern{"Accept pattern\nmatch?"}:::accent7 - FilterByDepth{"Depth ≤ max?"}:::accent7 - FilterByParent{"No-parent\ncheck?"}:::accent7 - ExcludeFilter{"Not in exclude\nlist?"}:::accent7 - Enqueue["Add to download queue"]:::accent1 - Skip["Skip"]:::accent4 + HTML[Parse HTML page] + Extract["Extract all links\n(a, img, link, script)"] + FilterByDomain{"Domain match?"} + FilterByPattern{"Accept pattern\nmatch?"} + FilterByDepth{"Depth ≤ max?"} + FilterByParent{"No-parent\ncheck?"} + ExcludeFilter{"Not in exclude\nlist?"} + Enqueue["Add to download queue"] + Skip["Skip"] HTML --> Extract --> FilterByDomain FilterByDomain -->|Yes| FilterByPattern @@ -53,10 +53,6 @@ flowchart TD FilterByParent -->|Fail| Skip ExcludeFilter -->|Pass| Enqueue ExcludeFilter -->|Fail| Skip - - classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff - classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff - classDef accent7 fill:#64748B,stroke:#475569,color:#fff ``` ### Accept Patterns