petrbalvin/goget
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: initial goget release — modern IPv6-first download utility

Browse Source
This commit is contained in:
petrbalvin
2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+43
View File
@@ -0,0 +1,43 @@
# EditorConfig: https://editorconfig.org
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
indent_size = 2
[*.{rb,erb,gemspec}]
indent_style = space
indent_size = 2
[{Gemfile,Rakefile}]
indent_style = space
indent_size = 2
[*.{js,html,css,vue}]
indent_style = space
indent_size = 2
[*.{yml,yaml,json,toml}]
indent_style = space
indent_size = 2
[*.{rs,cj}]
indent_style = space
indent_size = 4
[*.go]
indent_style = tab
indent_size = 4
[justfile]
indent_style = tab
[*.md]
indent_style = space
indent_size = 2
trim_trailing_whitespace = false
.forgejo/workflows/release.yml
+172
View File
@@ -0,0 +1,172 @@
name: Release
on:
push:
tags: ["v*"]
jobs:
build:
runs-on: codeberg-small
strategy:
fail-fast: false
matrix:
include:
- goos: linux
goarch: amd64
- goos: linux
goarch: arm64
- goos: linux
goarch: riscv64
- goos: linux
goarch: loong64
- goos: freebsd
goarch: amd64
- goos: freebsd
goarch: arm64
- goos: freebsd
goarch: riscv64
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- uses: https://data.forgejo.org/actions/setup-go@v5
with:
go-version: "1.26"
- name: Download dependencies
run: go mod download
- name: Build
id: build
env:
REF: ${{ forgejo.ref }}
run: |
set -euo pipefail
VERSION="${REF##*/}"
if [[ ! "$VERSION" =~ ^v[0-9]+(\.[0-9]+){0,2}([-+].*)?$ ]]; then
echo "ERROR: expected a semver tag like v1.2.3, got: '$VERSION' (ref: '$REF')"
exit 1
fi
VERSION_NO_V="${VERSION#v}"
echo "version_no_v=${VERSION_NO_V}" >> "$FORGEJO_OUTPUT"
LDFLAGS="-s -w -X codeberg.org/petrbalvin/goget/internal/core.Version=${VERSION}"
mkdir -p bin
GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} \
go build -ldflags "$LDFLAGS" \
-o "bin/goget-${VERSION_NO_V}-${{ matrix.goos }}-${{ matrix.goarch }}" \
./cmd/goget
- name: Upload artifact
uses: https://data.forgejo.org/actions/upload-artifact@v3
with:
name: goget-${{ matrix.goos }}-${{ matrix.goarch }}
path: bin/goget-${{ steps.build.outputs.version_no_v }}-${{ matrix.goos }}-${{ matrix.goarch }}
if-no-files-found: error
release:
runs-on: codeberg-tiny
needs: build
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- name: Download all artifacts
uses: https://data.forgejo.org/actions/download-artifact@v3
with:
path: dist
- name: Extract CHANGELOG section
env:
VERSION: ${{ forgejo.ref_name }}
run: |
set -euo pipefail
VERSION_NO_V="${VERSION#v}"
echo "Looking for CHANGELOG section: ${VERSION_NO_V}"
sed -n "/^## \[${VERSION_NO_V}\] /,/^## \[/p" CHANGELOG.md \
| sed '$d' \
| tail -n +2 \
> release-body.md
if [ ! -s release-body.md ]; then
echo "ERROR: no CHANGELOG section found for ${VERSION_NO_V}"
echo "Expected a heading like: ## [${VERSION_NO_V}] — YYYY-MM-DD"
exit 1
fi
echo "Release body preview:"
head -n 20 release-body.md
- name: Create release
env:
FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
run: |
set -euo pipefail
# JSON-escape the release body with sed: backslashes first, then
# quotes, tabs, and carriage returns, and finally fold newlines into
# \n. Wrap the result in quotes to form a JSON string.
BODY=$(sed -e 's/\\/\\\\/g' \
-e 's/"/\\"/g' \
-e 's/\t/\\t/g' \
-e 's/\r//g' \
release-body.md \
| sed ':a;N;$!ba;s/\n/\\n/g')
BODY="\"${BODY}\""
response=$(curl -sS -w '\n%{http_code}' \
-H "Authorization: token ${FORGEJO_TOKEN}" \
-H "Content-Type: application/json" \
-X POST \
"${FORGEJO_SERVER_URL}/api/v1/repos/${FORGEJO_REPOSITORY}/releases" \
-d "{
\"tag_name\": \"${FORGEJO_REF_NAME}\",
\"name\": \"${FORGEJO_REF_NAME}\",
\"body\": ${BODY},
\"draft\": false,
\"prerelease\": false
}")
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | sed '$d')
echo "HTTP ${http_code}"
if [ "$http_code" != "201" ]; then
echo "Failed to create release: ${body}"
exit 1
fi
RELEASE_ID=$(echo "$body" | grep -oE '"id"[[:space:]]*:[[:space:]]*[0-9]+' | head -1 | grep -oE '[0-9]+')
echo "Created release ID=${RELEASE_ID}"
printf '%s' "${RELEASE_ID}" > release-id.txt
- name: Upload assets
env:
FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
run: |
set -euo pipefail
RELEASE_ID=$(cat release-id.txt)
echo "Release ID: ${RELEASE_ID}"
for binary in dist/goget-*/goget-*; do
[ -f "$binary" ] || continue
fname=$(basename "$binary")
echo "Uploading ${fname}..."
http_code=$(curl -sS -o /dev/null -w '%{http_code}' \
-H "Authorization: token ${FORGEJO_TOKEN}" \
-H "Content-Type: application/octet-stream" \
-X POST \
--data-binary "@${binary}" \
"${FORGEJO_SERVER_URL}/api/v1/repos/${FORGEJO_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${fname}")
echo " HTTP ${http_code}"
if [ "$http_code" != "201" ]; then
echo "Failed to upload ${fname}"
exit 1
fi
done
echo
echo "Release ${FORGEJO_REF_NAME} is live with the following assets:"
ls -lh dist/goget-*/
.forgejo/workflows/test.yml
+79
View File
@@ -0,0 +1,79 @@
name: Test
on:
push:
branches: [development]
pull_request:
branches: [development]
jobs:
vet:
runs-on: codeberg-small
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- uses: https://data.forgejo.org/actions/setup-go@v5
with:
go-version: "1.26"
- name: Download dependencies
run: go mod download
- name: gofmt
# `gofmt -l` prints the names of unformatted files. An empty
# output is the only acceptable result.
run: gofmt -l cmd internal pkg
- name: go vet
run: go vet ./...
test:
runs-on: codeberg-small
needs: vet
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- uses: https://data.forgejo.org/actions/setup-go@v5
with:
go-version: "1.26"
- name: Download dependencies
run: go mod download
- name: go test -race
run: go test -race -count=1 ./...
- name: Coverage gate
run: |
go test -coverprofile=coverage.out ./...
coverage=$(go tool cover -func=coverage.out | awk '/^total:/ { gsub("%",""); print $3 }')
echo "Total coverage: ${coverage}%"
awk -v c="$coverage" 'BEGIN { exit !(c+0 < 40) }' \
&& { echo "Coverage ${coverage}% is below 40% threshold"; exit 1; } \
|| echo "Coverage ${coverage}% OK (threshold: 40%)"
fuzz:
runs-on: codeberg-small
needs: vet
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- uses: https://data.forgejo.org/actions/setup-go@v5
with:
go-version: "1.26"
- name: Download dependencies
run: go mod download
- name: go test -fuzz
run: |
for pkg in $(go list ./...); do
fuzz_funcs=$(go test -list='Fuzz.*' "$pkg" 2>/dev/null | grep '^Fuzz' || true)
if [ -n "$fuzz_funcs" ]; then
echo "==> Fuzzing $pkg"
for fn in $fuzz_funcs; do
echo " $fn …"
go test -parallel=2 -timeout=5m -fuzz="^${fn}$" -fuzztime=8s "$pkg"
done
fi
done
.gitignore
+35
View File
@@ -0,0 +1,35 @@
# goget .gitignore — build artifacts, IDE files, OS files, and logs
# Build output
bin/
# Test artifacts
*.test
*.out
coverage.html
# Go workspace (go.work and go.work.sum)
go.work
go.work.sum
# Vendored dependencies (we use Go modules)
vendor/
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Environment
.env
.env.local
.env.*.local
# Logs
*.log
BACKLOG.md
+187
View File
@@ -0,0 +1,187 @@
# Backlog — goget
This file tracks planned features and improvements across releases.
Items are grouped by target version and priority. Completed items
move to `CHANGELOG.md` when the version ships.
---
## v1.1.0 — Protocol completeness
Finish the protocol layer so every supported protocol handles
recursive, parallel, resume, and shutdown correctly.
### 🔴 Supreme Priority
- [ ] **Graceful shutdown for all protocols** — persist `.goget.meta` on
SIGINT for FTP single-file and SFTP parallel recursive paths.
- [ ] **WebDAV connection pooling** — shared `http.Transport` across WebDAV
requests instead of a fresh transport per request (currently defeats HTTP
keep-alive, TLS session resumption, and HTTP/2 multiplexing).
### 🟡 High Priority
- [ ] **WebDAV recursive parallel downloads** — extend `--recursive-parallel`
to WebDAV using the shared transport.
- [ ] **SFTP recursive parallel downloads** — parallel SFTP recursive with
session pooling.
- [ ] **FTP recursive parallel** — connection pool for independent control
connections in FTP recursive mode.
- [ ] **`--dry-run` for WebDAV recursive** — list matched entries without
downloading.
### 🟢 Medium Priority
- [ ] **`--adjust-extension` for non-HTML protocols** — apply `.html`
extension to text/html files from WebDAV, Gopher.
- [ ] **`--backup-converted` for WebDAV mirrors** — keep `.orig` backups
before link conversion.
- [ ] **Parallel recursive across all protocols** — unified worker pool
independent of protocol handler.
---
## v1.2.0 — curl/wget parity & UX
Close the gap with curl and wget, and improve scripting/automation UX.
### 🟡 High Priority
- [ ] **`--create-dirs`** — auto-create parent directories for `--output`
paths (curl `--create-dirs` / wget `--directory-prefix` equivalent).
- [ ] **Short flags** — `-O`, `-o`, `-s`, `-v` aliases for curl/wget
muscle memory. `--output-document` alias for `--output`.
- [ ] **Brotli decompression** — RFC 7932, written from scratch.
Currently only gzip/deflate/bzip2/LZW.
### 🟢 Medium Priority
- [ ] **`--progress=bar:force:noscroll`** — curl-compatible progress options.
- [ ] **`--metalink --follow-metalink=mem`** — keep Metalink in memory instead
of writing to disk.
- [ ] **Bandwidth throttling per-domain** — global rate limit only at present.
- [ ] **MIME type detection for local files** — determine Content-Type from
file extension when uploading.
- [ ] **`--cut-dirs` for recursive mode** — strip directory components from
output paths.
- [ ] **Notification hooks** — `--on-error` with richer environment variables
(exit code, error type, checksum).
- [ ] **Retry with backoff visualization** — show retry attempts in progress
bar (attempt 2/3, waiting 4s...).
- [ ] **Idempotent `--resume`** — auto-detect whether resume is possible
without manual flag.
- [ ] **`--show-metadata`** — display `.goget.meta` contents for debugging.
### 🔵 Lower Priority
- [ ] **URL glob expansion (`--glob`)** — expand `[1-3]` and `{a,b}` patterns
in URLs.
---
## v1.3.0 — Security hardening
TLS, authentication, and verification improvements.
### 🔴 Supreme Priority
- [ ] **PGP replacement** — `golang.org/x/crypto/openpgp` is frozen (archived
by Go team). Evaluate: inline a minimal subset, or wait for a community
`golang.org/x/` successor.
### 🟡 High Priority
- [ ] **`--pinned-cert` for all TLS protocols** — extend certificate pinning
to FTPS, SFTP, Gemini, and WebDAVS (currently HTTP/HTTPS only).
- [ ] **SOCKS5 proxy for non-HTTP protocols** — FTP and SFTP through SOCKS5
proxy (currently HTTP/HTTPS only).
- [ ] **DNS-over-HTTPS (DoH)** — optional DoH resolver. Must use only
`golang.org/x/` libraries.
### 🟢 Medium Priority
- [ ] **Structured error output with `--json`** — machine-readable error
objects with type, message, URL, and hint fields.
---
## v2.0.0 — New protocols
Cloud providers, additional network protocols, and long-lived services.
### 🔵 Lower Priority
- [ ] **S3** — generic S3-compatible protocol with signature v4 auth
(covers AWS S3, MinIO, and any S3-compatible storage).
- [ ] **Huawei OBS** — Huawei Cloud Object Storage.
- [ ] **Alibaba OSS** — Alibaba Cloud Object Storage Service.
- [ ] **Tencent COS** — Tencent Cloud Object Storage.
- [ ] **Rsync** — incremental file transfer. Written from scratch
(rolling checksum, delta transfer, rsync wire protocol).
- [ ] **TFTP** — trivial file transfer, useful for embedded devices and PXE.
- [ ] **NFS** — NFS v3/v4 client. Written from scratch
(ONC RPC/XDR, portmapper, mount protocol, NFS protocol).
- [ ] **goget-as-a-service** — daemon mode with REST API for remote download
management.
---
## v2.1.0 — Extensibility & UI
### 🔵 Lower Priority
- [ ] **Plugin system** — protocol handlers as external plugins.
- [ ] **TUI (Terminal User Interface)** — interactive download manager with
live queue, progress, and log views. Keyboard navigation, color-coded
status, inline log viewer.
- [ ] **Web GUI** — browser-based download manager (Vue 3 frontend, REST API,
live progress via SSE).
- [ ] **Download speed test / benchmark mode** — measure maximum throughput
against a target server without saving the file.
---
## 💡 Backlog (unscheduled)
Items that need more design, depend on external factors, or are
lower priority than the versioned roadmap.
### Performance
- [ ] **Memory-mapped I/O for large files** — `mmap` for checksum
verification of multi-gigabyte downloads.
- [ ] **Zero-copy for `file://` and `data:` protocols** — `sendfile(2)` /
`splice(2)` where available on Linux.
- [ ] **HTTP/3 (QUIC)** — when Go stdlib gains QUIC support.
### Advanced UX
- [ ] **Scheduling — pause/resume queued downloads** — ability to pause the
queue, reorder items, and resume later.
### Code Quality & Testing
- [ ] **Integration test matrix** — CI against real FTP, SFTP, and WebDAV
servers.
- [ ] **Coverage threshold increase** — raise from 40% to 60%.
- [ ] **Benchmark suite** — `go test -bench` in CI.
- [ ] **Fuzzing corpus in CI** — store and replay past crashes automatically.
- [ ] **`internal/log` migration** — migrate remaining ~140 `fmt.Fprintf` and
`cli.Print*` callsites to `log.Infof`.
- [ ] **`unsafe` audit** — verify no `unsafe` usage outside build-tag-gated
platform-specific code.
### Project Infrastructure
- [ ] **RPM spec file** — Fedora / RHEL / openEuler packaging.
- [ ] **FreeBSD port** — official FreeBSD ports tree entry.
- [ ] **systemd unit** — for daemon / scheduled mode.
### Documentation
- [ ] **Man page packaging** — already generated by `--generate-man-page`,
needs installation via packaging.
- [ ] **Architecture decision records** — document why specific choices were
made (no short flags, IPv6-first, long flags only, no global state).
- [ ] **Protocol-specific guides** — one doc per protocol with examples.
- [ ] **Troubleshooting guide** — common errors and solutions.
CHANGELOG.md
+177
View File
@@ -0,0 +1,177 @@
# Changelog
All notable changes to **goget** are documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [development]
## [1.0.0] — 2026-06-26
First public release: a modern IPv6-first command-line download utility built on
the Go standard library. goget provides parallel chunked downloads, resume
support, checksum verification, recursive website mirroring, and support for 9
network protocols — all in a single static binary with zero runtime dependencies.
### Added
**Core engine**
- Unified download and upload architecture via `core.Protocol` interface.
- Protocol registry with scheme normalization (https→http, ftps→ftp).
- Structured error types (`NETWORK`, `PROTOCOL`, `FILE`, `TIMEOUT`, `AUTH`,
`CHECKSUM`, `UNSUPPORTED`, `CANCELLED`).
- Request context with protocol chain tracking, attempt counting, and
cancellation.
- Smart timeout calculation based on file size, minimum speed, and safety factor.
- IPv6-first transport dialer with automatic IPv4 fallback.
**Protocols**
- **HTTP/HTTPS** — HTTP/1.1 and HTTP/2, TLS 1.2/1.3, content negotiation,
auto-decompression.
- **FTP/FTPS** — active and passive modes, explicit TLS (AUTH TLS), resume via
REST command.
- **SFTP** — custom SFTP v3 implementation over SSH, `known_hosts` verification,
resume, private key and password authentication.
- **WebDAV** — file download, recursive directory download, upload via HTTP PUT,
MKCOL for directory creation.
- **file://** — local file copy with recursive directory support.
- **data:** — RFC 2397 data URIs (base64 and plain text).
- **Gopher** — RFC 1436 with item type parsing.
- **Gemini** — TLS 1.2+ connections, redirect following, status code handling
(1x–6x).
**Network & transport**
- Concurrent chunked downloads for files over 100 MB with `Range` requests.
- Per-chunk resume metadata for interrupted parallel downloads.
- HTTP CONNECT and SOCKS5 (`socks5://`, `socks5h://`) proxy support with
username/password authentication.
- Custom DNS servers via `--dns-servers`.
- Network interface binding via `SO_BINDTODEVICE` (Linux).
- SSRF protection — blocks connections to private/internal IPs by default,
override with `--allow-private`.
- IPv4-mapped IPv6 normalization to prevent SSRF bypass.
- Token bucket rate limiting.
- Exponential backoff retry logic with configurable max retries.
- `--retry-all-errors` to retry on HTTP 4xx/5xx responses.
- `--pinned-cert` SHA-256 certificate pinning.
**Authentication**
- HTTP Basic authentication (RFC 7617).
- HTTP Digest authentication (RFC 7616).
- OAuth 2.0 — client credentials, authorization code, password, and refresh
token grant types with 30-second expiry grace period.
- OAuth error sanitization (RFC 6749 §5.2).
- `.netrc` auto-reading with machine and default entry resolution.
- Netscape-format cookie jar with import/export.
- Per-request cookies via `--cookie` CLI flag.
- SFTP SSH key auto-detection.
- `--password-file` for secure password loading.
- Config sanitization — strips credentials when saving to disk.
- `core.SafeURL()` — strips credentials from URLs in error messages and logs.
**Recursive downloading & mirroring**
- Recursive HTML link following with configurable depth limit.
- Full website mirroring with infinite depth and automatic link conversion.
- HTML parsing via `golang.org/x/net/html`.
- Link filtering by domain, accept pattern (glob and MIME), reject pattern, and
exclusion list.
- Page requisites — automatic CSS, JavaScript, and image downloading.
- `--no-parent`, `--span-hosts`, `--domains`, `--random-wait`.
- `--adjust-extension`, `--timestamping`, `--cut-dirs`.
- Link rewriting for offline viewing — absolute, protocol-relative, and
root-relative URLs.
- `--backup-converted` keeps `.orig` backups before link conversion.
- `robots.txt` compliance with `--no-robots` override.
**Verification & security**
- Checksum verification — SHA-256, SHA-512, SHA3-256, SHA3-512, BLAKE2b, MD5.
- SHA256SUMS file parsing (plain and binary-mode).
- PGP detached signature verification (`.asc`, `.sig`, `.gpg`).
- PGP decryption with private key and passphrase.
- HSTS cache (RFC 6797) — automatic HTTP→HTTPS upgrade.
- TLS certificate pinning, mTLS support, custom CA bundle.
- `--ssl-key-log` TLS session key logging (Wireshark-compatible).
- Atomic file writes with mode `0600` for config, resume metadata, and temp
files.
**Download management**
- Persistent download queue with priority (1–10) and JSON storage.
- Multi-source Metalink downloads (`.meta4`, `.metalink`) with automatic
failover.
- Batch URL loading from files.
- Cron-based scheduling (`MIN HOUR DOM MON DOW` and `YYYY-MM-DD HH:MM`).
- Post-download command hooks with environment variables.
- WARC (Web ARChive) output for archival compliance.
- Upload support — HTTP PUT and POST with multipart form data.
**Output & progress**
- Unicode progress bar with filled/empty blocks and real-time speed.
- Speed sparkline — mini ASCII chart of recent speed history.
- Estimated time remaining (ETA) calculation.
- Dot progress bar style, JSON progress output for scripting.
- `--write-out` metadata output, `--trace-time` HTTP timing breakdown.
- `--show-headers`, `--keep-timestamps`, `--content-disposition`.
- Color output with `always`, `never`, and `auto` modes (honors `NO_COLOR`).
**Archive & compression**
- Transparent decompression — gzip, deflate, zlib, bzip2, LZW.
- Archive extraction — TAR, TAR.GZ, TAR.BZ2, ZIP with auto-detection.
**Internationalization**
- IDN (International Domain Names) support with Punycode conversion via
`golang.org/x/net/idna`.
**CLI**
- Long flags only (`--url`, `--output`, `--verbose`).
- Shell completion generation — bash, zsh, and fish.
- Man page generation in troff format.
- Category-organized help text.
- `--info`, `--benchmark`, `--spider`, `--dry-run`, `--glob`.
**Configuration**
- TOML configuration file at `~/.config/goget/config.toml`.
- `GOGET_CONFIG` environment variable for alternative config path.
- CLI flag overrides merged on top of config and defaults.
- Config commands — init, get, set, list, unset.
**Library API (`pkg/api/`)**
- `Client` type with functional options (`WithTimeout`, `WithUserAgent`,
`WithVerbose`).
- `Client.Download()` and `Client.Upload()` with structured request/result types.
- `QueueClient` for programmatic queue management.
- `ParseMetalink()`, `FetchMetalink()`, `LoadConfig()`, `DefaultConfig()`,
`ParseSpeed()`, `Version()`.
- One-shot convenience functions: `Download()`, `Upload()`.
**Build & platforms**
- Linux — amd64, arm64, riscv64, loong64.
- FreeBSD — amd64, arm64, riscv64.
- Static binary with zero runtime dependencies (stdlib only +
`golang.org/x/` packages).
- Race detector enabled in tests, fuzz tests for security-critical parsers.
- Coverage gate at 40%.
**Project & tooling**
- Documentation set: architecture, API, authentication, CLI reference,
configuration, download pipeline, protocols, recursive & mirror, security.
- `just` task set (`install`, `run`, `build`, `test`, `uninstall`).
- Forgejo Actions CI (go vet, test + race, fuzz, release).
- `CONTRIBUTING.md` with Conventional Commits and release process.
[1.0.0]: https://codeberg.org/petrbalvin/goget/releases/tag/v1.0.0
CONTRIBUTING.md
+304
View File
@@ -0,0 +1,304 @@
# Contributing
Thank you for considering contributing to **goget** — a modern IPv6-first
download utility built on the Go standard library.
## Development Setup
### Requirements
- Go 1.26+
- Linux (amd64, arm64, riscv64, loong64) or FreeBSD (amd64, arm64, riscv64)
### Quick Start
```bash
git clone https://codeberg.org/petrbalvin/goget.git
cd goget
just install # go mod download
just test # go test -race -count=1 ./...
just build # go vet + gofmt check + build → bin/goget
```
### Running Tests
```bash
just test
```
The test suite (`*_test.go` files alongside the code) covers:
- Protocol handlers (HTTP, FTP, SFTP, WebDAV, file, data, Gopher, Gemini).
- Transport layer (IPv6-first dialer, proxy, retry, rate limiting).
- Checksum verification, PGP signature handling, HSTS cache.
- Recursive downloading, link rewriting, Metalink parsing.
- Configuration loading, merging, and sanitization.
- CLI flag parsing, output formatting, progress bar.
### Linting
```bash
just build # go vet + gofmt check (also part of CI)
```
CI runs both `go vet ./...` and a `gofmt -l` diff check. These are enforced —
make sure they pass before submitting a pull request.
## Code Style
- Follow [Effective Go](https://go.dev/doc/effective_go) conventions.
- Keep lines under 100 characters where practical.
- Use `camelCase` for variables and functions, `PascalCase` for exported
symbols.
- Prefix every file with a one-line comment describing its purpose.
- Use build tags `//go:build linux || freebsd` on all source files.
- Use long flags in CLI (`--header` not `-H`).
- Prefer the Go standard library over external dependencies. External
dependencies are only allowed from `golang.org/x/` unless justified.
- Error messages in English, lowercase, no trailing period, always with context
(`open config: permission denied`).
- `gofmt` all code before committing.
### Testing
- Write tests alongside the code in `_test.go` files.
- Use table-driven tests for multiple test cases.
- Include both happy path and error path tests.
- Name tests following Go convention: `TestFunctionName_Scenario`.
- Use `testing.F` for fuzz tests in security-critical parsing code.
## Commit Conventions
We follow [Conventional Commits](https://www.conventionalcommits.org/).
| Type | Use |
|------------|----------------------------------------------|
| `feat` | New feature |
| `fix` | Bug fix |
| `refactor` | Restructuring without behaviour change |
| `docs` | Documentation |
| `test` | Adding or updating tests |
| `chore` | Maintenance, CI, tooling |
| `perf` | Performance improvement |
```
feat: add Gemini protocol support
fix: handle timeout during parallel chunk downloads
refactor: unify transport dialer configuration
docs: add README quick start examples
test: add edge cases for cron schedule parser
chore: update golang.org/x/crypto
```
- English, lowercase subject, imperative mood (`add`, not `added`).
- No trailing period, max 72 characters.
- No version numbers in commit messages — versions belong to tags.
## Pull Request Flow
1. Create a feature branch from `development`.
2. Make your changes with Conventional-Commits messages.
3. Record your changes under the `## [development]` section at the top of
`CHANGELOG.md`, using the Keep a Changelog categories (`Added`, `Changed`,
`Fixed`, etc.).
4. Ensure `just build` and `just test` pass.
5. Add or update tests for your change.
6. Update documentation if the public API, configuration, or behaviour changes:
- `README.md` — high-level overview.
- `docs/configuration.md` — config keys.
- `docs/api.md` — Go library API.
- `docs/cli-reference.md` — CLI flags.
- `docs/architecture.md` — structural changes.
- `docs/security.md` — anything touching auth, TLS, or verification.
7. Open a pull request against `development`. Releases are cut by merging
`development` into `main` and pushing a `vX.Y.Z` tag — see
[Cutting a release](#cutting-a-release).
## Project Structure
```
goget/
├── cmd/
│ └── goget/ # CLI entry point
│ ├── main.go # Core run loop, flag setup, orchestrator
│ ├── commands.go # Command handlers (download, upload, queue, config, etc.)
│ ├── flags.go # CLI flag definitions (struct + defineFlags)
│ ├── completion.go # Shell completion generators (bash, zsh, fish)
│ ├── man_page.go # Man page generation
│ ├── schedule.go # Cron and datetime scheduling
│ └── verify.go # Checksum and PGP verification
├── internal/
│ ├── archive/ # Archive extraction (tar, targz, tarbz2, zip)
│ ├── auth/ # Authentication (basic, digest, OAuth 2.0, netrc)
│ ├── cli/ # CLI utilities (help text, progress bar, color output)
│ ├── compression/ # Decompression (gzip, bzip2, zlib, flate, lzw)
│ ├── config/ # TOML config loading, saving, and merging
│ ├── cookie/ # Netscape-format cookie jar
│ ├── core/ # Core types (DownloadRequest, Result, Protocol interface, errors)
│ ├── crypto/ # Checksum verification, PGP, certificate handling
│ ├── format/ # Human-readable formatting (bytes, speed)
│ ├── hsts/ # HSTS cache (RFC 6797)
│ ├── log/ # Structured logger (text/JSON output, level filtering)
│ ├── linkrewrite/ # HTML link rewriting for offline mirroring
│ ├── metalink/ # Metalink (multi-source downloads)
│ ├── mirror/ # Website mirroring logic
│ ├── output/ # Output writing (atomic writes, resume metadata)
│ ├── protocol/ # Protocol abstraction layer
│ │ ├── http/ # HTTP/HTTPS client, parallel download, upload
│ │ ├── ftp/ # FTP/FTPS client
│ │ ├── sftp/ # SFTP client
│ │ ├── file/ # Local file:// protocol
│ │ ├── dataurl/ # data: URI protocol
│ │ ├── gopher/ # Gopher protocol
│ │ ├── gemini/ # Gemini protocol
│ │ └── webdav/ # WebDAV protocol
│ ├── queue/ # Download queue management
│ ├── recursive/ # Recursive download (HTML parser, URL filter, crawler)
│ ├── warc/ # WARC archiving writer
│ └── transport/ # HTTP transport (dialer, proxy, TLS, retry, rate limiting)
├── pkg/
│ └── api/ # Public library API (Client, Download, Upload, Queue)
└── docs/ # Architecture and design documentation
```
## Architecture Overview
```mermaid
graph TD
CLI[cmd/goget/ - CLI Entry]
Flags[Flags parsing]
Config[internal/config/]
Core[internal/core/ - Types & Protocol Interface]
Registry[internal/protocol/ - Registry]
HTTP[internal/protocol/http/]
FTP[internal/protocol/ftp/]
SFTP[internal/protocol/sftp/]
FILE[internal/protocol/file/]
DataURL[internal/protocol/dataurl/]
Gopher[internal/protocol/gopher/]
Gemini[internal/protocol/gemini/]
WebDAV[internal/protocol/webdav/]
Transport[internal/transport/ - Dialer, Proxy, TLS]
Output[internal/output/ - Atomic write, Resume]
Crypto[internal/crypto/ - Checksum, PGP]
Auth[internal/auth/]
Cookie[internal/cookie/]
HSTS[internal/hsts/]
Recursive[internal/recursive/]
Mirror[internal/mirror/]
Metalink[internal/metalink/]
QueueInt[internal/queue/]
Archive[internal/archive/]
API[pkg/api/ - Public Library]
CLI --> Flags
CLI --> Config
CLI --> Core
CLI --> Auth
CLI --> Crypto
CLI --> Recursive
CLI --> Mirror
CLI --> Metalink
CLI --> QueueInt
CLI --> Archive
Core --> Registry
Registry --> HTTP
Registry --> FTP
Registry --> SFTP
Registry --> FILE
Registry --> DataURL
Registry --> Gopher
Registry --> Gemini
Registry --> WebDAV
HTTP --> Transport
HTTP --> Cookie
HTTP --> HSTS
HTTP --> Output
API --> Core
API --> Registry
```
### Key Design Decisions
- **Protocol abstraction** — Every network protocol implements `core.Protocol`
and registers with a global registry. The CLI and API resolve protocols by URL
scheme at runtime.
- **IPv6-first** — The transport dialer prefers IPv6 addresses and falls back to
IPv4 only when no IPv6 address is available.
- **Parallel downloads** — Files over 100 MB are automatically split into
chunks. Each chunk downloads via a separate connection with byte-range
requests.
- **Atomic output** — Downloads write to a temporary file first, then atomically
rename on completion. Resume metadata is stored alongside the partial file.
- **Config merging** — CLI flags override config file values, which override
defaults. The `config.Merge()` method applies CLI overrides after loading.
- **No global state** — The CLI is the only singleton. Internal packages are
designed for testability with injectable dependencies.
## Testing Strategy
| Layer | Approach | Tools |
|----------------|------------------------------------------------------------|---------------------|
| Unit tests | Table-driven tests for parsers, validators, handlers | `testing` stdlib |
| Integration | HTTP/HTTPS against local test servers, FTP against mock | `net/http/httptest` |
| Fuzz tests | Random input for checksum parsing, Metalink, HSTS cache | `testing.F` |
| Race detection | All tests run with `-race` in CI | `go test -race` |
## CI
goget uses **Forgejo Actions**; both workflows live under `.forgejo/workflows/`
and run on the `codeberg-small` / `codeberg-medium` runners.
### Test — `.forgejo/workflows/test.yml`
Triggered on push and pull requests targeting `development`.
| Job | What it does |
|-------|-----------------------------------------------|
| `vet` | `go vet ./...` (must report zero warnings) |
| `test`| `go test -race -count=1 ./...` + coverage gate |
| `fuzz`| `go test -fuzz` on every `FuzzXxx` function |
Run the same locally before opening a PR:
```bash
go vet ./...
just test
```
### Release — `.forgejo/workflows/release.yml`
Triggered by pushing a `v*` tag. It verifies the tag matches
`internal/core.Version`, runs the quality gate (go vet + tests), builds static
binaries for all 7 platform targets, extracts the matching `CHANGELOG.md`
section, creates a Forgejo release via the REST API, and attaches the binaries.
Requires a `FORGEJO_TOKEN` secret with permission to create releases.
### Cutting a release
1. Bump the `Version` var in `internal/core/version.go`.
2. Rename `## [development]` to `## [X.Y.Z] — YYYY-MM-DD` at the top of
`CHANGELOG.md`, and add a fresh empty `## [development]` section above it.
3. Ensure CI is green on `development`.
4. Merge `development` into `main` (the only time `main` changes outside a tag).
5. Tag and push:
```bash
git tag -a vX.Y.Z -m "goget vX.Y.Z"
git push origin main
git push origin vX.Y.Z
```
6. The release workflow builds the binaries and publishes the release from the
CHANGELOG section.
## Report a Bug
Open an issue at <https://codeberg.org/petrbalvin/goget/issues> with:
- Goget version (`goget --version`).
- Operating system and architecture.
- Command used and full output.
- Expected vs actual behaviour.
For **security issues**, please email **opensource@petrbalvin.org** rather than
opening a public issue.
LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+300
View File
@@ -0,0 +1,300 @@
# Goget — Modern IPv6-First Command-Line Download Utility
**goget** is a modern command-line download utility built on the **Go standard
library**. It provides IPv6-first connections, parallel chunked downloads,
resume support, checksum verification, recursive website mirroring, and support
for 9 network protocols — all in a single static binary with zero runtime
dependencies.
It is designed to be:
- **IPv6-first** with automatic IPv4 fallback — always connects via the fastest
available path.
- **Safe by default** — TLS 1.2 minimum, certificate verification, SSRF
protection, and HSTS cache (RFC 6797).
- **Self-contained** — one static binary, zero runtime dependencies beyond the
Go standard library and `golang.org/x/` packages.
- **Scriptable** — JSON progress output, `--write-out` metadata, post-download
hooks, and a public Go library API.
---
## Stack
| Concern | Choice |
|-----------------|------------------------------------------|
| Language | Go (≥ 1.26) |
| Protocols | HTTP/1.1, HTTP/2, FTP, SFTP, WebDAV, Gemini, Gopher |
| HTTP transport | Custom dialer with IPv6-first resolution |
| TLS | Go `crypto/tls` (1.2 minimum) |
| Markup | `golang.org/x/net/html` for recursive downloads |
| Crypto | stdlib + `golang.org/x/crypto` for PGP |
---
## Features
- **9 protocols** — HTTP/1.1, HTTP/2, FTP (active/passive, explicit TLS), SFTP
with `known_hosts` verification, WebDAV (`webdav://`, `webdavs://`) with
recursive directory download and upload, Gemini, Gopher, `data:`, `file://`.
- **Parallel chunked downloads** — automatically splits files over 100 MB into
concurrent chunks using byte-range requests.
- **Resume interrupted transfers** — metadata-driven resume with `.goget.meta`
tracking across HTTP, FTP, SFTP, and WebDAV.
- **Recursive downloading & mirroring** — follow links, filter by pattern or
MIME type, limit depth, stay within domain. Full site mirrors with link
conversion for offline viewing and `robots.txt` compliance.
- **Checksum verification** — SHA-256, SHA-512, SHA3-256, SHA3-512, BLAKE2b,
MD5; auto-parse SHA256SUMS files.
- **PGP support** — detached signature verification and OpenPGP decryption.
- **Metalink** — multi-source downloads with automatic failover (`.meta4`,
`.metalink`).
- **Download queue** — persistent queue with priority (1–10).
- **Scheduling** — absolute datetime or cron expressions.
- **OAuth 2.0** — client credentials, authorization code, and refresh token
flows.
- **Proxy support** — HTTP CONNECT, SOCKS5 (`socks5://`, `socks5h://`).
- **HSTS cache** — RFC 6797 compliance, automatic HTTP→HTTPS upgrade.
- **WARC output** — Web ARChive format for legal and archival compliance.
- **Library API** — `pkg/api/` package for programmatic downloads, uploads, and
queue management.
- **Archive extraction** — TAR, TAR.GZ, TAR.BZ2, ZIP with auto-detection.
- **Shell completion** — bash, zsh, and fish.
- **IDN support** — international domain names with automatic Punycode
conversion.
---
## Quick Start
```bash
git clone https://codeberg.org/petrbalvin/goget && cd goget && just install && just run --version
```
> `just install` downloads Go module dependencies. `just build` builds a static
> binary to `bin/goget`.
```bash
# Download a file
goget --url https://example.com/file.zip
# Parallel chunked download (auto-enabled for files >100 MB)
goget --url https://example.com/large.iso --parallel 4
# Resume interrupted download
goget --url https://example.com/file.zip --resume
# Multiple URLs
goget --url https://a.com/1.zip --url https://b.com/2.zip
# Batch download from file
goget --batch-file urls.txt
# Custom output directory
goget --url https://example.com/file.zip --output ./downloads/file.zip
```
---
## Usage
### Common Flags
```bash
# Custom HTTP headers
goget --header "Authorization: Bearer token" --url https://api.example.com
# Cookies
goget --cookie "session=abc123" --url https://example.com
# POST data
goget --data "key=value" --url https://httpbin.org/post
# Skip TLS verification (⚠ testing only)
goget --insecure --url https://self-signed.example.com
# Client certificate (mTLS)
goget --cert client.pem --key client.key --url https://mtls.example.com
# HTTP trace timing
goget --trace-time --url https://example.com
# → DNS: 12ms | TCP: 34ms | TLS: 56ms | TTFB: 78ms | Total: 180ms
# Timeouts
goget --max-time 30s --url https://slow.example.com
goget --connect-timeout 5s --url https://example.com
# Rate limiting
goget --rate-limit 1MB/s --url https://example.com/file.zip
# Proxy
goget --proxy http://proxy.example.com:8080 --url https://example.com/file.zip
goget --proxy socks5h://127.0.0.1:1080 --url https://example.com/file.zip
```
### Recursive & Mirroring
```bash
# Recursive download with depth limit
goget --url https://example.com/docs/ --recursive --max-depth 3
# Pattern filtering
goget --recursive --accept "*.pdf,*.html" --url https://example.com
# Dry-run — list without downloading
goget --url https://example.com/docs/ --recursive --dry-run
# Mirror entire website with link conversion
goget --url https://example.com --mirror --convert-links --output ./mirror
# WARC archiving
goget --warc-file archive.warc --url https://example.com/page.html
```
### Verification
```bash
# Checksum
goget --checksum abc123... --url https://example.com/file.zip
goget --checksum-file SHA256SUMS --url https://example.com/file.zip
# PGP
goget --pgp-verify --pgp-sig file.sig --url https://example.com/file
# Metalink multi-source
goget --metalink --url https://example.com/release.meta4
```
### Advanced
```bash
# Schedule a download
goget --schedule "2026-06-01 02:00" --url https://example.com/daily.zip
goget --schedule "0 2 * * *" --url https://example.com/daily.zip
# Post-download hook (env: GOGET_OUTPUT, GOGET_SIZE, GOGET_URL)
goget --on-complete "process.sh" --url https://example.com/data.csv
# JSON progress for scripting
goget --json --no-progress --url https://example.com/file.zip
# Shell completion
goget --completion bash > /etc/bash_completion.d/goget
goget --completion zsh > /usr/share/zsh/site-functions/_goget
goget --completion fish > ~/.config/fish/completions/goget.fish
# Configuration management
goget --init-config
goget --config-get timeout
goget --config-set timeout 5m
goget --config-list
```
---
## Configuration
Default location: `~/.config/goget/config.toml`. See
[`docs/configuration.md`](docs/configuration.md) for the full schema.
```toml
timeout = "30m0s"
parallel = 0
user_agent = "Goget/1.0.0"
max_speed = 0
auto_resume = true
auto_decompress = true
insecure_skip_verify = false
checksum_algo = "sha256"
color = "auto"
progress_style = "unicode"
max_retries = 0
proxy = ""
[recursive]
enabled = false
max_depth = 3
follow_external = false
delay = "100ms"
```
---
## Library Usage
```go
package main
import (
"fmt"
"log"
"time"
"codeberg.org/petrbalvin/goget/pkg/api"
)
func main() {
client := api.NewClient(
api.WithTimeout(5 * time.Minute),
api.WithUserAgent("MyApp/1.0"),
)
defer client.Close()
result, err := client.Download(&api.DownloadRequest{
URL: "https://example.com/file.zip",
Output: "output.zip",
Resume: true,
})
if err != nil {
log.Fatalf("download failed: %v", err)
}
fmt.Printf("Downloaded %d bytes in %v (%.1f MB/s, IPv%d)\n",
result.BytesDownloaded, result.Duration,
result.Speed/1024/1024, result.IPVersion)
}
```
Full API reference: [`docs/api.md`](docs/api.md).
---
## Development
### Requirements
- Go 1.26+
- Linux (amd64, arm64, riscv64, loong64) or FreeBSD (amd64, arm64, riscv64)
```bash
just # list recipes
just install # go mod download
just run # run from source (development)
just build # go vet + gofmt check + build → bin/goget
just test # go test -race -count=1 ./...
just uninstall # remove build artifacts
```
---
## Documentation
- [`docs/architecture.md`](docs/architecture.md) — components and request flow
- [`docs/configuration.md`](docs/configuration.md) — config schema (TOML)
- [`docs/api.md`](docs/api.md) — public Go library API
- [`docs/cli-reference.md`](docs/cli-reference.md) — full CLI flag reference
- [`docs/protocols.md`](docs/protocols.md) — protocol handler details
- [`docs/recursive-mirror.md`](docs/recursive-mirror.md) — recursive downloading
and mirroring
- [`docs/download-pipeline.md`](docs/download-pipeline.md) — download flow
- [`docs/authentication.md`](docs/authentication.md) — auth mechanisms
- [`docs/security.md`](docs/security.md) — threat model and controls
## Contributing
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, code style, commit
conventions, the pull request flow, and how releases are cut.
## License
MIT — see [LICENSE](LICENSE).
Copyright © 2026 [Petr Balvín](https://petrbalvin.org)
assets/goget-icon.svg
+43
View File
@@ -0,0 +1,43 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" width="512" height="512" role="img" aria-label="goget">
<defs>
<!-- Aurora gradient: cyan → sky → indigo -->
<linearGradient id="hexAurora" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" stop-color="#22D3EE"/>
<stop offset="38%" stop-color="#0EA5E9"/>
<stop offset="72%" stop-color="#2563EB"/>
<stop offset="100%" stop-color="#1E1B4B"/>
</linearGradient>
<!-- Inner radial highlight (subtle top sheen) -->
<radialGradient id="innerGlow" cx="50%" cy="26%" r="72%">
<stop offset="0%" stop-color="#ffffff" stop-opacity="0.22"/>
<stop offset="55%" stop-color="#ffffff" stop-opacity="0.04"/>
<stop offset="100%" stop-color="#ffffff" stop-opacity="0"/>
</radialGradient>
<!-- Refined soft shadow -->
<filter id="iconShadow" x="-30%" y="-30%" width="160%" height="160%">
<feDropShadow dx="0" dy="18" stdDeviation="28" flood-color="#0c1a4a" flood-opacity="0.30"/>
</filter>
</defs>
<!-- Hexagon base with shadow -->
<path d="M 256 56 L 429 156 L 429 356 L 256 456 L 83 356 L 83 156 Z"
fill="url(#hexAurora)"
filter="url(#iconShadow)"/>
<!-- Inner radial highlight overlay -->
<path d="M 256 56 L 429 156 L 429 356 L 256 456 L 83 356 L 83 156 Z"
fill="url(#innerGlow)"/>
<!-- Subtle inner hexagon outline -->
<path d="M 256 80 L 410 168 L 410 344 L 256 432 L 102 344 L 102 168 Z"
fill="none" stroke="#ffffff" stroke-width="2" opacity="0.18"/>
<!-- Three downward chevrons: streaming data / multi-protocol cascade -->
<g fill="none" stroke="#ffffff" stroke-width="22" stroke-linejoin="miter" stroke-miterlimit="6" stroke-linecap="round">
<polyline points="172,162 256,232 340,162"/>
<polyline points="172,242 256,312 340,242"/>
<polyline points="172,322 256,392 340,322"/>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.9 KiB

assets/goget-logo.svg
+55
View File
@@ -0,0 +1,55 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 560 140" width="560" height="140" role="img" aria-label="goget">
<defs>
<!-- Aurora gradient (matches icon) -->
<linearGradient id="hexAurora" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" stop-color="#22D3EE"/>
<stop offset="38%" stop-color="#0EA5E9"/>
<stop offset="72%" stop-color="#2563EB"/>
<stop offset="100%" stop-color="#1E1B4B"/>
</linearGradient>
<!-- Wordmark gradient for "go" -->
<linearGradient id="goGrad" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" stop-color="#22D3EE"/>
<stop offset="55%" stop-color="#0EA5E9"/>
<stop offset="100%" stop-color="#2563EB"/>
</linearGradient>
<!-- Inner radial highlight (matches icon) -->
<radialGradient id="innerGlow" cx="50%" cy="26%" r="72%">
<stop offset="0%" stop-color="#ffffff" stop-opacity="0.22"/>
<stop offset="55%" stop-color="#ffffff" stop-opacity="0.04"/>
<stop offset="100%" stop-color="#ffffff" stop-opacity="0"/>
</radialGradient>
<!-- Soft shadow for the small mark -->
<filter id="markShadow" x="-30%" y="-30%" width="160%" height="160%">
<feDropShadow dx="0" dy="5" stdDeviation="9" flood-color="#0c1a4a" flood-opacity="0.26"/>
</filter>
<!-- Single source of truth: identical geometry to goget-icon.svg -->
<symbol id="gogetMark" viewBox="0 0 512 512">
<path d="M 256 56 L 429 156 L 429 356 L 256 456 L 83 356 L 83 156 Z"
fill="url(#hexAurora)"/>
<path d="M 256 56 L 429 156 L 429 356 L 256 456 L 83 356 L 83 156 Z"
fill="url(#innerGlow)"/>
<path d="M 256 80 L 410 168 L 410 344 L 256 432 L 102 344 L 102 168 Z"
fill="none" stroke="#ffffff" stroke-width="2" opacity="0.18"/>
<g fill="none" stroke="#ffffff" stroke-width="22" stroke-linejoin="miter" stroke-miterlimit="6" stroke-linecap="round">
<polyline points="172,162 256,232 340,162"/>
<polyline points="172,242 256,312 340,242"/>
<polyline points="172,322 256,392 340,322"/>
</g>
</symbol>
</defs>
<!-- Mark: reuse the symbol at 100x100 -->
<use href="#gogetMark" x="20" y="20" width="100" height="100" filter="url(#markShadow)"/>
<!-- Wordmark: "goget" with aurora "go" + slate "get" -->
<text x="130" y="100"
font-family="'Inter', 'SF Pro Display', -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, Roboto, 'Helvetica Neue', Arial, sans-serif"
font-size="86"
font-weight="800"
letter-spacing="-3"><tspan fill="url(#goGrad)">go</tspan><tspan fill="#0F172A">get</tspan></text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

cmd/goget/app.go
+660
View File
@@ -0,0 +1,660 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"context"
"flag"
"fmt"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
"codeberg.org/petrbalvin/goget/internal/cli"
"codeberg.org/petrbalvin/goget/internal/config"
"codeberg.org/petrbalvin/goget/internal/cookie"
"codeberg.org/petrbalvin/goget/internal/core"
format "codeberg.org/petrbalvin/goget/internal/format"
"codeberg.org/petrbalvin/goget/internal/hsts"
"codeberg.org/petrbalvin/goget/internal/log"
"codeberg.org/petrbalvin/goget/internal/output"
"codeberg.org/petrbalvin/goget/internal/protocol"
protocolreg "codeberg.org/petrbalvin/goget/internal/protocol/register"
"codeberg.org/petrbalvin/goget/internal/transport"
)
// app holds the runtime state of a goget command execution.
type app struct {
cfg *config.Config
flags *Flags
fs *flag.FlagSet
configPath string
allURLs []string
sigCtx context.Context
sigCancel context.CancelFunc
}
// run is the main entry point. It is kept small — the work is delegated to
// init, dispatch, setup, and execute.
func (a *app) run() int {
// init() assigns a.sigCancel; a no-op here makes the early
// `return` paths (e.g. "missing required flag" on a fresh
// invocation) safe to clean up. Once init() runs, the real
// cancel function replaces this no-op.
defer func() {
if a.sigCancel != nil {
a.sigCancel()
}
}()
if code := a.init(); code != 0 {
return code
}
if code := a.dispatch(); code >= 0 {
return code
}
return exitCodeForRun(a.sigCtx, a.execute())
}
// exitCodeForRun maps the inner exit code to the final process exit
// code. If the signal context was cancelled (SIGINT/SIGTERM) but
// execute() returned 0 — because the download propagated the cancel
// cleanly — surface 130 so shell scripts can detect the interruption.
// On a normal completion (no signal) the inner code is returned
// unchanged; non-zero inner codes are preserved regardless of signal
// state so genuine errors are not masked.
func exitCodeForRun(sigCtx context.Context, innerCode int) int {
if innerCode != 0 {
return innerCode
}
if sigCtx != nil && sigCtx.Err() != nil {
return 130
}
return 0
}
// init loads config, sets up the signal context, and parses CLI flags.
func (a *app) init() int {
a.configPath = extractConfigPath(os.Args[1:])
cfg, err := config.Load(a.configPath)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err))
return 1
}
a.cfg = cfg
sigCtx, stop := signalContext()
a.sigCtx = sigCtx
a.sigCancel = stop
a.fs = flag.NewFlagSet("goget", flag.ExitOnError)
a.fs.Usage = func() { cli.PrintUsage(os.Stderr) }
a.flags = defineFlags(a.fs, a.cfg)
if err := a.fs.Parse(os.Args[1:]); err != nil {
cli.PrintError(os.Stderr, err)
return 1
}
a.allURLs = extractURLs(os.Args[1:])
return 0
}
// dispatch handles all standalone subcommands (help, version, config-*, info,
// benchmark, etc.). Returns the exit code, or -1 if no subcommand matched
// (meaning the caller should proceed to the download pipeline).
func (a *app) dispatch() int {
f := a.flags
cfg := a.cfg
if *f.Help {
cli.PrintUsage(os.Stdout)
return 0
}
if *f.Version {
fmt.Printf("goget version %s\n", core.Version)
return 0
}
if *f.ConfigList {
return handleConfigList(a.configPath)
}
if *f.ConfigGet != "" {
return handleConfigGet(a.configPath, *f.ConfigGet)
}
if *f.ConfigSet != "" {
return handleConfigSet(a.configPath, *f.ConfigSet, os.Args)
}
if *f.ConfigUnset != "" {
return handleConfigUnset(a.configPath, *f.ConfigUnset)
}
if *f.Info != "" {
return handleInfo(*f.Info, cfg)
}
if *f.Benchmark != "" {
return handleBenchmark(*f.Benchmark, cfg)
}
if *f.Spider {
return handleSpider(*f.URL, cfg)
}
if *f.GenManPage {
return handleGenManPage()
}
if *f.Completion != "" {
return handleCompletion(*f.Completion)
}
if *f.InitConfig {
return handleInitConfig(a.configPath)
}
if *f.BatchFile != "" {
return handleBatchFile(*f.BatchFile, *f.Output, cfg, a.sigCtx)
}
if *f.QueueList {
return handleQueueList(*f.QueueFile)
}
if *f.QueueProcess {
return handleQueueProcess(*f.QueueFile, cfg.Verbose)
}
if *f.QueueClear {
return handleQueueClear(*f.QueueFile, cfg.Verbose)
}
if *f.MetalinkFile != "" {
return handleMetalinkFile(*f.MetalinkFile, *f.Output, cfg.Verbose, *f.Queue, *f.QueuePriority, *f.QueueFile)
}
if *f.Metalink {
return handleMetalinkURL(*f.URL, *f.Output, cfg.Verbose, *f.Queue, *f.QueuePriority, *f.QueueFile)
}
if *f.Upload {
return handleUpload(*f.URL, *f.UploadMethod, *f.UploadFile, *f.UploadFormData, *f.UploadFileField, cfg.Verbose, cfg)
}
if len(a.allURLs) > 1 {
return handleMultipleURLs(a.allURLs, *f.Output, cfg, a.sigCtx)
}
return -1 // proceed to download pipeline
}
// execute runs the download pipeline after dispatch determined that a
// download should be performed. It handles URL validation, config merging,
// authentication, transport setup, and the actual download via the protocol
// handler.
func (a *app) execute() int {
f := a.flags
cfg := a.cfg
if *f.URL == "" {
cli.PrintError(os.Stderr, fmt.Errorf("--url is a required parameter"))
a.fs.Usage()
return 1
}
parsedURL, err := url.Parse(*f.URL)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("invalid url: %w", err))
return 1
}
// Convert internationalized domain name to Punycode
parsedURL = idnaURL(parsedURL)
excludePatterns := []string{}
if *f.ExcludePattern != "" {
for _, p := range strings.Split(*f.ExcludePattern, ",") {
p = strings.TrimSpace(p)
if p != "" {
excludePatterns = append(excludePatterns, p)
}
}
}
oauthScopes := []string{}
if *f.OAuthScopes != "" {
for _, s := range strings.Split(*f.OAuthScopes, ",") {
s = strings.TrimSpace(s)
if s != "" {
oauthScopes = append(oauthScopes, s)
}
}
}
if cfg.IsExcluded(parsedURL.String()) {
cli.PrintError(os.Stderr, fmt.Errorf("url matches exclude pattern: %s", parsedURL.String()))
return 1
}
maxSpeed := config.ParseSpeed(*f.MaxSpeed)
cfg.Merge(&config.CLIFlags{
Timeout: *f.Timeout,
Parallel: *f.Parallel,
Verbose: *f.Verbose,
Debug: *f.Debug,
MaxSpeed: maxSpeed,
Proxy: *f.Proxy,
NoIPv4: *f.NoIPv4,
NoDecompress: *f.NoDecompress,
Username: *f.Username,
Password: *f.Password,
CookieJar: *f.CookieJar,
Recursive: *f.Recursive,
MaxDepth: *f.MaxDepth,
FollowExternal: *f.FollowExternal,
ExcludePatterns: excludePatterns,
Extract: *f.Extract,
ExtractDir: *f.ExtractDir,
StripComponents: *f.StripComponents,
})
// Authentication settings
if *f.AuthType != "" {
cfg.Auth.AuthType = *f.AuthType
}
// OAuth 2.0 settings
if *f.OAuthClientID != "" || *f.OAuthClientSecret != "" || *f.OAuthTokenURL != "" ||
*f.OAuthAccessToken != "" || *f.OAuthRefreshToken != "" {
if *f.OAuthClientID != "" {
cfg.Auth.OAuth.ClientID = *f.OAuthClientID
}
if *f.OAuthClientSecret != "" {
cfg.Auth.OAuth.ClientSecret = *f.OAuthClientSecret
}
if *f.OAuthTokenURL != "" {
cfg.Auth.OAuth.TokenURL = *f.OAuthTokenURL
}
if *f.OAuthAuthURL != "" {
cfg.Auth.OAuth.AuthURL = *f.OAuthAuthURL
}
if *f.OAuthRedirectURI != "" {
cfg.Auth.OAuth.RedirectURI = *f.OAuthRedirectURI
}
if len(oauthScopes) > 0 {
cfg.Auth.OAuth.Scopes = oauthScopes
}
if *f.OAuthGrantType != "" {
cfg.Auth.OAuth.GrantType = *f.OAuthGrantType
}
if *f.OAuthAccessToken != "" {
cfg.Auth.OAuth.AccessToken = *f.OAuthAccessToken
}
if *f.OAuthRefreshToken != "" {
cfg.Auth.OAuth.RefreshToken = *f.OAuthRefreshToken
}
}
// Checksum algorithm settings
if *f.ChecksumAlgo != "" {
cfg.ChecksumAlgo = *f.ChecksumAlgo
}
// Password file handling
if *f.PasswordFile != "" {
data, err := os.ReadFile(*f.PasswordFile)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to read password file: %w", err))
return 1
}
cfg.Auth.Password = strings.TrimSpace(string(data))
} else if passwordEnv := os.Getenv("GOGET_PASSWORD"); passwordEnv != "" {
cfg.Auth.Password = passwordEnv
}
// Quiet mode: suppress verbose output
if *f.Quiet {
cfg.Verbose = false
}
// Configure structured logger based on CLI flags.
logLevel := log.InfoLevel
if cfg.Verbose {
logLevel = log.VerboseLevel
}
if cfg.Debug {
logLevel = log.DebugLevel
}
if *f.Quiet {
logLevel = log.WarnLevel
}
log.DefaultLogger = log.New(os.Stderr,
log.WithLevel(logLevel),
log.WithColor(cfg.ShouldUseColors()),
log.WithJSON(*f.JSON),
)
// Restart: delete existing file before download
if *f.Restart && *f.Output != "" {
os.Remove(*f.Output)
output.CleanupResumeFiles(*f.Output)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
log.Warn("Interrupted by user.")
cancel()
// Also cancel the app-level signal context so run() can
// observe the interruption and return 130.
a.sigCancel()
}()
// DNS servers
if *f.DNSServers != "" {
dnsServers := transport.ParseDNSServers(*f.DNSServers)
if len(dnsServers) > 0 {
cfg.DNSServers = dnsServers
}
}
if *f.Schedule != "" {
if code := waitForSchedule(*f.Schedule, cfg, sigChan); code != 0 {
a.sigCancel()
return code
}
}
// Checksum file handling
if *f.ChecksumFile != "" && *f.Checksum == "" {
expectedChecksum, err := parseChecksumFile(*f.ChecksumFile, *f.Output)
if err != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed to read checksum from file: %v", err))
} else if expectedChecksum != "" {
*f.Checksum = expectedChecksum
if cfg.Verbose {
cli.PrintInfo(os.Stderr, fmt.Sprintf("Using checksum from file: %s", expectedChecksum))
}
}
}
httpClient, err := protocolreg.RegisterAll(protocol.GlobalRegistry)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to register protocols: %w", err))
return 1
}
// HSTS cache
hstsCache := hsts.NewCache()
hstsPath := *f.HSTSFile
if hstsPath == "" {
hstsPath = hsts.DefaultCachePath()
}
if hstsPath != "" {
if err := hstsCache.Load(hstsPath); err != nil && cfg.Verbose {
cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to load hsts cache: %v", err))
}
httpClient.SetHSTSCache(hstsCache)
}
var cookieJar *cookie.Jar
cookieJarPath := cfg.CookieJar
if cookieJarPath != "" {
cookieJar, err = cookie.NewJar()
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to create cookie jar: %w", err))
return 1
}
if err := cookieJar.LoadFromFile(cookieJarPath); err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to load cookies: %w", err))
return 1
}
if cfg.Verbose {
cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Loaded cookies from %s", cookieJarPath))
}
}
if cfg.Verbose {
safeURL := core.SafeURL(parsedURL)
log.Infof("Goget %s", core.Version)
log.Infof("URL: %s", safeURL)
if cfg.Auth.Username != "" {
log.Infof("Auth: %s:%s", cfg.Auth.Username, auth.MaskPassword(cfg.Auth.Password))
}
if cookieJar != nil {
log.Infof("Cookies: %d loaded", cookieJar.GetCookieCount(parsedURL))
}
if *f.Mirror {
log.Info("Mirror: enabled (infinite depth, convert links)")
}
if *f.Recursive {
log.Infof("Recursive: enabled (max-depth=%d)", cfg.Recursive.MaxDepth)
}
if *f.Extract {
extractInfo := "enabled"
if cfg.Extract.OutputDir != "" {
extractInfo = fmt.Sprintf("enabled → %s", cfg.Extract.OutputDir)
}
log.Infof("Extract: %s", extractInfo)
}
if *f.Timeout > 0 {
log.Infof("Timeout: %v (explicit)", *f.Timeout)
} else {
log.Infof("Timeout: %v (auto)", cfg.Timeout)
}
if cfg.Parallel > 0 {
log.Infof("Parallel: %d connections", cfg.Parallel)
} else {
log.Info("Parallel: auto")
}
if cfg.MaxSpeed > 0 {
log.Infof("Max speed: %s/s", format.Bytes(cfg.MaxSpeed))
}
// DNS server information
if len(cfg.DNSServers) > 0 {
log.Infof("DNS: %v", cfg.DNSServers)
}
}
if !*f.Recursive && *f.Resume && *f.Output != "" {
canResume, meta, err := output.CanResume(*f.Output, parsedURL.String())
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("resume check failed: %w", err))
if !*f.Overwrite {
return 1
}
output.CleanupResumeFiles(*f.Output)
if cfg.Verbose {
cli.PrintProgressInfo(os.Stderr, "Starting fresh download (overwrite mode)")
}
} else if canResume && meta != nil {
if cfg.Verbose {
cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Resuming download: %s already downloaded", format.Bytes(meta.Downloaded)))
}
}
}
// --no-clobber: skip if output file exists
if !*f.Recursive && *f.NoClobber && *f.Output != "" && *f.Output != "-" {
if _, err := os.Stat(*f.Output); err == nil {
if cfg.Verbose {
cli.PrintInfo(os.Stderr, fmt.Sprintf("File exists, skipping: %s", *f.Output))
}
return 0
}
}
// Default output filename: derive from the URL path when no
// --output flag is given. Matches curl/wget behaviour and prevents
// binary data being dumped to stdout. Recursive downloads, stdout
// mode (--output -), and content-disposition mode are exempt.
if *f.Output == "" && !*f.Recursive && !*f.ContentDisp {
name := filepath.Base(parsedURL.Path)
if name == "" || name == "/" || name == "." {
name = "index.html"
}
*f.Output = name
}
writer, progressBar := setupOutputWriter(f, cfg)
if writer == nil && !*f.Recursive {
// Error already printed by setupOutputWriter
return 1
}
if writer != nil {
defer writer.Close()
}
httpCfg := setupHTTPConfig(f, cfg, cookieJar, parsedURL)
if *f.Recursive {
httpCfg.Recursive.Enabled = true
httpCfg.Recursive.MaxDepth = cfg.Recursive.MaxDepth
httpCfg.Recursive.FollowExternal = cfg.Recursive.FollowExternal
httpCfg.Recursive.ExcludePatterns = cfg.Recursive.ExcludePatterns
httpCfg.Output.OutputDir = *f.Output
if *f.Accept != "" {
httpCfg.Recursive.Accept = *f.Accept
}
if *f.NoParent {
httpCfg.Recursive.NoParent = true
}
if *f.SpanHosts {
httpCfg.Recursive.SpanHosts = true
httpCfg.Recursive.FollowExternal = true
}
if *f.PageRequisites {
httpCfg.Recursive.PageRequisites = true
}
if *f.Wait != "" {
if d, err := time.ParseDuration(*f.Wait); err == nil {
httpCfg.Recursive.Wait = d
}
}
if *f.RandomWait {
httpCfg.Recursive.RandomWait = true
}
if *f.ConvertLinks {
httpCfg.Recursive.ConvertLinks = true
}
}
if *f.DryRun {
httpCfg.Recursive.DryRun = true
}
if *f.Mirror {
httpCfg.Recursive.Mirror = true
httpCfg.Recursive.ConvertLinks = !*f.NoConvertLinks
httpCfg.Recursive.MirrorAssets = !*f.NoMirrorAssets
httpCfg.Recursive.RespectRobots = !*f.NoRobots
httpCfg.Output.OutputDir = *f.Output
httpCfg.Recursive.Enabled = false // Mirror takes precedence
// Parse rate limit
if *f.RateLimit != "" {
rate, perr := parseRateLimit(*f.RateLimit)
if perr != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("ignoring --max-speed: %v; download will be unthrottled", perr))
}
httpCfg.RateLimit = rate
}
}
if *f.Extract {
httpCfg.Output.AutoExtract = true
httpCfg.Output.ExtractDir = cfg.Extract.OutputDir
}
if *f.Proxy != "" {
proxyCfg, perr := transport.ParseProxyConfig(*f.Proxy)
if perr == nil {
httpCfg.Transport.Proxy = proxyCfg
}
}
proto, err := protocol.Get(parsedURL)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("protocol error: %w", err))
return 1
}
var resumeInfo *core.ResumeInfo
if !*f.Recursive && *f.Resume && *f.Output != "" {
meta, _ := output.LoadResumeMetadata(*f.Output)
if meta != nil {
resumeInfo, _ = output.GetResumeInfo(meta, *f.Output)
}
}
req := &core.DownloadRequest{
URL: parsedURL,
Output: *f.Output,
Writer: writer,
Resume: *f.Resume,
Timeout: cfg.GetEffectiveTimeout(-1, *f.Timeout),
Verbose: cfg.Verbose,
DebugTransport: cfg.Debug,
Checksum: *f.Checksum,
Headers: make(map[string]string),
Proxy: cfg.Proxy,
AutoDecompress: cfg.AutoDecompress,
ProgressCallback: nil,
ResumeInfo: resumeInfo,
Parallel: httpCfg.Parallel,
Recursive: *f.Recursive,
RecursiveParallel: httpCfg.Recursive.Parallel,
DryRun: httpCfg.Recursive.DryRun,
MaxDepth: cfg.Recursive.MaxDepth,
AcceptPatterns: parseStringSlice(*f.Accept),
RejectPatterns: httpCfg.Recursive.Reject,
KeepTimestamps: *f.KeepTimestamps,
Ctx: ctx,
SSHKnownHosts: *f.KnownHosts,
SSHInsecure: *f.SSHInsecure,
}
// Configure TLS for WebDAV if supported.
if webdavProto, ok := proto.(*webdavProtocol); ok {
tlsCfg := httpCfg.Transport.TLS
insecure := httpCfg.Transport.TLS != nil && httpCfg.Transport.TLS.InsecureSkipVerify
webdavProto.ConfigureTLS(tlsCfg, insecure)
if cfg.Proxy != "" {
webdavProto.ConfigureProxy(cfg.Proxy)
}
}
if cfg.Verbose {
if *f.Recursive {
cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Starting recursive download: %s", core.SafeURL(parsedURL)))
} else {
cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Starting download: %s", core.SafeURL(parsedURL)))
}
}
var result *core.DownloadResult
var downloadErr error
// Wire the progress bar to the download via a callback so the
// user sees live speed and ETA during the transfer, not just a
// static "Starting download" line. The parallel downloader
// aggregates per-chunk progress; the sequential path ignores
// the callback when the bar was not created.
if progressBar != nil && !*f.Recursive {
req.ProgressCallback = func(current, total int64, speed float64) {
progressBar.SetTotal(total)
progressBar.Update(current)
}
}
startTime := time.Now()
result, downloadErr = proto.Download(ctx, req)
if progressBar != nil {
if downloadErr != nil {
progressBar.Cancel()
} else {
progressBar.Finish()
}
}
if downloadErr != nil {
cli.PrintError(os.Stderr, downloadErr)
return 1
}
return runPostDownload(f, cfg, result, cookieJar, cookieJarPath, hstsCache, writer, parsedURL, startTime)
}
cmd/goget/commands.go
+1022
View File
File diff suppressed because it is too large Load Diff
cmd/goget/completion.go
+185
View File
@@ -0,0 +1,185 @@
//go:build linux || freebsd
// +build linux freebsd
package main
func generateBashCompletion() string {
return `# goget bash completion
_goget() {
local cur prev opts
COMPREPLY=()
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
opts="--url --output --resume --overwrite --verbose --debug
--timeout --proxy --no-ipv4 --checksum --checksum-algo
--no-decompress --parallel --max-speed --username --password
--password-file --auth-type --cookie-jar --cookie --recursive --recursive-parallel --max-depth
--follow-external --mirror --no-convert-links --no-mirror-assets
--no-robots --rate-limit --exclude-pattern --extract --extract-dir
--strip-components --upload --upload-method --upload-file
--form-data --file-field --pgp-decrypt --pgp-verify --pgp-sig
--pgp-key --pgp-passphrase --oauth-client-id --oauth-client-secret
--oauth-token-url --oauth-auth-url --oauth-redirect-uri
--oauth-scopes --oauth-grant-type --oauth-access-token
--oauth-refresh-token --metalink --metalink-file --queue
--queue-list --queue-process --queue-clear --queue-file
--queue-priority --help --version --batch-file --quiet
--init-config --content-disposition --restart --json --info
--show-headers --keep-timestamps --dns-servers --max-retries
--no-redirect --retry-all-errors --ssl-key-log --completion --config"
case "${prev}" in
--url|--output|--proxy|--checksum|--password|--password-file|--cookie-jar)
return 0
;;
--timeout|--max-speed|--rate-limit)
return 0
;;
--batch-file|--metalink-file|--pgp-sig|--pgp-key|--pgp-passphrase)
return 0
;;
--dns-servers|--max-retries|--parallel|--recursive-parallel|--max-depth|--queue-priority)
return 0
;;
--completion)
COMPREPLY=($(compgen -W "bash zsh fish" -- ${cur}))
return 0
;;
--auth-type)
COMPREPLY=($(compgen -W "basic digest auto" -- ${cur}))
return 0
;;
--checksum-algo)
COMPREPLY=($(compgen -W "sha256 sha512 blake2b sha3-256 sha3-512 md5" -- ${cur}))
return 0
;;
--upload-method)
COMPREPLY=($(compgen -W "PUT POST" -- ${cur}))
return 0
;;
--exclude-pattern|--form-data|--oauth-scopes)
return 0
;;
esac
if [[ ${cur} == -* ]]; then
COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
fi
}
complete -F _goget goget
`
}
func generateZshCompletion() string {
return `#compdef goget
_goget() {
local -a opts
opts=(
'--url[URL to download]:url:_urls'
'--output[Output file]:file:_files'
'--verbose[Verbose output]'
'--debug[Debug mode]'
'--resume[Resume interrupted download]'
'--quiet[Suppress all output except errors]'
'--json[JSON progress output]'
'--info[Show file information without downloading]'
'--show-headers[Print HTTP response headers]'
'--keep-timestamps[Set file modification time from server]'
'--restart[Delete existing file and restart download]'
'--timeout[Timeout (0 = auto)]:timeout'
'--proxy[Proxy server URL]:proxy:_urls'
'--no-ipv4[Disable IPv4 fallback]'
'--checksum[Expected checksum]:checksum'
'--checksum-algo[Checksum algorithm]:algorithm:(sha256 sha512 blake2b sha3-256 sha3-512 md5)'
'--no-decompress[Disable automatic decompression]'
'--parallel[Number of parallel connections]:number'
'--max-speed[Max speed]:speed'
'--username[Username]:username'
'--password[Password]:password'
'--password-file[Read password from file]:file:_files'
'--auth-type[Auth type]:(basic digest auto)'
'--batch-file[Download URLs from file]:file:_files'
'--cookie[Cookie name=value]'
'--retry-all-errors[Retry on any HTTP error]'
'--ssl-key-log[TLS key log file]:file:_files'
'--max-retries[Maximum retries]:number'
'--dns-servers[Custom DNS servers]:dns'
'--no-redirect[Disable following redirects]'
'--completion[Generate completion]:shell:(bash zsh fish)'
'--content-disposition[Use server-provided filename]'
'--init-config[Generate default config file]'
'--help[Show help]'
'--version[Show version]'
)
_describe 'goget' opts
}
compdef _goget goget
`
}
func generateFishCompletion() string {
return `# goget fish completion
function __fish_goget_no_subcommand
for i in (commandline -opc)
if contains -- $i (__fish_goget_commands)
return 1
end
end
return 0
end
function __fish_goget_commands
echo url output resume overwrite verbose debug
echo timeout proxy no-ipv4 checksum checksum-algo
echo no-decompress parallel max-speed username password
echo password-file auth-type cookie-jar cookie recursive recursive-parallel max-depth
echo retry-all-errors ssl-key-log
echo follow-external mirror no-convert-links no-mirror-assets
echo no-robots rate-limit exclude-pattern extract extract-dir
echo strip-components upload upload-method upload-file
echo form-data file-field pgp-decrypt pgp-verify pgp-sig
echo pgp-key pgp-passphrase help version batch-file quiet
echo init-config content-disposition restart json info
echo show-headers keep-timestamps dns-servers max-retries
echo no-redirect completion
end
# Flags
complete -c goget -l url -d "URL to download" -r
complete -c goget -l output -d "Output file" -r
complete -c goget -l verbose -d "Verbose output"
complete -c goget -l debug -d "Debug mode"
complete -c goget -l resume -d "Resume interrupted download"
complete -c goget -l quiet -d "Suppress all output except errors"
complete -c goget -l json -d "JSON progress output"
complete -c goget -l info -d "Show file information without downloading"
complete -c goget -l show-headers -d "Print HTTP response headers"
complete -c goget -l keep-timestamps -d "Set file modification time from server"
complete -c goget -l restart -d "Delete existing file and restart download"
complete -c goget -l init-config -d "Generate default config file"
complete -c goget -l content-disposition -d "Use server-provided filename"
complete -c goget -l no-redirect -d "Disable following redirects"
complete -c goget -l batch-file -d "Download URLs from file" -r
complete -c goget -l max-retries -d "Maximum retries" -r
complete -c goget -l dns-servers -d "Custom DNS servers" -r
complete -c goget -l completion -d "Generate completion" -r -xa "bash zsh fish"
complete -c goget -l timeout -d "Timeout (0 = auto)" -r
complete -c goget -l proxy -d "Proxy server URL" -r
complete -c goget -l checksum -d "Expected checksum" -r
complete -c goget -l checksum-algo -d "Algorithm" -r -xa "sha256 sha512 blake2b sha3-256 sha3-512 md5"
complete -c goget -l parallel -d "Parallel connections" -r
complete -c goget -l max-speed -d "Max speed" -r
complete -c goget -l username -d "Username" -r
complete -c goget -l password -d "Password" -r
complete -c goget -l password-file -d "Read password from file" -r
complete -c goget -l auth-type -d "Auth type" -r -xa "basic digest auto"
complete -c goget -l cookie -d "Cookie name=value" -r
complete -c goget -l retry-all-errors -d "Retry on any HTTP error"
complete -c goget -l ssl-key-log -d "TLS key log file" -r
complete -c goget -l help -d "Show help"
complete -c goget -l version -d "Show version"
`
}
cmd/goget/flags.go
+316
View File
@@ -0,0 +1,316 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"flag"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/config"
)
// headerFlags accumulates multiple --header values.
type headerFlags []string
func (h *headerFlags) String() string { return strings.Join(*h, ", ") }
func (h *headerFlags) Set(v string) error {
*h = append(*h, v)
return nil
}
// formFlags accumulates multiple --form values.
type formFlags []string
func (f *formFlags) String() string { return strings.Join(*f, ", ") }
func (f *formFlags) Set(v string) error {
*f = append(*f, v)
return nil
}
// Flags holds all CLI flag values.
type Flags struct {
URL *string
Output *string
Resume *bool
Overwrite *bool
Restart *bool
Verbose *bool
Quiet *bool
NoProgress *bool
JSON *bool
Debug *bool
Timeout *time.Duration
Proxy *string
NoIPv4 *bool
NoRedirect *bool
MaxRetries *int
DNSServers *string
Checksum *string
ChecksumAlgo *string
ChecksumFile *string
NoDecompress *bool
Parallel *int
MaxSpeed *string
Username *string
Password *string
PasswordFile *string
AuthType *string
CookieJar *string
ContentDisp *bool
ShowHeaders *bool
KeepTimestamps *bool
Recursive *bool
RecursiveParallel *int
MaxDepth *int
FollowExternal *bool
Mirror *bool
NoConvertLinks *bool
NoMirrorAssets *bool
NoRobots *bool
RateLimit *string
ExcludePattern *string
Extract *bool
ExtractDir *string
StripComponents *int
Upload *bool
UploadMethod *string
UploadFile *string
UploadFormData *string
UploadFileField *string
PGPDecrypt *bool
PGPVerify *bool
PGPSignature *string
PGPKey *string
PGPPassphrase *string
PGPPassphraseFile *string
OAuthClientID *string
OAuthClientSecret *string
OAuthTokenURL *string
OAuthAuthURL *string
OAuthRedirectURI *string
OAuthScopes *string
OAuthGrantType *string
OAuthAccessToken *string
OAuthRefreshToken *string
Metalink *bool
MetalinkFile *string
Queue *bool
QueueList *bool
QueueProcess *bool
QueueClear *bool
QueueFile *string
QueuePriority *int
BatchFile *string
InitConfig *bool
ConfigGet *string
ConfigSet *string
ConfigList *bool
ConfigUnset *string
Completion *string
Info *string
Benchmark *string
Schedule *string
BindInterface *string
KnownHosts *string
SSHInsecure *bool
DryRun *bool
OnComplete *string
GenManPage *bool
Help *bool
Version *bool
// curl/wget compatible flags
Headers headerFlags
Cookies headerFlags // --cookie, repeatable
Insecure *bool
CertFile *string
KeyFile *string
SSLKeyLogFile *string
Data *string
MaxTime *time.Duration
MaxFileSize *string
Accept *string
NoParent *bool
Spider *bool
WriteOut *string
ProgressStyle *string
WarcFile *string
TraceTime *bool
Form formFlags
PageRequisites *bool
SpanHosts *bool
AdjustExt *bool
NoPrivate *bool // deprecated, use AllowPrivate inversely
Wait *string
RandomWait *bool
ConvertLinks *bool
Fail *bool
Compressed *bool
Referer *string
RetryDelay *time.Duration
PinnedCert *string
AllowPrivate *bool
Range *string
ConnectTimeout *time.Duration
Domains *string
Reject *string
CutDirs *int
PostFile *string
CACertificate *string
NoClobber *bool
RetryHTTPError *string
RetryAllErrors *bool
Glob *bool
InputFile *string
Timestamping *bool
BackupConverted *bool
Continue *bool
ProtoDirs *bool
HSTSFile *string
CreateDirs *bool
}
// defineFlags registers all CLI flags on the given FlagSet and returns them.
func defineFlags(fs *flag.FlagSet, cfg *config.Config) *Flags {
f := &Flags{
URL: fs.String("url", "", "URL to download (can be specified multiple times)"),
Output: fs.String("output", "", "Output file path"),
Resume: fs.Bool("resume", cfg.AutoResume, "Resume interrupted download"),
Overwrite: fs.Bool("overwrite", false, "Overwrite existing file"),
Restart: fs.Bool("restart", false, "Delete existing file and restart download from scratch"),
Verbose: fs.Bool("verbose", cfg.Verbose, "Verbose output"),
Quiet: fs.Bool("quiet", false, "Suppress all output except errors"),
NoProgress: fs.Bool("no-progress", false, "Hide progress bar (keep other output)"),
JSON: fs.Bool("json", false, "Output machine-readable progress as JSON lines"),
Debug: fs.Bool("debug", cfg.Debug, "Debug mode"),
Timeout: fs.Duration("timeout", 0, "Timeout (0 = auto)"),
Proxy: fs.String("proxy", cfg.Proxy, "Proxy server URL (http://, https://, socks5://, socks5h://)"),
NoIPv4: fs.Bool("no-ipv4", !cfg.IPv4Fallback, "Disable IPv4 fallback"),
NoRedirect: fs.Bool("no-redirect", false, "Disable following HTTP redirects"),
MaxRetries: fs.Int("max-retries", 0, "Maximum number of retries on failure (0 = default)"),
DNSServers: fs.String("dns-servers", "", "Custom DNS servers (comma-separated, e.g. 1.1.1.1,8.8.8.8)"),
Checksum: fs.String("checksum", "", "Expected checksum value"),
ChecksumAlgo: fs.String("checksum-algo", cfg.ChecksumAlgo, "Checksum algorithm (sha256, sha512, blake2b, sha3-256, sha3-512, md5)"),
ChecksumFile: fs.String("checksum-file", "", "Read checksum from file (SHA256SUMS format)"),
NoDecompress: fs.Bool("no-decompress", !cfg.AutoDecompress, "Disable automatic decompression"),
Parallel: fs.Int("parallel", cfg.Parallel, "Number of parallel connections (0 = auto)"),
MaxSpeed: fs.String("max-speed", "", "Max speed (e.g., 10MB/s, 0 = unlimited)"),
Username: fs.String("username", cfg.Auth.Username, "Username for HTTP Basic/Digest Auth"),
Password: fs.String("password", "", "Password for HTTP Basic/Digest Auth"),
PasswordFile: fs.String("password-file", "", "Read password from file"),
AuthType: fs.String("auth-type", cfg.Auth.AuthType, "Auth type: basic, digest, auto"),
CookieJar: fs.String("cookie-jar", cfg.CookieJar, "File for loading/saving cookies"),
ContentDisp: fs.Bool("content-disposition", false, "Use server-provided filename from Content-Disposition header"),
ShowHeaders: fs.Bool("show-headers", false, "Print HTTP response headers during download"),
KeepTimestamps: fs.Bool("keep-timestamps", false, "Set file modification time from Last-Modified header"),
Recursive: fs.Bool("recursive", cfg.Recursive.Enabled, "Recursive downloading"),
RecursiveParallel: fs.Int("recursive-parallel", cfg.Recursive.Parallel, "Number of concurrent file downloads in recursive mode (0 = sequential)"),
MaxDepth: fs.Int("max-depth", cfg.Recursive.MaxDepth, "Maximum recursion depth"),
FollowExternal: fs.Bool("follow-external", cfg.Recursive.FollowExternal, "Follow external domains"),
Mirror: fs.Bool("mirror", false, "Mirror entire website (infinite depth, convert links)"),
NoConvertLinks: fs.Bool("no-convert-links", false, "Don't convert links for local viewing in mirror mode"),
NoMirrorAssets: fs.Bool("no-mirror-assets", false, "Don't download assets (CSS, JS, images) in mirror mode"),
NoRobots: fs.Bool("no-robots", false, "Don't respect robots.txt in mirror mode"),
RateLimit: fs.String("rate-limit", "", "Rate limit (e.g., 100KB/s, 1MB/s, 0 = unlimited)"),
ExcludePattern: fs.String("exclude-pattern", strings.Join(cfg.Recursive.ExcludePatterns, ","), "Exclude URL patterns (comma-separated)"),
Extract: fs.Bool("extract", cfg.Extract.Enabled, "Auto-extract archive after download"),
ExtractDir: fs.String("extract-dir", cfg.Extract.OutputDir, "Directory for extracted files"),
StripComponents: fs.Int("strip-components", cfg.Extract.StripComponents, "Strip N path components during extraction"),
Upload: fs.Bool("upload", false, "Upload file instead of downloading"),
UploadMethod: fs.String("upload-method", "PUT", "Upload method: PUT or POST"),
UploadFile: fs.String("upload-file", "", "File to upload"),
UploadFormData: fs.String("form-data", "", "Form data (key=value, comma-separated)"),
UploadFileField: fs.String("file-field", "file", "Form field name for file upload"),
PGPDecrypt: fs.Bool("pgp-decrypt", false, "Decrypt PGP encrypted file after download"),
PGPVerify: fs.Bool("pgp-verify", false, "Verify PGP signature after download"),
PGPSignature: fs.String("pgp-sig", "", "Path to PGP signature file (.asc, .sig)"),
PGPKey: fs.String("pgp-key", "", "Path to PGP key file (public or private)"),
PGPPassphrase: fs.String("pgp-passphrase", "", "Passphrase for PGP private key"),
PGPPassphraseFile: fs.String("pgp-passphrase-file", "", "Read PGP passphrase from file"),
OAuthClientID: fs.String("oauth-client-id", cfg.Auth.OAuth.ClientID, "OAuth 2.0 Client ID"),
OAuthClientSecret: fs.String("oauth-client-secret", cfg.Auth.OAuth.ClientSecret, "OAuth 2.0 Client Secret"),
OAuthTokenURL: fs.String("oauth-token-url", cfg.Auth.OAuth.TokenURL, "OAuth 2.0 Token URL"),
OAuthAuthURL: fs.String("oauth-auth-url", cfg.Auth.OAuth.AuthURL, "OAuth 2.0 Authorization URL"),
OAuthRedirectURI: fs.String("oauth-redirect-uri", cfg.Auth.OAuth.RedirectURI, "OAuth 2.0 Redirect URI"),
OAuthScopes: fs.String("oauth-scopes", strings.Join(cfg.Auth.OAuth.Scopes, ","), "OAuth 2.0 Scopes (comma-separated)"),
OAuthGrantType: fs.String("oauth-grant-type", cfg.Auth.OAuth.GrantType, "OAuth 2.0 Grant Type (client_credentials, authorization_code, password, refresh_token)"),
OAuthAccessToken: fs.String("oauth-access-token", cfg.Auth.OAuth.AccessToken, "OAuth 2.0 Access Token (direct)"),
OAuthRefreshToken: fs.String("oauth-refresh-token", cfg.Auth.OAuth.RefreshToken, "OAuth 2.0 Refresh Token"),
Metalink: fs.Bool("metalink", false, "Treat URL as Metalink (multi-source download)"),
MetalinkFile: fs.String("metalink-file", "", "Path to local Metalink file"),
Queue: fs.Bool("queue", false, "Add to download queue instead of downloading immediately"),
QueueList: fs.Bool("queue-list", false, "List items in download queue"),
QueueProcess: fs.Bool("queue-process", false, "Process download queue"),
QueueClear: fs.Bool("queue-clear", false, "Clear completed items from queue"),
QueueFile: fs.String("queue-file", "", "Custom queue file path"),
QueuePriority: fs.Int("queue-priority", 5, "Priority for queue item (1-10)"),
BatchFile: fs.String("batch-file", "", "Download multiple URLs from a file (one per line)"),
InitConfig: fs.Bool("init-config", false, "Generate default config file"),
ConfigGet: fs.String("config-get", "", "Get a config value (e.g. --config-get timeout)"),
ConfigSet: fs.String("config-set", "", "Set a config value (e.g. --config-set timeout 5m)"),
ConfigList: fs.Bool("config-list", false, "List all config values"),
ConfigUnset: fs.String("config-unset", "", "Unset a config value (e.g. --config-unset timeout)"),
Completion: fs.String("completion", "", "Generate shell completion script (bash, zsh, fish)"),
Info: fs.String("info", "", "Show file information without downloading"),
Benchmark: fs.String("benchmark", "", "Benchmark download speed from URL (downloads first 10MB)"),
Schedule: fs.String("schedule", "", "Schedule download at specified time (e.g. '2026-06-01 02:00' or cron '0 2 * * *')"),
BindInterface: fs.String("bind-interface", "", "Bind to specific network interface"),
KnownHosts: fs.String("known-hosts", "", "Path to SSH known_hosts file for SFTP (default: ~/.ssh/known_hosts)"),
SSHInsecure: fs.Bool("ssh-insecure", false, "Skip SSH host key verification for SFTP"),
DryRun: fs.Bool("dry-run", false, "List URLs without saving files in recursive/mirror mode"),
OnComplete: fs.String("on-complete", "", "Run command after successful download"),
GenManPage: fs.Bool("generate-man-page", false, "Generate man page and exit"),
Help: fs.Bool("help", false, "Show help"),
Version: fs.Bool("version", false, "Show version"),
// curl/wget compatible flags
Insecure: fs.Bool("insecure", false, "Skip TLS certificate verification"),
CertFile: fs.String("cert", "", "Client certificate file (PEM, for mTLS)"),
KeyFile: fs.String("key", "", "Client private key file (PEM)"),
SSLKeyLogFile: fs.String("ssl-key-log", "", "File to log TLS session keys to (SSLKEYLOGFILE for Wireshark)"),
Data: fs.String("data", "", "Send POST data (e.g., --data 'key=value')"),
MaxTime: fs.Duration("max-time", 0, "Maximum total transfer time (0 = unlimited)"),
MaxFileSize: fs.String("max-filesize", "", "Maximum file size to download (e.g., 100MB)"),
Accept: fs.String("accept", "", "Accept patterns for recursive download (e.g., '*.pdf,*.html')"),
NoParent: fs.Bool("no-parent", false, "Do not ascend to parent directory in recursive download"),
Spider: fs.Bool("spider", false, "Check URL existence only (HEAD request, no download)"),
WriteOut: fs.String("write-out", "", "Output metadata after download (curl format: %{http_code}, %{size_download}, %{time_total})"),
ProgressStyle: fs.String("progress", "", "Progress display style: bar (default) or dot"),
WarcFile: fs.String("warc-file", "", "Write WARC output file for web archiving"),
TraceTime: fs.Bool("trace-time", false, "Print HTTP timing breakdown after download"),
PageRequisites: fs.Bool("page-requisites", false, "Download all assets (CSS, JS, images) needed to display the page"),
SpanHosts: fs.Bool("span-hosts", false, "Follow links to external domains in recursive download"),
AdjustExt: fs.Bool("adjust-extension", false, "Append .html extension to downloaded HTML files"),
Wait: fs.String("wait", "", "Wait between requests in recursive/mirror mode (e.g., 1s, 500ms)"),
RandomWait: fs.Bool("random-wait", false, "Randomize wait time (0.5x to 1.5x) in recursive/mirror mode"),
ConvertLinks: fs.Bool("convert-links", false, "Convert links for local viewing in recursive mode"),
Fail: fs.Bool("fail", false, "Exit with error on HTTP 4xx/5xx status codes"),
Compressed: fs.Bool("compressed", false, "Send Accept-Encoding header for auto-decompression"),
Referer: fs.String("referer", "", "Send Referer header"),
RetryDelay: fs.Duration("retry-delay", 0, "Delay between retry attempts"),
PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning"),
AllowPrivate: fs.Bool("allow-private", false, "Allow connections to private/internal IPs (disables SSRF protection)"),
Range: fs.String("range", "", "Download only a range of bytes (e.g., 0-1000)"),
ConnectTimeout: fs.Duration("connect-timeout", 0, "Timeout for connection establishment"),
Domains: fs.String("domains", "", "Only follow these domains in recursive mode (comma-separated)"),
Reject: fs.String("reject", "", "Reject URL patterns in recursive mode (glob, comma-separated)"),
CutDirs: fs.Int("cut-dirs", 0, "Strip N directory levels when saving files"),
PostFile: fs.String("post-file", "", "Send file contents as POST body"),
CACertificate: fs.String("ca-certificate", "", "Path to custom CA certificate"),
NoClobber: fs.Bool("no-clobber", false, "Skip download if output file already exists"),
RetryHTTPError: fs.String("retry-on-http-error", "", "Retry on these HTTP status codes (comma-separated, e.g. 503,429)"),
RetryAllErrors: fs.Bool("retry-all-errors", false, "Retry on any HTTP error (4xx/5xx) instead of just selected codes"),
Glob: fs.Bool("glob", false, "Expand [1-3], {a,b} patterns in URL"),
Timestamping: fs.Bool("timestamping", false, "Download only if remote file is newer"),
BackupConverted: fs.Bool("backup-converted", false, "Backup original file before link conversion"),
Continue: fs.Bool("continue", false, "Auto-resume download if output file exists"),
ProtoDirs: fs.Bool("protocol-directories", false, "Create .http/, .ftp/ subdirectories in recursive mode"),
InputFile: fs.String("input-file", "", "Read URLs from file (use - for stdin)"),
HSTSFile: fs.String("hsts-file", "", "HSTS cache file (default: ~/.config/goget/hsts)"),
CreateDirs: fs.Bool("create-dirs", true, "Create missing parent directories of --output (curl --create-dirs)"),
}
fs.Var(&f.Headers, "header", "Custom HTTP header (repeatable, e.g. --header 'Key: Value')")
fs.Var(&f.Cookies, "cookie", "Set cookie (repeatable, e.g. --cookie 'name=value')")
fs.Var(&f.Form, "form", "Multipart form data (repeatable, e.g. --form 'field=value' or --form 'field=@file')")
return f
}
cmd/goget/main.go
+356
View File
@@ -0,0 +1,356 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"context"
"fmt"
"net/url"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"codeberg.org/petrbalvin/goget/internal/archive"
"codeberg.org/petrbalvin/goget/internal/cli"
"codeberg.org/petrbalvin/goget/internal/config"
"codeberg.org/petrbalvin/goget/internal/cookie"
"codeberg.org/petrbalvin/goget/internal/core"
format "codeberg.org/petrbalvin/goget/internal/format"
"codeberg.org/petrbalvin/goget/internal/hsts"
"codeberg.org/petrbalvin/goget/internal/output"
"codeberg.org/petrbalvin/goget/internal/protocol/http"
"codeberg.org/petrbalvin/goget/internal/protocol/webdav"
"codeberg.org/petrbalvin/goget/internal/warc"
"golang.org/x/net/idna"
)
func main() {
os.Exit(run())
}
// signalContext returns a context that is cancelled when SIGINT or SIGTERM
// is received. Extracted as a helper so it can be unit-tested in
// isolation (testing the real signal flow inside run() would race with
// other tests in the same package).
func signalContext() (context.Context, context.CancelFunc) {
return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
}
// run is a thin wrapper that delegates to the app struct's run method.
func run() int {
var a app
return a.run()
}
// setupOutputWriter creates the output writer and progress bar for non-recursive downloads.
func setupOutputWriter(f *Flags, cfg *config.Config) (*output.Writer, *cli.ProgressBar) {
var progressBar *cli.ProgressBar
if cfg.Verbose && !*f.Recursive && !*f.NoProgress {
progressCfg := cli.DefaultProgressBarConfig()
progressCfg.Total = -1
progressCfg.Colors = cfg.ShouldUseColors()
progressBar = cli.NewProgressBar(progressCfg)
}
var writer *output.Writer
if !*f.Recursive {
writerCfg := output.DefaultWriterConfig()
writerCfg.Output = *f.Output
writerCfg.Resume = *f.Resume
writerCfg.Verbose = cfg.Verbose
writerCfg.Atomic = !*f.Overwrite
writerCfg.CreateDirs = *f.CreateDirs
if progressBar != nil || *f.JSON || *f.ProgressStyle == "dot" {
writerCfg.ProgressCallback = func(current, total int64, speed float64) {
if *f.ProgressStyle == "dot" {
// Wget-style dot progress: print dot per 100KB
if current > 0 && current%(100*1024) < 32*1024 {
fmt.Fprintf(os.Stderr, ".")
}
}
if *f.JSON {
// JSON-lines progress output
eta := ""
if total > 0 && current < total && speed > 0 {
remaining := float64(total-current) / speed
eta = fmt.Sprintf(`,"eta_ms":%d`, int64(remaining*1000))
}
totalJSON := "null"
if total > 0 {
totalJSON = fmt.Sprintf("%d", total)
}
fmt.Printf(`{"type":"progress","downloaded":%d,"total":%s,"speed":%d%s}`+"\n",
current, totalJSON, int64(speed), eta)
}
if progressBar != nil && total > 0 {
progressBar.Update(current)
}
}
}
var err error
writer, err = output.NewWriter(writerCfg)
if err != nil {
cli.PrintError(os.Stderr, err)
return nil, progressBar
}
}
return writer, progressBar
}
// setupHTTPConfig populates an http.Config from CLI flags, parsed config, and cookie jar.
func setupHTTPConfig(f *Flags, cfg *config.Config, cookieJar *cookie.Jar, parsedURL *url.URL) *http.Config {
httpCfg := http.DefaultConfig()
ApplyFlags(httpCfg, f, cfg, cookieJar, parsedURL)
return httpCfg
}
// runPostDownload handles WARC, cookie saving, checksum, PGP, write-out, trace-time,
// on-complete hook, verbose output, and HSTS save. Returns the exit code.
func runPostDownload(
f *Flags,
cfg *config.Config,
result *core.DownloadResult,
cookieJar *cookie.Jar,
cookieJarPath string,
hstsCache *hsts.Cache,
writer *output.Writer,
parsedURL *url.URL,
startTime time.Time,
) int {
if !*f.Recursive {
if cerr := writer.Close(); cerr != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to close writer: %v\n", cerr)
}
// Adjust extension: append .html to HTML files without extension
if *f.AdjustExt && *f.Output != "" {
if filepath.Ext(*f.Output) == "" {
newPath := *f.Output + ".html"
if err := os.Rename(*f.Output, newPath); err == nil {
*f.Output = newPath
result.OutputPath = newPath
if cfg.Verbose {
cli.PrintInfo(os.Stderr, fmt.Sprintf("Adjusted extension: %s", newPath))
}
}
}
}
}
// Dot progress newline
if *f.ProgressStyle == "dot" {
fmt.Fprintf(os.Stderr, "\n")
}
// WARC writing
if *f.WarcFile != "" && !*f.Recursive && *f.Output != "" {
w := warc.NewWriter(*f.WarcFile)
if err := w.WriteRecordFile(core.SafeURL(parsedURL), *f.Output); err != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed to write WARC: %v", err))
} else if cfg.Verbose {
cli.PrintInfo(os.Stderr, fmt.Sprintf("WARC record saved to %s", *f.WarcFile))
}
}
if cookieJar != nil && cookieJarPath != "" {
if err := cookieJar.SaveToFile(cookieJarPath); err != nil {
if cfg.Verbose {
fmt.Fprintf(os.Stderr, "Warning: failed to save cookies: %v\n", err)
}
} else if cfg.Verbose {
cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Saved cookies to %s", cookieJarPath))
}
}
if !*f.Recursive && *f.Checksum != "" && *f.Output != "" {
ok, err := verifyChecksum(*f.Output, *f.Checksum, cfg.ChecksumAlgo, cfg.Verbose)
if err != nil {
cli.PrintError(os.Stderr, err)
return 1
}
if !ok {
return 1
}
}
// PGP passphrase from file (more secure than CLI flag)
if *f.PGPPassphraseFile != "" && *f.PGPPassphrase == "" {
data, err := os.ReadFile(*f.PGPPassphraseFile)
if err != nil {
cli.PrintError(os.Stderr, fmt.Errorf("failed to read pgp passphrase file: %w", err))
return 1
}
pass := strings.TrimSpace(string(data))
f.PGPPassphrase = &pass
}
// PGP verification and/or decryption
if !*f.Recursive && (*f.PGPVerify || *f.PGPDecrypt) && *f.Output != "" {
ok, err := verifyPGP(*f.Output, *f.PGPVerify, *f.PGPDecrypt, f.PGPKey, f.PGPPassphrase, f.PGPSignature, cfg.Verbose)
if err != nil {
cli.PrintError(os.Stderr, err)
return 1
}
if !ok {
return 1
}
}
duration := time.Since(startTime)
// JSON completion event
if *f.JSON && !*f.Recursive {
fmt.Printf(`{"type":"complete","downloaded":%d,"total":%d,"speed":%d,"elapsed_ms":%d,"output":"%s"}`+"\n",
result.BytesDownloaded, result.TotalSize, int64(result.Speed), duration.Milliseconds(), result.OutputPath)
}
// On-complete hook — run command directly without shell to avoid injection
if *f.OnComplete != "" && !*f.Recursive {
if cfg.Verbose {
cli.PrintInfo(os.Stderr, fmt.Sprintf("Running: %s", *f.OnComplete))
}
// Split command into executable and arguments (basic shell-like tokenization)
tokens := splitCommand(*f.OnComplete)
if len(tokens) == 0 {
cli.PrintWarning(os.Stderr, "On-complete hook: empty command")
} else {
cmd := exec.Command(tokens[0], tokens[1:]...)
cmd.Env = append(os.Environ(),
fmt.Sprintf("GOGET_OUTPUT=%s", result.OutputPath),
fmt.Sprintf("GOGET_SIZE=%d", result.BytesDownloaded),
fmt.Sprintf("GOGET_URL=%s", core.SafeURL(parsedURL)),
)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("On-complete hook failed: %v", err))
}
}
}
if cfg.Verbose {
fmt.Fprintf(os.Stderr, "\n")
if *f.Recursive {
cli.PrintProgressSuccess(os.Stderr, fmt.Sprintf("Recursive download complete in %v", duration.Round(time.Second)))
fmt.Fprintf(os.Stderr, " Files downloaded: %d\n", result.ChunksCount)
fmt.Fprintf(os.Stderr, " Total size: %s\n", format.Bytes(result.BytesDownloaded))
fmt.Fprintf(os.Stderr, " Average speed: %s/s\n", format.Speed(result.Speed))
fmt.Fprintf(os.Stderr, " Output directory: %s\n", result.OutputPath)
} else {
resumeNote := ""
if result.Resumed {
resumeNote = " [resumed]"
}
parallelNote := ""
if result.Parallel {
parallelNote = fmt.Sprintf(" [%d parallel chunks]", result.ChunksCount)
}
extractNote := ""
if *f.Extract && archive.IsArchiveFormat(*f.Output) {
extractNote = " [auto-extracted]"
}
cli.PrintProgressSuccess(os.Stderr, fmt.Sprintf("Downloaded %s in %v (%s)%s%s%s",
format.Bytes(result.BytesDownloaded),
duration.Round(time.Millisecond),
format.Speed(result.Speed),
resumeNote,
parallelNote,
extractNote))
fmt.Fprintf(os.Stderr, " Protocol: %s | IPv%d | Saved: %s\n",
result.Protocol, result.IPVersion, result.OutputPath)
}
}
// Write-out (curl-compatible metadata output)
if *f.WriteOut != "" && !*f.Recursive {
handleWriteOut(*f.WriteOut, result, result.BytesDownloaded, duration, result.HTTPStatusCode)
}
// Trace timing output
if *f.TraceTime && result.TraceTimings != nil {
fmt.Fprintf(os.Stderr, "Timing breakdown: %s\n", result.TraceTimings.String())
}
// Save HSTS cache
if hstsCache != nil {
if err := hstsCache.Save(); err != nil && cfg.Verbose {
cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to save hsts cache: %v", err))
}
}
return 0
}
// splitCommand splits a shell-like command string into tokens respecting double quotes.
// Used to parse --on-complete and --on-complete-url without invoking a shell.
func splitCommand(raw string) []string {
var tokens []string
current := ""
inQuote := false
for i := 0; i < len(raw); i++ {
c := raw[i]
switch {
case c == '"':
inQuote = !inQuote
case c == ' ' || c == '\t':
if inQuote {
current += string(c)
} else if current != "" {
tokens = append(tokens, current)
current = ""
}
default:
current += string(c)
}
}
if current != "" {
tokens = append(tokens, current)
}
return tokens
}
// idnaURL converts internationalized domain names to Punycode.
func idnaURL(u *url.URL) *url.URL {
if u == nil || u.Hostname() == "" {
return u
}
host := u.Hostname()
ascii, err := idna.Lookup.ToASCII(host)
if err != nil || ascii == host {
return u
}
// Replace hostname with Punycode version
if u.Port() != "" {
ascii = ascii + ":" + u.Port()
}
result := *u
result.Host = ascii
return &result
}
// webdavProtocol is a type alias for WebDAV protocol handler.
type webdavProtocol = webdav.Protocol
// parseStringSlice parses comma-separated string into a slice of strings.
func parseStringSlice(s string) []string {
if s == "" {
return nil
}
var parts []string
for _, p := range strings.Split(s, ",") {
p = strings.TrimSpace(p)
if p != "" {
parts = append(parts, p)
}
}
return parts
}
cmd/goget/main_test.go
+381
View File
@@ -0,0 +1,381 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"context"
"flag"
"os"
"path/filepath"
"strings"
"testing"
"time"
"codeberg.org/petrbalvin/goget/internal/config"
)
func TestExtractConfigPath(t *testing.T) {
tests := []struct {
name string
args []string
expected string
}{
{"no config flag", []string{"--url", "https://example.com"}, ""},
{"--config with value", []string{"--config", "/path/to/config.json"}, "/path/to/config.json"},
{"--config=value", []string{"--config=/path/to/config.json"}, "/path/to/config.json"},
{"config after url", []string{"--url", "https://example.com", "--config", "/custom/config.json"}, "/custom/config.json"},
{"empty args", []string{}, ""},
{"--config without value", []string{"--config"}, ""},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result := extractConfigPath(tc.args)
if result != tc.expected {
t.Errorf("extractConfigPath(%v) = %q, want %q", tc.args, result, tc.expected)
}
})
}
}
func TestParseRateLimit(t *testing.T) {
tests := []struct {
input string
expected int64
}{
{"", 0},
{"0", 0},
{"100", 100},
{"1KB/s", 1000},
{"1MB/s", 1000000},
{"1GB/s", 1000000000},
{"500KB/s", 500000},
{"10MB/s", 10000000},
}
for _, tc := range tests {
t.Run(tc.input, func(t *testing.T) {
result, err := parseRateLimit(tc.input)
if err != nil {
t.Errorf("parseRateLimit(%q) returned unexpected error: %v", tc.input, err)
}
if result != tc.expected {
t.Errorf("parseRateLimit(%q) = %d, want %d", tc.input, result, tc.expected)
}
})
}
}
// TestParseRateLimitInvalid is a regression guard for the BACKLOG entry
// "`parseRateLimit` silently returns 0 on parse error". The previous
// implementation fell back to 0 without telling the caller, so a typo
// like "1oMB/s" (letter 'o' instead of digit '0') silently disabled rate
// limiting. The fixed implementation returns a non-nil error so the caller
// can warn the user.
func TestParseRateLimitInvalid(t *testing.T) {
tests := []struct {
name string
input string
}{
{"letter_o_instead_of_zero", "1oMB/s"},
{"pure_garbage", "abc"},
{"trailing_unit_only", "MB"},
{"only_unit_letter", "B"},
{"whitespace_around_digits", " abc "},
{"multiple_dots", "1.2.3MB"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result, err := parseRateLimit(tc.input)
if err == nil {
t.Errorf("parseRateLimit(%q) = %d, nil err; want non-nil error", tc.input, result)
}
if result != 0 {
t.Errorf("parseRateLimit(%q) = %d on error, want 0", tc.input, result)
}
})
}
}
func TestExtractURLs(t *testing.T) {
tests := []struct {
name string
args []string
expected int
}{
{"single url", []string{"--url", "https://example.com/file.zip"}, 1},
{"multiple urls", []string{"--url", "https://a.com", "--url", "https://b.com"}, 2},
{"with other flags", []string{"--url", "https://a.com", "--verbose", "--url", "https://b.com"}, 2},
{"url= format", []string{"--url=https://a.com", "--url=https://b.com"}, 2},
{"no url", []string{"--verbose", "--debug"}, 0},
{"empty args", []string{}, 0},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result := extractURLs(tc.args)
if len(result) != tc.expected {
t.Errorf("extractURLs(%v) returned %d URLs, want %d", tc.args, len(result), tc.expected)
}
})
}
}
func TestBatchFileURLParsing(t *testing.T) {
// Test that the URL parsing logic in handleBatchFile would work
// by testing the individual steps
content := `https://example.com/file1.zip
https://example.com/file2.zip
# This is a comment
// This is also a comment
https://example.com/file3.zip`
lines := strings.Split(strings.TrimSpace(content), "\n")
var urls []string
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") {
continue
}
urls = append(urls, line)
}
if len(urls) != 3 {
t.Errorf("Expected 3 URLs, got %d: %v", len(urls), urls)
}
}
// TestConfigSetFilePermissions is a regression guard for the BACKLOG
// entry "os.WriteFile with 0644 on config containing OAuth tokens".
// The previous implementation used 0644, which leaves the config file
// (and any embedded access_token / refresh_token) world-readable on
// multi-user systems. After the fix, both handleConfigSet and
// handleConfigUnset must write the file with mode 0600.
func TestConfigFilePermissions(t *testing.T) {
tests := []struct {
name string
run func(t *testing.T, configPath string)
}{
{
name: "handleConfigSet writes 0600",
run: func(t *testing.T, configPath string) {
if rc := handleConfigSet(configPath, "timeout", []string{
"--config-set", "timeout", "30",
}); rc != 0 {
t.Fatalf("handleConfigSet returned %d", rc)
}
},
},
{
name: "handleConfigUnset writes 0600",
run: func(t *testing.T, configPath string) {
if rc := handleConfigSet(configPath, "timeout", []string{
"--config-set", "timeout", "30",
}); rc != 0 {
t.Fatalf("seed handleConfigSet returned %d", rc)
}
if rc := handleConfigUnset(configPath, "timeout"); rc != 0 {
t.Fatalf("handleConfigUnset returned %d", rc)
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.toml")
// Seed a minimal config so config.Load succeeds.
seed := []byte("proxy = \"\"\nuser_agent = \"\"\ntimeout = 0\n")
if err := os.WriteFile(configPath, seed, 0600); err != nil {
t.Fatalf("seed write: %v", err)
}
tc.run(t, configPath)
info, err := os.Stat(configPath)
if err != nil {
t.Fatalf("stat: %v", err)
}
perm := info.Mode().Perm()
// Mask out the umask bits by checking against 0600; the file
// must not be group- or world-readable.
if perm != 0600 {
t.Errorf("config file mode = %04o, want 0600", perm)
}
})
}
}
func TestNextCronTime(t *testing.T) {
// Use a fixed reference time for deterministic tests (UTC)
ref, _ := time.Parse("2006-01-02 15:04", "2026-06-01 10:00")
ref = ref.UTC()
t.Run("every minute", func(t *testing.T) {
next, err := nextCronTime("* * * * *", ref)
if err != nil {
t.Fatal(err)
}
expected := ref.Truncate(time.Minute).Add(time.Minute)
if !next.Equal(expected) {
t.Errorf("got %v, want %v", next, expected)
}
})
t.Run("specific hour and minute", func(t *testing.T) {
next, err := nextCronTime("30 14 * * *", ref)
if err != nil {
t.Fatal(err)
}
expected := time.Date(2026, 6, 1, 14, 30, 0, 0, time.UTC)
if !next.Equal(expected) {
t.Errorf("got %v, want %v", next, expected)
}
})
t.Run("daily at midnight", func(t *testing.T) {
next, err := nextCronTime("0 0 * * *", ref)
if err != nil {
t.Fatal(err)
}
expected := time.Date(2026, 6, 2, 0, 0, 0, 0, time.UTC)
if !next.Equal(expected) {
t.Errorf("got %v, want %v", next, expected)
}
})
t.Run("specific day of week", func(t *testing.T) {
// June 1, 2026 is Monday (1)
next, err := nextCronTime("0 9 * * 3", ref) // Wednesday
if err != nil {
t.Fatal(err)
}
if next.Weekday() != time.Wednesday {
t.Errorf("expected Wednesday, got %v", next.Weekday())
}
})
t.Run("comma separated hours", func(t *testing.T) {
next, err := nextCronTime("0 8,16 * * *", ref)
if err != nil {
t.Fatal(err)
}
expected := time.Date(2026, 6, 1, 16, 0, 0, 0, time.UTC)
if !next.Equal(expected) {
t.Errorf("got %v, want %v", next, expected)
}
})
t.Run("step values", func(t *testing.T) {
next, err := nextCronTime("*/15 * * * *", ref)
if err != nil {
t.Fatal(err)
}
if next.Minute()%15 != 0 {
t.Errorf("minute not a multiple of 15: %d", next.Minute())
}
})
t.Run("invalid fields", func(t *testing.T) {
_, err := nextCronTime("invalid", ref)
if err == nil {
t.Error("expected error for invalid cron expression")
}
})
t.Run("too many fields", func(t *testing.T) {
_, err := nextCronTime("1 2 3 4 5 6", ref)
if err == nil {
t.Error("expected error for 6 fields")
}
})
}
// TestExitCodeForRun covers the contract that graceful shutdown
// surfaces exit code 130 to the shell when the user interrupts the
// process. A clean exit (no signal) returns the inner code unchanged;
// a non-zero inner code (real error) is preserved even if the signal
// context was also cancelled.
func TestExitCodeForRun(t *testing.T) {
t.Run("clean exit on no signal returns inner code 0", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if got := exitCodeForRun(ctx, 0); got != 0 {
t.Errorf("got %d, want 0", got)
}
})
t.Run("cancelled signal context maps clean exit to 130", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // simulate SIGINT/SIGTERM propagation
if got := exitCodeForRun(ctx, 0); got != 130 {
t.Errorf("got %d, want 130", got)
}
})
t.Run("non-zero inner code is preserved when signal is also cancelled", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
// Even with the signal context cancelled, a real error (e.g.
// download failure) should not be masked by the 130 mapping.
if got := exitCodeForRun(ctx, 7); got != 7 {
t.Errorf("got %d, want 7 (inner code preserved)", got)
}
})
t.Run("non-zero inner code is returned unchanged with no signal", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if got := exitCodeForRun(ctx, 42); got != 42 {
t.Errorf("got %d, want 42", got)
}
})
t.Run("nil context returns inner code unchanged", func(t *testing.T) {
if got := exitCodeForRun(context.TODO(), 5); got != 5 {
t.Errorf("got %d, want 5 (nil context safe)", got)
}
})
}
// TestCreateDirsFlagRegisteredAndDefaults is a regression guard for the
// BACKLOG entry "Add --create-dirs flag (curl / wget compatibility)". It
// verifies the flag is registered on the command-line FlagSet, defaults to
// true (matching curl / wget behaviour so common `goget --output
// ./downloads/file.zip` invocations just work), and can be flipped to
// false via the standard flag parser. We do not exercise the full
// download path here — that is covered by TestNewWriterCreateDirsMissingParent
// in internal/output/writer_test.go.
func TestCreateDirsFlagRegisteredAndDefaults(t *testing.T) {
cfg := config.DefaultConfig()
fs := newTestFlagSet("goget-test")
flags := defineFlags(fs, cfg)
if flags.CreateDirs == nil {
t.Fatal("CreateDirs flag was not registered on the FlagSet")
}
if !*flags.CreateDirs {
t.Errorf("CreateDirs default = false, want true (curl / wget parity)")
}
// Flip the flag through the standard flag parser to make sure the
// long name is wired up correctly and the value reaches the struct.
fs2 := newTestFlagSet("goget-test-2")
flags2 := defineFlags(fs2, cfg)
if err := fs2.Parse([]string{"--create-dirs=false"}); err != nil {
t.Fatalf("Parse --create-dirs=false: %v", err)
}
if *flags2.CreateDirs {
t.Errorf("CreateDirs after --create-dirs=false = true, want false")
}
}
// newTestFlagSet builds an isolated flag.FlagSet for CLI tests. We use
// flag.ContinueOnError so a bad flag in a test fails the test instead of
// calling os.Exit and tearing down the rest of the test binary.
func newTestFlagSet(name string) *flag.FlagSet {
return flag.NewFlagSet(name, flag.ContinueOnError)
}
cmd/goget/man_page.go
+217
View File
@@ -0,0 +1,217 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"fmt"
"codeberg.org/petrbalvin/goget/internal/core"
)
func handleGenManPage() int {
fmt.Printf(`.TH GOGET 1 "2026-06" "goget %s" "User Commands"
.SH NAME
goget \- modern IPv6-first command-line download utility
.SH SYNOPSIS
.B goget
\fB\-\-url\fR \fIURL\fR [\fIOPTIONS\fR]
.SH DESCRIPTION
goget is a modern replacement for curl and wget built on Go stdlib.
Features IPv6-first connections, parallel downloads, resume support,
compression, checksum verification, and recursive mirroring.
.SH OPTIONS
.SS "Target"
.TP
\fB\-\-url\fR=\fIURL\fR
URL to download (can be specified multiple times)
.SS "Basic Options"
.TP
\fB\-\-output\fR=\fIFILE\fR
Output filename (default: auto from URL)
.TP
\fB\-\-verbose\fR
Verbose output with progress bar (default: enabled)
.TP
\fB\-\-no-progress\fR
Hide progress bar (keep other output)
.TP
\fB\-\-quiet\fR
Suppress all output except errors
.TP
\fB\-\-debug\fR
Debug mode with detailed info
.TP
\fB\-\-resume\fR
Resume interrupted download
.TP
\fB\-\-restart\fR
Delete existing file and restart download
.TP
\fB\-\-overwrite\fR
Overwrite existing file
.TP
\fB\-\-no-decompress\fR
Disable automatic decompression
.TP
\fB\-\-parallel\fR=\fIN\fR
Parallel connections (0=auto, 1=sequential)
.TP
\fB\-\-json\fR
Output progress as JSON (for scripting)
.TP
\fB\-\-info\fR
Show file information without downloading
.TP
\fB\-\-benchmark\fR=\fIURL\fR
Benchmark download speed
.SS "Network"
.TP
\fB\-\-timeout\fR=\fIDURATION\fR
Connection timeout (0 = auto based on size)
.TP
\fB\-\-proxy\fR=\fIURL\fR
Proxy server URL (http://, https://, socks5://, socks5h://)
.TP
\fB\-\-dns-servers\fR=\fIIPs\fR
Custom DNS servers (comma-separated)
.TP
\fB\-\-max-retries\fR=\fIN\fR
Maximum number of retries on failure
.TP
\fB\-\-no-ipv4\fR
Disable IPv4 fallback
.TP
\fB\-\-no-redirect\fR
Disable following HTTP redirects
.TP
\fB\-\-show-headers\fR
Print HTTP response headers
.TP
\fB\-\-keep-timestamps\fR
Set file modification time from Last-Modified header
.TP
\fB\-\-content-disposition\fR
Use server-provided filename from Content-Disposition header
.SS "Authentication"
.TP
Credentials are loaded automatically from \fB~/.netrc\fR when no \fB\-\-username\fR is given.
.TP
\fB\-\-username\fR=\fIUSER\fR
Username for HTTP Basic/Digest Auth
.TP
\fB\-\-password\fR=\fIPASS\fR
Password for HTTP Basic/Digest Auth
.TP
\fB\-\-password-file\fR=\fIFILE\fR
Read password from file
.TP
\fB\-\-auth-type\fR=\fITYPE\fR
Auth type: basic, digest, auto
.TP
\fB\-\-cookie-jar\fR=\fIFILE\fR
File for loading/saving cookies
.TP
\fB\-\-cookie\fR=\fINAME=VALUE\fR
Set cookie from CLI (repeatable, curl-compatible)
.TP
\fB\-\-retry-all-errors\fR
Retry on any HTTP 4xx/5xx response
.TP
\fB\-\-ssl-key-log\fR=\fIFILE\fR
Log TLS session keys to file (SSLKEYLOGFILE for Wireshark)
.SS "Config"
.TP
\fB\-\-config\fR=\fIFILE\fR
Path to config file
.TP
\fB\-\-config-get\fR=\fIKEY\fR
Get a config value
.TP
\fB\-\-config-set\fR=\fIKEY\fR \fIVALUE\fR
Set a config value
.TP
\fB\-\-config-list\fR
List all config values
.TP
\fB\-\-config-unset\fR=\fIKEY\fR
Unset a config value
.TP
\fB\-\-init-config\fR
Generate default config file
.SS "Mirror & Recursive"
.TP
\fB\-\-mirror\fR
Mirror entire website (infinite depth)
.TP
\fB\-\-recursive\fR
Enable recursive downloading
.TP
\fB\-\-max-depth\fR=\fIN\fR
Maximum recursion depth
.SS "Scheduling & Hooks"
.TP
\fB\-\-schedule\fR=\fITIME\fR
Schedule download ("YYYY-MM-DD HH:MM" or cron "0 2 * * *")
.TP
\fB\-\-on-complete\fR=\fICMD\fR
Run shell command after successful download
.TP
\fB\-\-bind-interface\fR=\fIIFACE\fR
Bind to specific network interface
.SS "Other"
.TP
\fB\-\-batch-file\fR=\fIFILE\fR
Download multiple URLs from a file
.TP
\fB\-\-checksum-file\fR=\fIFILE\fR
Read expected checksum from SHA256SUMS file
.TP
\fB\-\-generate-man-page\fR
Generate this man page
.TP
\fB\-\-help\fR
Show help
.TP
\fB\-\-version\fR
Show version
.SH EXAMPLES
.TP
Download a file:
goget --url https://example.com/file.zip
.TP
Parallel download:
goget --url https://example.com/large.iso --parallel 4
.TP
Resume interrupted download:
goget --url https://example.com/file.zip --resume
.TP
Mirror website:
goget --url https://example.com --mirror --output ./mirror
.TP
Recursive download:
goget --url https://example.com --recursive --max-depth 2
.TP
JSON progress (for scripting):
goget --url https://example.com/file.zip --json --no-progress > progress.jsonl
.TP
Scheduled download at 2 AM:
goget --url https://example.com/daily.zip --schedule "0 2 * * *"
.TP
Run import script after download:
goget --url https://example.com/data.csv --on-complete "import.sh"
.SH FILES
.TP
~/.config/goget/config.json
Configuration file
.TP
<file>.goget.meta
Resume metadata file
.SH BUGS
Report bugs at https://codeberg.org/petrbalvin/goget/issues
.SH AUTHOR
Petr Balvin
.SH LICENSE
MIT License`, core.Version)
return 0
}
cmd/goget/schedule.go
+145
View File
@@ -0,0 +1,145 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"fmt"
"os"
"strconv"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/cli"
"codeberg.org/petrbalvin/goget/internal/config"
)
// waitForSchedule blocks until the scheduled time or returns an error code.
func waitForSchedule(schedule string, cfg *config.Config, sigChan <-chan os.Signal) int {
var scheduledTime time.Time
var isCron bool
var cronSchedule string
parsedTime, parseErr := time.Parse("2006-01-02 15:04", schedule)
if parseErr == nil {
scheduledTime = parsedTime
} else {
fields := strings.Fields(schedule)
if len(fields) == 5 {
isCron = true
cronSchedule = schedule
next, cerr := nextCronTime(cronSchedule, time.Now())
if cerr != nil {
cli.PrintError(os.Stderr, fmt.Errorf("invalid schedule: %w (use 'YYYY-MM-DD HH:MM' or cron 'MIN HOUR DOM MON DOW')", cerr))
return 1
}
scheduledTime = next
} else {
cli.PrintError(os.Stderr, fmt.Errorf("invalid schedule format: %w (use 'YYYY-MM-DD HH:MM' or cron 'MIN HOUR DOM MON DOW')", parseErr))
return 1
}
}
delay := time.Until(scheduledTime)
if delay <= 0 {
cli.PrintError(os.Stderr, fmt.Errorf("scheduled time is in the past"))
return 1
}
if cfg.Verbose {
label := "Scheduled"
if isCron {
label = fmt.Sprintf("Cron (%s) scheduled", cronSchedule)
}
cli.PrintInfo(os.Stderr, fmt.Sprintf("%s download in %v (at %s)", label, delay.Round(time.Second), scheduledTime.Format(time.RFC3339)))
}
select {
case <-time.After(delay):
case <-sigChan:
return 1
}
return 0
}
// nextCronTime calculates the next scheduled time from a cron expression.
func nextCronTime(expr string, from time.Time) (time.Time, error) {
fields := strings.Fields(expr)
if len(fields) != 5 {
return time.Time{}, fmt.Errorf("cron expression must have 5 fields, got %d", len(fields))
}
minute := fields[0]
hour := fields[1]
dom := fields[2]
month := fields[3]
dow := fields[4]
t := from.Truncate(time.Minute).Add(time.Minute)
deadline := from.AddDate(2, 0, 0)
for t.Before(deadline) {
if matchCronField(minute, t.Minute(), 0, 59) &&
matchCronField(hour, t.Hour(), 0, 23) &&
matchCronField(dom, t.Day(), 1, 31) &&
matchCronField(month, int(t.Month()), 1, 12) &&
matchCronField(dow, int(t.Weekday()), 0, 6) {
return t, nil
}
t = t.Add(time.Minute)
}
return time.Time{}, fmt.Errorf("no matching time found within 2 years")
}
func matchCronField(pattern string, value, min, max int) bool {
if pattern == "*" {
return true
}
for _, part := range strings.Split(pattern, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if strings.Contains(part, "/") {
parts := strings.SplitN(part, "/", 2)
base := parts[0]
step, err := strconv.Atoi(parts[1])
if err != nil || step <= 0 {
continue
}
if base == "*" {
if (value-min)%step == 0 {
return true
}
continue
}
start, err := strconv.Atoi(base)
if err != nil {
continue
}
if value >= start && (value-start)%step == 0 {
return true
}
continue
}
if strings.Contains(part, "-") {
parts := strings.SplitN(part, "-", 2)
start, err1 := strconv.Atoi(parts[0])
end, err2 := strconv.Atoi(parts[1])
if err1 != nil || err2 != nil {
continue
}
if value >= start && value <= end {
return true
}
continue
}
if v, err := strconv.Atoi(part); err == nil && v == value {
return true
}
}
return false
}
cmd/goget/setup.go
+262
View File
@@ -0,0 +1,262 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"fmt"
stdhttp "net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
"codeberg.org/petrbalvin/goget/internal/cli"
"codeberg.org/petrbalvin/goget/internal/config"
"codeberg.org/petrbalvin/goget/internal/cookie"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol/http"
"codeberg.org/petrbalvin/goget/internal/transport"
)
// ApplyFlags populates the Config from CLI flags, global config, and cookie jar.
// It is the single function that maps all CLI flags to config fields — adding a
// new flag only requires adding one block here instead of hunting through
// scattered setup code.
func ApplyFlags(c *http.Config, f *Flags, cfg *config.Config, cookieJar *cookie.Jar, parsedURL *url.URL) {
// ── Transport defaults from global config ──
c.Transport.Timeout = cfg.GetEffectiveTimeout(-1, *f.Timeout)
c.Transport.Dialer.IPv4Fallback = cfg.IPv4Fallback
c.Transport.Dialer.Debug = cfg.Debug
c.UserAgent = cfg.UserAgent
c.Output.Verbose = cfg.Verbose
c.Parallel = &core.ParallelConfig{
Connections: cfg.GetParallelConnections(-1),
MinSize: 100 * 1024 * 1024,
MinChunkSize: 5 * 1024 * 1024,
MaxChunkSize: 50 * 1024 * 1024,
}
if cfg.MaxSpeed > 0 {
c.RateLimiter = transport.NewTokenBucket(cfg.MaxSpeed, cfg.MaxSpeed/10)
}
// ── DNS / interface ──
if len(cfg.DNSServers) > 0 {
c.Transport.DNSServers = cfg.DNSServers
}
if *f.BindInterface != "" {
c.Transport.BindInterface = *f.BindInterface
}
c.Transport.BlockPrivateIPs = !*f.AllowPrivate
// ── Authentication ──
if cfg.Auth.Username == "" && cfg.Auth.OAuth.AccessToken == "" {
if user, pass, ok := auth.LookupNetrc(parsedURL.Hostname()); ok {
cfg.Auth.Username = user
cfg.Auth.Password = pass
}
}
if cfg.Auth.Username != "" || cfg.Auth.OAuth.AccessToken != "" {
c.Auth.Username = cfg.Auth.Username
c.Auth.Password = cfg.Auth.Password
c.Auth.AuthType = cfg.Auth.AuthType
c.Auth.OAuth = &cfg.Auth.OAuth
}
if cookieJar != nil {
c.CookieJar = cookieJar
}
// ── TLS and certificates ──
if *f.Insecure && c.Transport.TLS != nil {
c.Transport.TLS.InsecureSkipVerify = true
}
if *f.CertFile != "" && c.Transport.TLS != nil {
c.Transport.TLS.CertFile = *f.CertFile
}
if *f.KeyFile != "" && c.Transport.TLS != nil {
c.Transport.TLS.KeyFile = *f.KeyFile
}
if *f.PinnedCert != "" && c.Transport.TLS != nil {
c.Transport.TLS.PinnedCertHash = *f.PinnedCert
}
if *f.CACertificate != "" && c.Transport.TLS != nil {
c.Transport.TLS.CACertFile = *f.CACertificate
}
if *f.SSLKeyLogFile != "" && c.Transport.TLS != nil {
c.Transport.TLS.KeyLogFile = *f.SSLKeyLogFile
}
if *f.ConnectTimeout > 0 && c.Transport.Dialer != nil {
c.Transport.Dialer.Timeout = *f.ConnectTimeout
}
// ── Retry and redirects ──
if *f.NoRedirect {
c.Transport.FollowRedirects = false
}
if *f.MaxRetries > 0 {
c.Transport.MaxRetries = *f.MaxRetries
if c.Transport.Retry != nil {
c.Transport.Retry.MaxRetries = *f.MaxRetries
}
}
if *f.RetryDelay > 0 {
c.Transport.RetryDelay = *f.RetryDelay
}
if *f.RetryHTTPError != "" {
for _, code := range strings.Split(*f.RetryHTTPError, ",") {
if code, err := strconv.Atoi(strings.TrimSpace(code)); err == nil {
c.Transport.RetryHTTPCodes = append(c.Transport.RetryHTTPCodes, code)
}
}
}
if *f.RetryAllErrors {
c.Transport.RetryAllErrors = true
}
// ── Timeouts and limits ──
if *f.ConnectTimeout > 0 {
c.Transport.ConnectTimeout = *f.ConnectTimeout
}
if *f.MaxTime > 0 {
c.Transport.MaxTime = *f.MaxTime
}
if *f.MaxFileSize != "" {
size, perr := parseRateLimit(*f.MaxFileSize)
if perr != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("ignoring --max-filesize: %v; size limit disabled", perr))
}
c.Transport.MaxFileSize = size
}
// ── Output flags ──
if *f.ContentDisp {
c.Output.ContentDisposition = true
}
if *f.ShowHeaders {
c.Output.ShowHeaders = true
}
if *f.KeepTimestamps {
c.Output.KeepTimestamps = true
}
if *f.JSON {
c.Output.JSONOutput = true
}
if *f.OnComplete != "" {
c.Output.OnComplete = *f.OnComplete
}
if *f.WarcFile != "" {
c.Output.WarcFile = *f.WarcFile
}
if *f.TraceTime {
c.Output.TraceTime = true
}
if *f.WriteOut != "" {
c.Output.WriteOut = *f.WriteOut
}
// ── Request flags ──
if len(f.Headers) > 0 {
if c.CustomHeaders == nil {
c.CustomHeaders = make(map[string]string)
}
for _, h := range f.Headers {
parts := strings.SplitN(h, ":", 2)
key := strings.TrimSpace(parts[0])
val := ""
if len(parts) > 1 {
val = strings.TrimSpace(parts[1])
}
if strings.ContainsAny(key, "\r\n") || strings.ContainsAny(val, "\r\n") {
cli.PrintWarning(os.Stderr, fmt.Sprintf("Skipping header with CRLF: %s", key))
continue
}
c.CustomHeaders[key] = val
}
}
if *f.Data != "" {
c.PostData = *f.Data
}
if *f.PostFile != "" {
c.PostFile = *f.PostFile
}
if *f.Range != "" {
c.Range = *f.Range
}
if *f.Compressed {
c.Compressed = true
}
if *f.Referer != "" {
c.Referer = *f.Referer
}
if *f.Fail {
c.FailOnHTTPError = true
}
if *f.NoClobber {
c.Transport.NoClobber = true
}
if len(f.Form) > 0 {
c.FormData = f.Form
}
// ── Recursive flags ──
if *f.RecursiveParallel > 0 {
c.Recursive.Parallel = *f.RecursiveParallel
}
if *f.Wait != "" {
if d, err := time.ParseDuration(*f.Wait); err == nil {
c.Recursive.Wait = d
}
}
if *f.RandomWait {
c.Recursive.RandomWait = true
}
if *f.ConvertLinks {
c.Recursive.ConvertLinks = true
}
if *f.Accept != "" {
c.Recursive.Accept = *f.Accept
}
if *f.Domains != "" {
c.Recursive.Domains = strings.Split(*f.Domains, ",")
}
if *f.Reject != "" {
c.Recursive.Reject = strings.Split(*f.Reject, ",")
}
if *f.CutDirs > 0 {
c.Recursive.CutDirs = *f.CutDirs
}
if *f.PageRequisites {
c.Recursive.PageRequisites = true
}
if *f.SpanHosts {
c.Recursive.SpanHosts = true
}
if *f.AdjustExt {
c.Recursive.AdjustExt = true
}
// ── Cookies from CLI flags ──
if len(f.Cookies) > 0 {
if cookieJar == nil {
var err error
cookieJar, err = cookie.NewJar()
if err != nil {
cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to create cookie jar: %v", err))
}
}
if cookieJar != nil {
for _, cstr := range f.Cookies {
parts := strings.SplitN(cstr, "=", 2)
if len(parts) < 2 {
cli.PrintWarning(os.Stderr, fmt.Sprintf("invalid cookie format (use name=value): %s", cstr))
continue
}
ck := &stdhttp.Cookie{Name: parts[0], Value: parts[1]}
cookieJar.SetCookies(parsedURL, []*stdhttp.Cookie{ck})
}
c.CookieJar = cookieJar
}
}
}
cmd/goget/verify.go
+200
View File
@@ -0,0 +1,200 @@
//go:build linux || freebsd
// +build linux freebsd
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"codeberg.org/petrbalvin/goget/internal/crypto"
)
// parseChecksumFile reads a checksum file (SHA256SUMS format) and finds
// the expected checksum for the given filename.
func parseChecksumFile(checksumFile, outputFile string) (string, error) {
data, err := os.ReadFile(checksumFile)
if err != nil {
return "", fmt.Errorf("failed to read checksum file: %w", err)
}
targetName := filepath.Base(outputFile)
lines := strings.Split(string(data), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
// Format: <checksum> <filename> or <checksum> *<filename>
parts := strings.Fields(line)
if len(parts) < 2 {
continue
}
checksum := parts[0]
filename := strings.TrimPrefix(parts[1], "*")
if filename == targetName {
return checksum, nil
}
// Also check if the output file matches
if filename == outputFile {
return checksum, nil
}
}
return "", fmt.Errorf("no checksum found for %s in %s", targetName, checksumFile)
}
// verifyChecksum verifies the checksum of a downloaded file.
// Returns true if the checksum matches, false otherwise.
// Returns an error if verification could not be performed.
func verifyChecksum(outputFile, checksumFlag, checksumAlgo string, verbose bool) (bool, error) {
if verbose {
fmt.Fprintf(os.Stderr, "\nVerifying checksum...\n")
}
algo := crypto.SHA256
switch strings.ToLower(checksumAlgo) {
case "sha256":
algo = crypto.SHA256
case "sha512":
algo = crypto.SHA512
case "blake2b":
algo = crypto.BLAKE2b
case "sha3-256", "sha3_256":
algo = crypto.SHA3_256
case "sha3-512", "sha3_512":
algo = crypto.SHA3_512
case "md5":
algo = crypto.MD5
default:
return false, fmt.Errorf("unsupported checksum algorithm: %s", checksumAlgo)
}
verifier, verr := crypto.NewChecksumVerifier(algo, checksumFlag)
if verr != nil {
return false, verr
}
match, actual, verr := verifier.VerifyFile(outputFile)
if verr != nil {
return false, verr
}
if !match {
return false, fmt.Errorf("checksum mismatch\n Expected: %s\n Actual: %s", checksumFlag, actual)
}
if verbose {
fmt.Fprintf(os.Stderr, "Checksum OK: %s\n", actual)
}
return true, nil
}
// verifyPGP handles PGP signature verification and PGP decryption.
// Returns true on success, false on failure, and any error encountered.
func verifyPGP(outputFile string, pgpVerify, pgpDecrypt bool, pgpKeyFlag, pgpPassphraseFlag, pgpSignatureFlag *string, verbose bool) (bool, error) {
// PGP signature verification
if pgpVerify && outputFile != "" {
if verbose {
fmt.Fprintf(os.Stderr, "\nVerifying PGP signature...\n")
}
pgpCfg := &crypto.PGPConfig{
PublicKeyPath: *pgpKeyFlag,
Passphrase: *pgpPassphraseFlag,
}
verifier := crypto.NewPGPVerifier(pgpCfg)
var valid bool
var identity string
var err error
if *pgpSignatureFlag != "" {
// Detached signature
valid, identity, err = verifier.VerifyFile(outputFile, *pgpSignatureFlag)
} else {
// Try to find signature file automatically
sigFiles := []string{
outputFile + ".asc",
outputFile + ".sig",
outputFile + ".gpg",
}
for _, sigFile := range sigFiles {
if _, statErr := os.Stat(sigFile); statErr == nil {
valid, identity, err = verifier.VerifyFile(outputFile, sigFile)
if valid {
break
}
}
}
if !valid {
err = fmt.Errorf("signature file not found or invalid (tried: %s)", strings.Join(sigFiles, ", "))
}
}
if err != nil {
return false, fmt.Errorf("pgp verification failed: %w", err)
}
if !valid {
return false, fmt.Errorf("pgp signature verification failed")
}
if verbose {
fmt.Fprintf(os.Stderr, "PGP signature verified OK (signed by: %s)\n", identity)
}
}
// PGP decryption
if pgpDecrypt && outputFile != "" {
if verbose {
fmt.Fprintf(os.Stderr, "\nDecrypting PGP file...\n")
}
// Read file to check if it's PGP encrypted
data, err := os.ReadFile(outputFile)
if err != nil {
return false, fmt.Errorf("failed to read file for pgp check: %w", err)
}
if !crypto.IsPGPEncrypted(data) {
if verbose {
fmt.Fprintf(os.Stderr, "File is not PGP encrypted, skipping decryption\n")
}
return true, nil
}
pgpCfg := &crypto.PGPConfig{
PrivateKeyPath: *pgpKeyFlag,
Passphrase: *pgpPassphraseFlag,
}
decryptor := crypto.NewPGPDecryptor(pgpCfg)
// Determine output path
decryptedPath := outputFile
if strings.HasSuffix(outputFile, ".gpg") {
decryptedPath = strings.TrimSuffix(outputFile, ".gpg")
} else if strings.HasSuffix(outputFile, ".pgp") {
decryptedPath = strings.TrimSuffix(outputFile, ".pgp")
} else if strings.HasSuffix(outputFile, ".asc") {
decryptedPath = strings.TrimSuffix(outputFile, ".asc")
} else {
decryptedPath = outputFile + ".decrypted"
}
if err := decryptor.DecryptFile(outputFile, decryptedPath); err != nil {
return false, fmt.Errorf("pgp decryption failed: %w", err)
}
if verbose {
fmt.Fprintf(os.Stderr, "PGP decryption successful: %s\n", decryptedPath)
}
// Remove encrypted file
if err := os.Remove(outputFile); err != nil {
if verbose {
fmt.Fprintf(os.Stderr, "Warning: failed to remove encrypted file: %v\n", err)
}
}
}
return true, nil
}
docs/api.md
+454
View File
@@ -0,0 +1,454 @@
# API Reference
**goget** exposes a public Go library in `pkg/api/` for programmatic downloads, uploads, queue management, and configuration.
## Installation
```go
import "codeberg.org/petrbalvin/goget/pkg/api"
```
## Types
### `Client`
The main client for downloads and uploads. Created via `NewClient()`.
```go
type Client struct { /* unexported */ }
func NewClient(opts ...ClientOption) *Client
func (c *Client) Download(req *DownloadRequest) (*DownloadResult, error)
func (c *Client) Upload(req *UploadRequest) (*UploadResult, error)
func (c *Client) Close()
```
### `ClientOption`
Functional options for client configuration.
```go
func WithTimeout(d time.Duration) ClientOption
func WithUserAgent(ua string) ClientOption
func WithVerbose(v bool) ClientOption
```
### `DownloadRequest`
| Field | Type | Description |
|---|---|---|
| `URL` | `string` | URL to download (required) |
| `Output` | `string` | Output file path (empty = auto-detect) |
| `Resume` | `bool` | Resume interrupted download |
| `Verbose` | `bool` | Enable verbose logging |
### `DownloadResult`
| Field | Type | Description |
|---|---|---|
| `BytesDownloaded` | `int64` | Total bytes transferred |
| `TotalSize` | `int64` | Total file size (-1 if unknown) |
| `Speed` | `float64` | Average speed in bytes/second |
| `Duration` | `time.Duration` | Total transfer time |
| `OutputPath` | `string` | Path to saved file |
| `Protocol` | `string` | Protocol used (e.g., "http") |
| `IPVersion` | `int` | 4 or 6 |
| `Resumed` | `bool` | True if download was resumed |
| `Parallel` | `bool` | True if parallel chunks were used |
| `ChunksCount` | `int` | Number of parallel chunks |
### `UploadRequest`
| Field | Type | Description |
|---|---|---|
| `URL` | `string` | Destination URL (required) |
| `FilePath` | `string` | Path to file to upload (required) |
| `Method` | `string` | HTTP method: `"PUT"` or `"POST"` |
| `Verbose` | `bool` | Enable verbose logging |
### `UploadResult`
| Field | Type | Description |
|---|---|---|
| `BytesUploaded` | `int64` | Total bytes uploaded |
| `Duration` | `time.Duration` | Total upload time |
| `Protocol` | `string` | Protocol used |
| `ResultURL` | `string` | Result URL (may differ from input after redirects) |
---
## Client
### `NewClient`
Creates a new goget client. Registers HTTP and FTP protocol handlers automatically.
```go
func NewClient(opts ...ClientOption) *Client
```
**Options:**
| Option | Effect |
|---|---|
| `WithTimeout(5 * time.Minute)` | Set request timeout (default: 30 minutes) |
| `WithUserAgent("MyApp/1.0")` | Custom User-Agent header |
| `WithVerbose(true)` | Enable verbose logging |
**Example:**
```go
client := api.NewClient(
api.WithTimeout(10 * time.Minute),
api.WithUserAgent("MyDownloader/1.0"),
)
defer client.Close()
```
### `Client.Download`
Downloads a file from a URL.
```go
func (c *Client) Download(req *DownloadRequest) (*DownloadResult, error)
```
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `req.URL` | `string` | URL to download — supports http, https, ftp, sftp, file, data, gopher, gemini |
| `req.Output` | `string` | Output path. Empty string uses the filename derived from the URL. |
| `req.Resume` | `bool` | Resume an interrupted download if resume metadata exists |
**Returns:** `*DownloadResult` — bytes transferred, speed, duration, protocol info.
**Throws:**
- `error` — if URL is empty or invalid
- `error` — if protocol is unsupported
- `*core.GogetError` — typed errors for network, protocol, timeout, auth, checksum failures
**Example:**
```go
result, err := client.Download(&api.DownloadRequest{
URL: "https://example.com/file.zip",
Output: "downloads/file.zip",
Resume: true,
})
if err != nil {
log.Printf("Download failed: %v", err)
return
}
log.Printf("Downloaded %d bytes at %.1f KB/s", result.BytesDownloaded, result.Speed/1024)
```
### `Client.Upload`
Uploads a file to a URL. The protocol handler must implement `core.Uploader`.
```go
func (c *Client) Upload(req *UploadRequest) (*UploadResult, error)
```
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `req.URL` | `string` | Destination URL (required) |
| `req.FilePath` | `string` | Path to local file (required) |
| `req.Method` | `string` | `"PUT"` or `"POST"` (default: `"PUT"`) |
| `req.Verbose` | `bool` | Enable verbose logging |
**Returns:** `*UploadResult` — bytes uploaded, duration, protocol, result URL.
**Throws:**
- `error` — if URL or file path is empty
- `error` — if protocol does not support upload
**Example:**
```go
result, err := client.Upload(&api.UploadRequest{
URL: "https://httpbin.org/put",
FilePath: "./data.csv",
Method: "PUT",
})
if err != nil {
log.Printf("Upload failed: %v", err)
return
}
log.Printf("Uploaded %d bytes", result.BytesUploaded)
```
### `Client.Close`
Releases resources and cancels any pending operations.
```go
func (c *Client) Close()
```
---
## Convenience Functions
### `Download`
One-shot download. Creates a client, downloads, and closes.
```go
func Download(urlStr, output string) (*DownloadResult, error)
```
**Example:**
```go
result, err := api.Download("https://example.com/file.zip", "output.zip")
```
### `Upload`
One-shot upload. Creates a client, uploads, and closes.
```go
func Upload(urlStr, filePath, method string) (*UploadResult, error)
```
**Example:**
```go
result, err := api.Upload("https://httpbin.org/put", "./data.csv", "PUT")
```
---
## Queue API
### `QueueClient`
Manages a download queue persisted to a JSON file.
```go
type QueueClient struct { /* unexported */ }
func NewQueueClient(queueFile string) (*QueueClient, error)
func (qc *QueueClient) Add(urlStr, output string, priority int) string
func (qc *QueueClient) List() []*queue.QueueItem
func (qc *QueueClient) Stats() queue.QueueStats
func (qc *QueueClient) Remove(id string) bool
func (qc *QueueClient) ClearCompleted() int
```
### `NewQueueClient`
```go
func NewQueueClient(queueFile string) (*QueueClient, error)
```
Creates a queue client backed by a JSON file.
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `queueFile` | `string` | Path to the queue JSON file |
**Returns:** `*QueueClient`, `error` — error if the file cannot be read or created.
### `QueueClient.Add`
```go
func (qc *QueueClient) Add(urlStr, output string, priority int) string
```
Adds a URL to the queue.
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `urlStr` | `string` | URL to download |
| `output` | `string` | Output file path |
| `priority` | `int` | Priority (1–10, higher = processed first) |
**Returns:** `string` — unique ID for the queued item.
### `QueueClient.List`
```go
func (qc *QueueClient) List() []*queue.QueueItem
```
Returns all items in the queue.
**Returns:** `[]*queue.QueueItem`
### `QueueClient.Stats`
```go
func (qc *QueueClient) Stats() queue.QueueStats
```
Returns queue statistics (total, pending, completed, failed counts).
### `QueueClient.Remove`
```go
func (qc *QueueClient) Remove(id string) bool
```
Removes an item from the queue by ID.
**Returns:** `bool` — true if the item was found and removed.
### `QueueClient.ClearCompleted`
```go
func (qc *QueueClient) ClearCompleted() int
```
Removes all completed items from the queue.
**Returns:** `int` — number of removed items.
---
## Metalink API
### `ParseMetalink`
Parses a local Metalink file.
```go
func ParseMetalink(path string) (*metalink.Metalink, error)
```
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `path` | `string` | Path to a local `.meta4` or `.metalink` file |
**Returns:** `*metalink.Metalink` — parsed metalink with sources, checksums, and metadata.
### `FetchMetalink`
Downloads and parses a Metalink from a URL.
```go
func FetchMetalink(urlStr string) (*metalink.Metalink, error)
```
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `urlStr` | `string` | URL pointing to a `.meta4` or `.metalink` file |
**Returns:** `*metalink.Metalink` — parsed metalink with sources, checksums, and metadata.
---
## Config API
### `LoadConfig`
```go
func LoadConfig(path string) (*config.Config, error)
```
Loads the TOML configuration file. Returns default config if the file does not exist.
**Parameters:**
| Parameter | Type | Description |
|---|---|---|
| `path` | `string` | Path to config file (empty = default location) |
**Returns:** `*config.Config`, `error`
### `DefaultConfig`
```go
func DefaultConfig() *config.Config
```
Returns the default configuration values.
### `ParseSpeed`
```go
func ParseSpeed(s string) int64
```
Parses a speed string like `"10MB/s"` into bytes per second.
| Input | Output |
|---|---|
| `"1MB/s"` | `1000000` |
| `"500KB/s"` | `500000` |
| `"1GB/s"` | `1000000000` |
---
## Version
### `Version`
```go
func Version() string
```
Returns the current goget version.
---
## Full Example
```go
package main
import (
"fmt"
"log"
"time"
"codeberg.org/petrbalvin/goget/pkg/api"
)
func main() {
// Create client with custom timeout
client := api.NewClient(
api.WithTimeout(5 * time.Minute),
api.WithVerbose(true),
)
defer client.Close()
// Download
result, err := client.Download(&api.DownloadRequest{
URL: "https://example.com/large-file.iso",
Output: "./downloads/large-file.iso",
Resume: true,
})
if err != nil {
log.Fatalf("download failed: %v", err)
}
fmt.Printf("Downloaded %d bytes in %v\n", result.BytesDownloaded, result.Duration)
fmt.Printf("Average speed: %.1f MB/s\n", result.Speed/1024/1024)
fmt.Printf("Protocol: %s (IPv%d)\n", result.Protocol, result.IPVersion)
if result.Resumed {
fmt.Println("Download was resumed from previous state")
}
if result.Parallel {
fmt.Printf("Used %d parallel chunks\n", result.ChunksCount)
}
// Queue a download
qc, err := api.NewQueueClient("./queue.json")
if err != nil {
log.Fatalf("queue init failed: %v", err)
}
id := qc.Add("https://example.com/another.zip", "./downloads/", 5)
fmt.Printf("Queued item ID: %s\n", id)
fmt.Printf("Queue stats: %+v\n", qc.Stats())
}
```
docs/architecture.md
+222
View File
@@ -0,0 +1,222 @@
# Architecture
**goget** is a command-line download utility built on the Go standard library. It follows a layered architecture with a protocol abstraction layer, a transport layer, and utility modules for authentication, compression, output, and more.
## High-Level Overview
```mermaid
graph TD
User[User / Terminal]:::accent6
Script[Script / CI]:::accent6
CLI[cmd/goget/ - Main entry]:::accent0
API[pkg/api/ - Library interface]:::accent3
Core[internal/core/ - Types, Protocol Interface, Errors]:::accent1
Registry[internal/protocol/ - Protocol Registry]:::accent1
HTTP[internal/protocol/http/]:::accent1
FTP[internal/protocol/ftp/]:::accent1
SFTP[internal/protocol/sftp/]:::accent1
FILE[internal/protocol/file/]:::accent1
DataURL[internal/protocol/dataurl/]:::accent1
Gopher[internal/protocol/gopher/]:::accent1
Gemini[internal/protocol/gemini/]:::accent1
WebDAV[internal/protocol/webdav/]:::accent1
Transport[internal/transport/ - Dialer, Proxy, TLS, Rate Limit]:::accent7
Config[internal/config/]:::accent7
Output[internal/output/ - Atomic write, Resume, WARC]:::accent2
Auth[internal/auth/ - Basic, Digest, OAuth, netrc]:::accent7
Crypto[internal/crypto/ - Checksum, PGP, Certs]:::accent7
Cookie[internal/cookie/ - Netscape jar]:::accent7
HSTS[internal/hsts/ - RFC 6797 cache]:::accent7
Archive[internal/archive/ - Extraction]:::accent7
Recursive[internal/recursive/ - Crawler, Parser, Filter]:::accent2
Mirror[internal/mirror/]:::accent2
Metalink[internal/metalink/]:::accent2
Queue[internal/queue/]:::accent2
Compression[internal/compression/ - gzip, bzip2, zlib, flate, lzw]:::accent7
Log[internal/log/ - Structured logging]:::accent7
WARC[internal/warc/ - Archiving]:::accent7
Register[internal/protocol/register/ - Single registration]:::accent7
Format[internal/format/ - Bytes, Speed formatting]:::accent7
LinkRewrite[internal/linkrewrite/ - HTML link conversion]:::accent7
CLIUtil[internal/cli/ - Help, Progress bar, Colors]:::accent7
User --> CLI
Script --> API
CLI --> Log
CLI --> CLIUtil
CLI --> Config
CLI --> Core
CLI -.-> API
API --> Core
API --> Registry
subgraph "Download Pipeline"
Core --> Registry
Registry --> HTTP
Registry --> FTP
Registry --> SFTP
Registry --> FILE
Registry --> DataURL
Registry --> Gopher
Registry --> Gemini
Registry --> WebDAV
HTTP --> Transport
HTTP --> Cookie
HTTP --> HSTS
HTTP --> Output
FTP --> Transport
SFTP --> Transport
end
subgraph "Post-Download"
CLI --> Crypto
CLI --> Archive
CLI --> Compression
end
subgraph "Download Management"
CLI --> Register
CLI --> WARC
CLI --> Recursive
CLI --> Mirror
CLI --> Metalink
CLI --> Queue
Recursive --> WARC
Recursive --> LinkRewrite
Mirror --> LinkRewrite
end
Transport --> Auth
Transport --> Config
classDef accent0 fill:#3B82F6,stroke:#2563EB,color:#fff
classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff
classDef accent2 fill:#F59E0B,stroke:#D97706,color:#000
classDef accent3 fill:#A855F7,stroke:#7C3AED,color:#fff
classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff
classDef accent7 fill:#64748B,stroke:#475569,color:#fff
```
## Layers
### 1. Entry Points
| Layer | Package | Role |
|---|---|---|
| **CLI** | `cmd/goget/` | Flag parsing, command routing, signal handling |
| **Library API** | `pkg/api/` | Public Go API for programmatic downloads, uploads, and queue management |
The CLI and API share the same internal packages. The CLI is built on top of the same `internal/core` types that the API exposes.
### 2. Protocol Abstraction
Defined in `internal/core` and `internal/protocol`:
```go
// core.Protocol is the interface every protocol handler implements.
type Protocol interface {
Scheme() string
CanHandle(url *url.URL) bool
Download(ctx context.Context, req *DownloadRequest) (*DownloadResult, error)
Capabilities() []string
SupportsResume() bool
SupportsCompression() bool
SupportsParallel() bool
SupportsRecursive() bool
}
// core.Uploader is an optional interface for protocols supporting uploads.
type Uploader interface {
Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error)
SupportsUpload() bool
}
```
Protocols register themselves by scheme in a global `Registry`. Scheme normalization maps HTTPS → HTTP, FTPS → FTP, etc. Supported protocols include HTTP, FTP, SFTP, File, DataURL, Gopher, Gemini, and WebDAV (new in `internal/protocol/webdav/`).
### 3. Transport Layer
`internal/transport` provides a custom HTTP transport with:
- **IPv6-first dialer** — Resolves DNS, prefers IPv6 addresses, falls back to IPv4 only when no IPv6 record exists
- **Interface binding** — Linux-specific socket binding via `SO_BINDTODEVICE`
- **Rate limiting** — Token bucket algorithm for bandwidth capping
- **Retry logic** — Exponential backoff with configurable max retries
- **TLS configuration** — Certificate pinning, client certificates, key log file
- **Proxy support** — HTTP CONNECT and SOCKS5 (`socks5://` with local DNS, `socks5h://` with remote DNS, RFC 1929 username/password auth)
### 4. Output Layer
`internal/output` handles atomic file writes:
1. Write to a `.goget.tmp` file during download
2. Store resume metadata in `.goget.meta` alongside the temp file
3. Atomically rename from `.goget.tmp` to the final filename on completion
### 5. Utility Modules
| Module | Responsibility |
|---|---|
| `auth` | HTTP Basic, Digest, OAuth 2.0 (client credentials, authorization code, refresh token), `.netrc` auto-reading |
| `cookie` | Netscape-format cookie jar with import/export |
| `crypto` | Checksum verification (SHA-256, SHA-512, SHA3-256/512, BLAKE2b, MD5), PGP signature verification and decryption, certificate handling |
| `compression` | Decompression wrappers for gzip, bzip2, zlib, flate, LZW |
| `hsts` | HSTS cache (RFC 6797) for automatic HTTPS upgrade |
| `archive` | Archive extraction for tar, tar.gz, tar.bz2, zip with auto-detection |
| `format` | Human-readable byte and speed formatting |
| `linkrewrite` | HTML link rewriting for offline mirror viewing |
| `log` | Structured logger with text and JSON output modes, level filtering, ANSI colors |
| `warc` | WARC (Web ARChive) format writing for legal and archival compliance |
| `cli` | ANSI color output, progress bar, help text, completion |
## Download Flow
```mermaid
sequenceDiagram
participant User
participant CLI as cmd/goget/
participant Config as internal/config/
participant Core as internal/core/
participant Registry as internal/protocol/
participant Proto as Protocol Handler
participant Transport as internal/transport/
participant Output as internal/output/
User->>CLI: goget --url https://example.com/file.zip
CLI->>Config: Load config file
Config-->>CLI: Config (with defaults + overrides)
CLI->>Core: Create DownloadRequest
CLI->>Registry: Get protocol for URL scheme
Registry-->>CLI: HTTP protocol handler
CLI->>Proto: Download(ctx, request)
Proto->>Transport: Dial + send HTTP request
Transport-->>Proto: HTTP response stream
Proto->>Output: Write chunks atomically
Output-->>Proto: Progress + completion
Proto-->>CLI: DownloadResult
CLI-->>User: Progress bar + summary
```
## Config Merging
Configuration is resolved in this priority order (highest first):
1. CLI flags (`--timeout 5m`)
2. Config file (`~/.config/goget/config.toml`)
3. Default values
The `config.Merge()` method applies CLI override top of the loaded config.
## Error Handling
All operations return structured `core.GogetError` with:
- **Type** — Category (`NETWORK`, `PROTOCOL`, `FILE`, `TIMEOUT`, `AUTH`, `CHECKSUM`, `UNSUPPORTED`, `CANCELLED`)
- **Message** — Human-readable description
- **Cause** — Wrapped underlying error (available via `errors.Unwrap`)
- **URL** — The URL that caused the error (credentials stripped)
- **Details** — Optional key-value metadata (e.g., expected vs actual checksum)
## Build Tags
All source files use the build constraint `//go:build linux || freebsd`. The project targets Linux and FreeBSD only. Platform-specific code (e.g., network interface binding) lives in `internal/transport/bind_linux.go` vs `bind_other.go`.
docs/authentication.md
+251
View File
@@ -0,0 +1,251 @@
# Authentication
goget supports multiple authentication mechanisms for HTTP, FTP, and SFTP connections. Credentials are resolved in priority order, with `.netrc` auto-reading as a fallback.
## Priority Order
1. CLI flags (`--username`, `--password`, `--password-file`)
2. OAuth 2.0 tokens (if configured via CLI or config)
3. `.netrc` auto-reading (`~/.netrc`)
4. Anonymous / no auth
## HTTP Basic Authentication (RFC 7617)
The simplest mechanism. Credentials are Base64-encoded in the `Authorization` header.
```bash
goget --username admin --password secret --url https://api.example.com/data
```
```
Authorization: Basic YWRtaW46c2VjcmV0
```
### Security Notes
- Basic auth sends credentials in plain text (Base64 is not encryption)
- Always use HTTPS with Basic auth
- Prefer `--password-file` for scripting to avoid credentials in shell history:
```bash
goget --username admin --password-file ./secret.txt --url https://api.example.com
```
## HTTP Digest Authentication (RFC 7616)
Digest auth never sends the password in plain text. Instead, it uses a challenge-response mechanism with MD5 or SHA-256 hashing:
```mermaid
sequenceDiagram
participant Client
participant Server
Client->>Server: GET /protected
Server-->>Client: 401 + WWW-Authenticate: Digest nonce="abc", realm="app"
Client->>Server: GET /protected + Authorization: Digest response="hash"
Server-->>Client: 200 OK
```
### Usage
```bash
goget --username admin --password secret --auth-type digest --url https://api.example.com
goget --username admin --password secret --auth-type auto --url https://api.example.com
```
With `--auth-type auto`, goget first tries to make the request. If the server returns 401 with a `WWW-Authenticate: Digest` header, it automatically switches to Digest auth.
### Implementation
The `internal/auth` package implements Digest auth per RFC 7616:
- **HA1** = `MD5(username:realm:password)`
- **HA2** = `MD5(method:digestURI)`
- **response** = `MD5(HA1:nonce:nc:cnonce:qop:HA2)`
The `cnonce` (client nonce) is generated using `crypto/rand` for each request.
## OAuth 2.0
goget supports four OAuth 2.0 grant types:
### Client Credentials Grant
For server-to-server communication without user interaction:
```bash
goget \
--oauth-client-id "my-client" \
--oauth-client-secret "my-secret" \
--oauth-token-url "https://auth.example.com/token" \
--oauth-grant-type client_credentials \
--oauth-scopes "read,write" \
--url https://api.example.com/data
```
### Authorization Code Grant
For web applications with user interaction:
```bash
# Step 1: Generate authorization URL
goget \
--oauth-client-id "my-client" \
--oauth-auth-url "https://auth.example.com/authorize" \
--oauth-redirect-uri "http://localhost:8080/callback" \
--url https://api.example.com/data
# Step 2: Exchange code for token (after user authorizes)
goget \
--oauth-client-id "my-client" \
--oauth-client-secret "my-secret" \
--oauth-token-url "https://auth.example.com/token" \
--oauth-grant-type authorization_code \
--url https://api.example.com/data
```
### Refresh Token Grant
For maintaining long-lived access:
```bash
goget \
--oauth-client-id "my-client" \
--oauth-client-secret "my-secret" \
--oauth-token-url "https://auth.example.com/token" \
--oauth-refresh-token "ref_xxx" \
--oauth-grant-type refresh_token \
--url https://api.example.com/data
```
### Token Management
```go
client := auth.NewOAuthClient(cfg)
// Check if token is valid (not expired)
if !client.IsTokenValid() {
// Refresh automatically
client.RefreshToken(ctx)
}
// Apply token to HTTP request
client.ApplyToken(httpReq)
// → Authorization: Bearer eyJhbG...
```
The token expiry is tracked with a 30-second grace period.
### Security
- **Token URL enforced to HTTPS** — plain HTTP token endpoints are rejected unless they're `localhost` or `127.0.0.1`
- **Tokens never logged** — The config `Save()` method strips `access_token`, `refresh_token`, and `client_secret` before writing to disk
## .netrc Auto-Reading
When no explicit credentials are provided via CLI flags or OAuth config, goget automatically checks `~/.netrc`:
```
machine api.example.com
login admin
password secret
machine default
login guest
password guest123
```
### Usage
```bash
# No --username or --password needed if ~/.netrc has a matching entry
goget --url https://api.example.com/data
```
### Resolution Order
1. **Exact hostname match** — `api.example.com` matches `machine api.example.com`
2. **Default entry** — `machine default` used as fallback
3. **No match** — proceeds without credentials
### Skip .netrc
Use `--no-private` to disable .netrc and auth file reading:
```bash
goget --no-private --url https://api.example.com
```
## Cookies
goget supports Netscape-format cookie jars:
### CLI Cookies
```bash
goget --cookie "session=abc123" --cookie "theme=dark" --url https://example.com
```
### Cookie Jar
Persist cookies across sessions:
```bash
# Save cookies to file
goget --cookie-jar cookies.txt --url https://example.com
# Load cookies from file
goget --cookie-jar cookies.txt --url https://example.com/protected
```
The cookie jar uses the standard Netscape format:
## SFTP Authentication
SFTP connections authenticate via SSH:
### SSH Key Authentication
goget auto-detects keys from standard locations:
- `~/.ssh/id_rsa`
- `~/.ssh/id_ecdsa`
- `~/.ssh/id_ed25519`
```bash
goget --url sftp://user@host:/path/to/file.zip
```
### Password Authentication
```bash
goget --url sftp://user@host:/path/to/file.zip --password "secret"
```
### Known Hosts
Host key verification uses `~/.ssh/known_hosts` by default:
```bash
# Custom known_hosts file
goget --url sftp://user@host:/path/to/file.zip --known-hosts ~/.ssh/known_hosts
# Skip verification (first connection)
goget --url sftp://user@host:/path/to/file.zip --ssh-insecure
```
## FTP Authentication
FTP connections default to anonymous login. For authenticated FTP:
```bash
# Username/password
goget --url ftp://ftp.example.com/file.zip --username user --password pass
# Anonymous (default)
goget --url ftp://anonymous@ftp.example.com/file.zip
```
## Access Tokens in URLs
While `https://user:pass@example.com` URLs are supported, prefer `--username` and `--password` CLI flags. When a URL contains credentials, they're automatically stripped from error messages and logs via `core.SafeURL()`.
docs/cli-reference.md
+225
View File
@@ -0,0 +1,225 @@
# CLI Reference
Complete reference for all goget command-line flags, organized by category.
```bash
goget --url <URL> [OPTIONS]
```
## Target
| Flag | Argument | Description |
|---|---|---|
| `--url` | `<URL>` | URL to download (repeatable for multiple URLs) |
## Output
| Flag | Argument | Description |
|---|---|---|
| `--output` | `<FILE>` | Output filename or directory (default: auto-derived from URL) |
| `--restart` | — | Delete existing file and restart download from scratch |
| `--overwrite` | — | Overwrite existing file without atomic rename |
| `--content-disposition` | — | Use filename from server's `Content-Disposition` header |
| `--create-dirs` | — | Create missing parent directories of `--output` (default: true). Use `--create-dirs=false` to disable (curl `--create-dirs` / wget behavior). Ignored for stdout output |
| `--keep-timestamps` | — | Set file modification time to server's `Last-Modified` value |
## Progress
| Flag | Argument | Description |
|---|---|---|
| `--verbose` | — | Verbose output with Unicode progress bar (default enabled) |
| `--no-progress` | — | Hide progress bar, keep other output |
| `--progress` | `dot` | Dot progress bar style |
| `--quiet` | — | Suppress all output except errors |
| `--json` | — | Output progress as JSON lines for scripting (format: `{"type":"progress","current":X,"total":X,"speed":X}`) |
| `--debug` | — | Enable debug output with detailed connection info |
## Network
| Flag | Argument | Description |
|---|---|---|
| `--timeout` | `<DURATION>` | Connection timeout (`0` = smart auto-calculation based on file size) |
| `--connect-timeout` | `<DURATION>` | TCP connection establishment timeout |
| `--max-time` | `<DURATION>` | Maximum total transfer time (abort if exceeded) |
| `--max-retries` | `<N>` | Maximum retry attempts on failure |
| `--max-filesize` | `<SIZE>` | Abort if file exceeds size limit (e.g. `100MB`, `1GB`) |
| `--proxy` | `<URL>` | Proxy server URL — `http://` for HTTP CONNECT, `socks5://` (local DNS) or `socks5h://` (remote DNS) for SOCKS5, with optional `user:pass@` authentication |
| `--no-ipv4` | — | Disable IPv4 fallback (IPv6 only) |
| `--no-redirect` | — | Disable following HTTP redirects (3xx) |
| `--dns-servers` | `<IPs>` | Custom DNS servers, comma-separated (e.g. `1.1.1.1,8.8.8.8`) |
| `--bind-interface` | `<IFACE>` | Bind outgoing connections to a specific network interface (Linux only) |
| `--allow-private` | — | Allow connections to private/internal IPs (disabled by default for SSRF protection) |
| `--pinned-cert` | `<SHA256>` | Certificate pinning — abort if server cert SHA-256 doesn't match |
## HTTP / Network Flags
| Flag | Argument | Description |
|---|---|---|
| `--header` | `"K: V"` | Custom HTTP header (repeatable) |
| `--insecure` | — | Skip TLS certificate verification |
| `--cert` | `<FILE>` | Client certificate in PEM format (mTLS) |
| `--key` | `<FILE>` | Client private key in PEM format (mTLS) |
| `--data` | `"key=value"` | Send POST data (implies POST) |
| `--form` | `"field=value"` | Multipart form field (repeatable; `--form "file=@./path"` for file upload) |
| `--spider` | — | Check URL existence via HEAD request — no body downloaded |
| `--write-out` | `<FORMAT>` | Print metadata after transfer (format: `%{http_code}`, `%{size_download}`, `%{time_total}`, `%{speed_download}`) |
| `--trace-time` | — | Add HTTP timing breakdown to output (DNS, TCP, TLS, TTFB, Total) |
| `--fail` | — | Exit non-zero on HTTP 4xx/5xx response |
| `--compressed` | — | Accept-Encoding header for compressed transfer |
| `--range` | `<N>-<M>` | Download a byte range (e.g. `--range 0-999`) |
| `--post-file` | `<FILE>` | POST file contents as request body |
| `--referer` | `<URL>` | Set Referer header |
## Recursive / Spider Flags
| Flag | Argument | Description |
|---|---|---|
| `--accept` | `"*.pdf"` | Accept patterns — filename glob or MIME type (comma-separated) |
| `--no-parent` | — | Do not ascend to parent directories during recursive download |
| `--adjust-extension` | — | Append `.html` extension to text/html files without one |
| `--wait` | `<DURATION>` | Delay between requests in recursive mode |
| `--random-wait` | — | Randomize wait time (0.5x–1.5x of `--wait` value) |
| `--page-requisites` | — | Download CSS, JS, and images required to render HTML pages |
| `--span-hosts` | — | Follow links to external domains in recursive mode |
| `--convert-links` | — | Rewrite links in downloaded HTML for local offline viewing |
| `--warc-file` | `<FILE>` | Write WARC (Web ARChive) output for archival |
| `--dry-run` | — | List all URLs that would be downloaded in recursive/mirror mode without saving files to disk |
| `--progress` | `dot` | Dot progress bar style |
| `--no-clobber` | — | Skip download if file already exists |
## Recursive / Mirror
| Flag | Argument | Description |
|---|---|---|
| `--recursive` | — | Enable recursive downloading — follow links from HTML pages |
| `--mirror` | — | Mirror entire website (infinite depth, convert links) — shorthand for `--recursive --convert-links --page-requisites` |
| `--max-depth` | `<N>` | Maximum recursion depth |
| `--follow-external` | — | Follow links to domains other than the starting domain |
| `--exclude-pattern` | `<P>` | Exclude URL patterns (glob, comma-separated) |
| `--no-convert-links` | — | Don't convert links in HTML during mirror (faster, but not offline-ready) |
| `--no-mirror-assets` | — | Don't download CSS, JS, and images during mirror |
| `--no-robots` | — | Don't respect `robots.txt` during mirroring |
| `--rate-limit` | `<RATE>` | Maximum download speed (e.g. `1MB/s`, `500KB/s`, `0` = unlimited) |
| `--domains` | `<LIST>` | Restrict recursion to listed domains (comma-separated) |
| `--reject` | `<PATTERN>` | Reject URL patterns (glob) |
| `--cut-dirs` | `<N>` | Ignore N directory components when creating local filenames |
| `--proto-dirs` | — | Create protocol-prefixed directories (http/, ftp/ etc.) |
| `--adjust-extension` | — | Append `.html` to text/html files missing an extension |
| `--timestamping` | — | Only download files newer than the local copy |
| `--backup-converted` | — | Back up original `.orig` files before link conversion |
| `--recursive-parallel` | `<N>` | Number of concurrent file downloads in recursive mode across all protocols (HTTP, WebDAV, FTP, SFTP). `0` = sequential (default) |
| `--continue` | — | Continue getting partially-downloaded file |
## Download Management
| Flag | Argument | Description |
|---|---|---|
| `--queue` | — | Add URL to the download queue (see `--queue-file`) |
| `--queue-list` | — | List all items in the queue |
| `--queue-process` | — | Process (download) all pending items in the queue |
| `--queue-clear` | — | Remove completed items from the queue |
| `--queue-file` | `<PATH>` | Custom queue file path (default: `~/.config/goget/queue.json`) |
| `--queue-priority` | `<N>` | Priority for queued item (1–10, higher = processed first) |
| `--batch-file` | `<FILE>` | Read URLs from a file, one URL per line |
| `--input-file` | `<FILE>` | Read URLs from a file (alias for `--batch-file`) |
| `--schedule` | `<TIME>` | Schedule download — absolute (`"2026-06-01 02:00"`) or cron (`"0 2 * * *"`) |
| `--on-complete` | `<CMD>` | Run shell command after successful download (env: `GOGET_OUTPUT`, `GOGET_SIZE`, `GOGET_URL`) |
## Authentication
| Flag | Argument | Description |
|---|---|---|
| `--username` | `<USER>` | Username for HTTP Basic/Digest authentication |
| `--password` | `<PASS>` | Password (prefer `--password-file` for security) |
| `--password-file` | `<FILE>` | Read password from file |
| `--auth-type` | `<TYPE>` | Authentication type: `basic`, `digest`, or `auto` |
| `--cookie-jar` | `<FILE>` | Load and save cookies in Netscape format |
| `--cookie` | `"n=v"` | Set cookie from CLI (repeatable) |
| `--oauth-client-id` | `<ID>` | OAuth 2.0 Client ID |
| `--oauth-client-secret` | `<S>` | OAuth 2.0 Client Secret |
| `--oauth-token-url` | `<URL>` | OAuth 2.0 Token endpoint URL |
| `--oauth-auth-url` | `<URL>` | OAuth 2.0 Authorization endpoint URL |
| `--oauth-redirect-uri` | `<URI>` | OAuth 2.0 Redirect URI |
| `--oauth-scopes` | `<S>` | OAuth 2.0 scopes (comma-separated) |
| `--oauth-grant-type` | `<TYPE>` | OAuth 2.0 grant type: `client_credentials`, `authorization_code`, `password`, `refresh_token` |
| `--oauth-access-token` | `<T>` | OAuth 2.0 access token |
| `--oauth-refresh-token` | `<T>` | OAuth 2.0 refresh token |
## Upload
| Flag | Argument | Description |
|---|---|---|
| `--upload` | — | Upload file instead of downloading |
| `--upload-method` | `<M>` | HTTP method: `PUT` or `POST` (default: `PUT`) |
| `--upload-file` | `<FILE>` | Path to file to upload |
| `--form-data` | `"k=v"` | Form data for multipart upload (comma-separated) |
| `--file-field` | `<NAME>` | Form field name for file upload (default: `file`) |
## Verification
| Flag | Argument | Description |
|---|---|---|
| `--checksum` | `<HASH>` | Expected checksum (hex string) |
| `--checksum-algo` | `<ALG>` | Algorithm: `sha256`, `sha512`, `blake2b`, `sha3-256`, `sha3-512`, `md5` (default: `sha256`) |
| `--checksum-file` | `<FILE>` | Read expected checksum from SHA256SUMS-format file |
## PGP / GPG
| Flag | Argument | Description |
|---|---|---|
| `--pgp-decrypt` | — | Decrypt PGP-encrypted file after download |
| `--pgp-verify` | — | Verify PGP detached or inline signature |
| `--pgp-sig` | `<FILE>` | Path to detached signature file (`.asc`, `.sig`) |
| `--pgp-key` | `<FILE>` | Path to PGP key file |
| `--pgp-passphrase` | `<PASS>` | Passphrase for private key |
| `--pgp-passphrase-file` | `<F>` | Read passphrase from file |
## Metalink
| Flag | Argument | Description |
|---|---|---|
| `--metalink` | — | Treat URL as a Metalink file (multi-source download) |
| `--metalink-file` | `<PATH>` | Path to local Metalink file (`.meta4`, `.metalink`) |
## SSL / TLS
| Flag | Argument | Description |
|---|---|---|
| `--ssl-key-log` | `<FILE>` | Log TLS master secrets to file (SSLKEYLOGFILE for Wireshark) |
| `--known-hosts` | `<FILE>` | Custom SSH `known_hosts` file path for SFTP |
| `--ssh-insecure` | — | Skip SSH host key verification for SFTP (accept any key) |
| `--cacert` | `<FILE>` | Custom CA certificate bundle |
## Configuration
| Flag | Argument | Description |
|---|---|---|
| `--config` | `<FILE>` | Path to configuration file |
| `--init-config` | — | Generate default config file at `~/.config/goget/config.toml` |
| `--config-get` | `<KEY>` | Get value (e.g. `timeout`, `parallel`) |
| `--config-set` | `<K>` `<V>` | Set value (e.g. `--config-set timeout 5m`) |
| `--config-list` | — | List all config keys and values |
| `--config-unset` | `<KEY>` | Remove config key |
## Information
| Flag | Argument | Description |
|---|---|---|
| `--info` | `<URL>` | Show file metadata without downloading (size, type, server info) |
| `--benchmark` | `<URL>` | Benchmark download speed (downloads first 10 MB) |
| `--generate-man-page` | — | Generate man page in troff format and exit |
| `--completion` | `<SHELL>` | Generate shell completion for `bash`, `zsh`, or `fish` |
| `--help` | — | Show help text and exit |
| `--version` | — | Show version and exit |
## Additional Flags
| Flag | Argument | Description |
|---|---|---|
| `--parallel` | `<N>` | Number of parallel chunk connections (`0` = auto, `1` = sequential) |
| `--max-speed` | `<RATE>` | Maximum download speed in bytes/sec or human format |
| `--no-private` | — | Disable `netrc` and auth file reading |
| `--glob` | `<PATTERN>` | URL glob pattern expansion (e.g. `--glob "https://example.com/file[1-3].zip"`) |
| `--retry-all-errors` | — | Retry on any HTTP 4xx/5xx response |
| `--retry-delay` | `<DURATION>` | Custom delay before retry |
| `--hsts-file` | `<FILE>` | Custom HSTS cache file path |
docs/configuration.md
+122
View File
@@ -0,0 +1,122 @@
# Configuration Reference
goget loads configuration from a TOML file at `~/.config/goget/config.toml`. CLI flags override config file values, which override defaults. The `GOGET_CONFIG` environment variable can be used to specify an alternative config path.
## Quick Start
```bash
# Generate default config
goget --init-config
# Read a value
goget --config-get timeout
# Set a value
goget --config-set timeout 5m
# List all values
goget --config-list
# Remove a value
goget --config-unset timeout
```
## Configuration Keys
### Network
| Key | Type | Default | Description |
|---|---|---|---|
| `timeout` | duration string | `30m` | Request timeout. Set to `0` for auto-calculation based on file size |
| `parallel` | int | `0` | Parallel connections. `0` = auto (4 for files >100 MB, 1 otherwise) |
| `proxy` | string | `""` | Proxy server URL (HTTP CONNECT) |
| `max_speed` | int64 | `0` | Bytes per second cap. `0` = unlimited |
| `max_retries` | int | `0` | Retry attempts on failure |
| `dns_servers` | []string | `[]` | Custom DNS servers (e.g. `["1.1.1.1","8.8.8.8"]`) |
| `ipv4_fallback` | bool | `true` | Whether to fall back to IPv4 if IPv6 is unavailable |
### HTTP
| Key | Type | Default | Description |
|---|---|---|---|
| `user_agent` | string | `Goget/1.0.0` | HTTP User-Agent header value |
| `auto_decompress` | bool | `true` | Auto-decompress gzip/deflate responses |
| `insecure_skip_verify` | bool | `false` | Skip TLS certificate verification (insecure) |
| `auto_resume` | bool | `true` | Resume download when `.goget.meta` metadata exists |
| `cookie_jar` | string | `""` | Path to Netscape-format cookie jar file |
| `keep_metadata` | bool | `false` | Keep `.goget.meta` files after download completes |
### Authentication
| Key | Type | Default | Description |
|---|---|---|---|
| `auth.username` | string | `""` | Default username for HTTP Auth |
| `auth.password` | string | `""` | Default password (⚠ stored in plain text — prefer `password_file`) |
| `auth.password_file` | string | `""` | Path to file containing the password |
| `auth.auth_type` | string | `"auto"` | `basic`, `digest`, or `auto` |
| `auth.oauth.client_id` | string | `""` | OAuth 2.0 Client ID |
| `auth.oauth.client_secret` | string | `""` | OAuth 2.0 Client Secret |
| `auth.oauth.token_url` | string | `""` | OAuth 2.0 Token endpoint URL |
| `auth.oauth.auth_url` | string | `""` | OAuth 2.0 Authorization endpoint URL |
| `auth.oauth.redirect_uri` | string | `""` | OAuth 2.0 Redirect URI |
| `auth.oauth.scopes` | []string | `[]` | OAuth 2.0 scopes |
| `auth.oauth.access_token` | string | `""` | OAuth 2.0 Access Token |
| `auth.oauth.refresh_token` | string | `""` | OAuth 2.0 Refresh Token |
| `auth.oauth.grant_type` | string | `""` | `client_credentials`, `authorization_code`, `password`, `refresh_token` |
### Recursive Downloads
| Key | Type | Default | Description |
|---|---|---|---|
| `recursive.enabled` | bool | `false` | Enable recursive downloading |
| `recursive.max_depth` | int | `3` | Maximum recursion depth |
| `recursive.follow_external` | bool | `false` | Follow links to external domains |
| `recursive.exclude_patterns` | []string | `[]` | URL patterns to exclude (simple substring match) |
| `recursive.include_patterns` | []string | `[]` | URL patterns to include (simple substring match) |
| `recursive.delay` | duration string | `100ms` | Delay between requests in recursive mode |
| `recursive.parallel` | int | `0` | Concurrent file downloads in recursive mode. `0` = sequential, `N` = up to N concurrent workers (applies to HTTP, WebDAV, FTP, SFTP) |
### Archive Extraction
| Key | Type | Default | Description |
|---|---|---|---|
| `extract.enabled` | bool | `false` | Auto-extract archives after download |
| `extract.output_dir` | string | `""` | Extraction target directory (empty = same as download dir) |
| `extract.strip_components` | int | `0` | Number of leading path components to strip |
| `extract.exclude_patterns` | []string | `[]` | File patterns to exclude from extraction |
| `extract.preserve_permissions` | bool | `true` | Preserve file permissions from archive |
| `extract.auto_detect` | bool | `true` | Auto-detect archive format (tar, tar.gz, tar.bz2, zip) |
### Output / Display
| Key | Type | Default | Description |
|---|---|---|---|
| `output_dir` | string | `"."` | Default output directory |
| `verbose` | bool | `true` | Verbose output with progress bar |
| `debug` | bool | `false` | Debug output with detailed connection info |
| `color` | string | `"auto"` | Color output: `always`, `never`, or `auto` (honors `NO_COLOR` env var) |
| `progress_style` | string | `"unicode"` | Progress bar style: `"unicode"` or `"dot"` |
| `show_eta` | bool | `true` | Display estimated time remaining |
| `checksum_algo` | string | `"sha256"` | Default checksum algorithm |
## Password Security
Passwords are stored in plain text in the config file by default. For better security, use `auth.password_file` to reference an external file, and set `GOGET_PASSWORD` as an environment variable — the config loader will prefer it.
When `config.Save()` is called, the `password`, `client_secret`, `access_token`, and `refresh_token` fields are stripped from the output to prevent accidental credential leakage.
## Config Resolution Order
1. **CLI flags** (highest priority) — override everything below
2. **Config file** — `~/.config/goget/config.toml` (or `GOGET_CONFIG` path)
3. **Defaults** (lowest priority) — hardcoded in `DefaultConfig()`
The `config.Merge()` method applies CLI overrides after loading:
```go
cfg, _ := config.Load("")
cfg.Merge(&config.CLIFlags{
Timeout: 5 * time.Minute,
Parallel: 8,
})
```
docs/download-pipeline.md
+312
View File
@@ -0,0 +1,312 @@
# Download Pipeline
This document explains how goget processes a download request from start to finish — the phase structure, parallel chunking, resume logic, and output writing.
## High-Level Flow
```mermaid
flowchart TD
Start([User invokes goget]):::accent0
ParseFlags["Parse CLI flags"]:::accent7
LoadConfig["Load config + merge overrides"]:::accent7
ResolveProtocol["Resolve protocol handler\nfrom URL scheme"]:::accent1
CreateRequest["Build core.DownloadRequest"]:::accent1
CheckResume{"Resume metadata\nexists?"}:::accent2
LoadResume["Load .goget.meta\n(resume info + chunk map)"]:::accent2
SetupOutput["Setup output writer\n(atomic temp file + resume)"]:::accent2
CreateDirs{"--create-dirs\n(parent missing)?"}:::accent2
MkdirAll["os.MkdirAll → create\nparent directories"]:::accent2
CheckParallel{"File > 100 MB\nor --parallel set?"}:::accent1
SequentialDownload["Sequential download\n(single connection)"]:::accent1
ParallelDownload["Parallel chunked download\n(multiple Range requests)"]:::accent1
Decompress{"Auto-decompress\nenabled?"}:::accent7
Decompression["Decompress response\n(gzip, deflate, bzip2, zlib)"]:::accent7
VerifyChecksum{"Checksum\nspecified?"}:::accent7
ChecksumVerify["Verify checksum\n(SHA-256, SHA-512, etc.)"]:::accent7
PGPVerify{"PGP verify/\ndecrypt?"}:::accent7
PGPProcess["Verify signature /\ndecrypt file"]:::accent7
SaveHSTS["Save HSTS cache\n(RFC 6797)"]:::accent7
AtomicRename["Atomic rename\ntemp → final"]:::accent2
SigInt{"SIGINT received?"}:::accent4
SaveSidecar["Save .goget.meta\nfor resume"]:::accent2
Exit130["Exit 130"]:::accent4
RunHook{"--on-complete\nset?"}:::accent7
PostHook["Run post-download command\n(GOGET_OUTPUT, GOGET_SIZE, GOGET_URL)"]:::accent7
Done([Done]):::accent0
Start --> ParseFlags --> LoadConfig --> ResolveProtocol
ResolveProtocol --> CreateRequest --> CheckResume
CheckResume -->|Yes| LoadResume --> SetupOutput
CheckResume -->|No| SetupOutput
SetupOutput --> CreateDirs
CreateDirs -->|Yes| MkdirAll --> CheckParallel
CreateDirs -->|No| CheckParallel
CheckParallel -->|Yes| ParallelDownload
CheckParallel -->|No| SequentialDownload
SequentialDownload --> Decompress
ParallelDownload --> Decompress
Decompress -->|Yes| Decompression --> VerifyChecksum
Decompress -->|No| VerifyChecksum
VerifyChecksum -->|Yes| ChecksumVerify -->|pass| PGPVerify
ChecksumVerify -->|fail| Done
VerifyChecksum -->|No| PGPVerify
PGPVerify -->|Yes| PGPProcess --> SaveHSTS
PGPVerify -->|No| SaveHSTS
SaveHSTS --> AtomicRename
AtomicRename --> SigInt
SigInt -->|Yes| SaveSidecar --> Exit130
SigInt -->|No| RunHook
RunHook -->|Yes| PostHook --> Done
RunHook -->|No| Done
classDef accent0 fill:#3B82F6,stroke:#2563EB,color:#fff
classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff
classDef accent2 fill:#F59E0B,stroke:#D97706,color:#000
classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff
classDef accent7 fill:#64748B,stroke:#475569,color:#fff
```
## Phase 1: Request Construction
A `core.DownloadRequest` is built from CLI flags and config:
```go
req := &core.DownloadRequest{
URL: parsedURL,
Output: outputPath,
Resume: resumeEnabled,
Timeout: effectiveTimeout,
Verbose: verbose,
Headers: customHeaders,
Proxy: proxyURL,
Checksum: expectedChecksum,
Parallel: parallelConfig,
Recursive: recursiveEnabled,
MaxDepth: maxDepth,
ProgressCallback: progressFn,
Ctx: ctx,
}
```
## Phase 2: Protocol Resolution
The protocol registry resolves the handler by URL scheme:
```mermaid
flowchart LR
URL["https://example.com"]:::accent6
Parse["Parse scheme\n→ https"]:::accent7
Normalized["Normalize\nhttps → http"]:::accent7
Registry["Registry lookup\nprotocols[http]"]:::accent1
Handler["HTTP protocol\nhandler"]:::accent1
URL --> Parse --> Normalized --> Registry --> Handler
classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff
classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff
classDef accent7 fill:#64748B,stroke:#475569,color:#fff
```
Scheme normalization maps:
- `https://` → `http` (same handler)
- `ftps://` → `ftp` (same handler)
- `webdavs://` → `webdav` (same handler)
## Phase 3: Output Setup
The `internal/output` package creates an atomic writer:
1. **Temp file** — Writes to `<filename>.goget.tmp` during download
2. **Parent directories** — If `--create-dirs` is enabled (default: `true`), missing parent directories are created via `os.MkdirAll` before opening the output file. Pass `--create-dirs=false` to restore strict behaviour
3. **Resume metadata** — Reads `<filename>.goget.meta` if resuming
4. **Progress callback** — Hooks into the writer for real-time speed/ETA
```go
writer, err := output.NewWriter(&output.WriterConfig{
Output: outputFile,
Atomic: true,
Resume: resumeEnabled,
CreateDirs: true, // auto-create parent dirs (curl --create-dirs)
ProgressCallback: progressFunc,
})
```
### Resume Metadata Format
```json
{
"downloaded_bytes": 524288000,
"etag": "\"abc123\"",
"last_modified": "Mon, 01 Jun 2026 12:00:00 GMT",
"url": "https://example.com/file.zip",
"last_write": "2026-06-01T12:05:00Z",
"chunks": {
"0": 131072000,
"1": 131072000,
"2": 131072000,
"3": 131072000
}
}
```
For parallel downloads, each chunk's progress is individually tracked.
## Phase 4: Download Strategy
### Sequential Download
Used when the file is under 100 MB or `--parallel 1` is set:
```go
resp, _ := client.Do(request)
io.Copy(writer, resp.Body)
```
### Parallel Chunked Download
Triggered automatically for files over 100 MB, or explicitly with `--parallel N`:
```mermaid
sequenceDiagram
participant Main
participant Chunk1
participant Chunk2
participant Chunk3
participant Chunk4
participant Writer as Atomic Writer
Main->>Main: HEAD request → get file size
Main->>Main: Split into N equal chunks
Main->>Chunk1: Start: bytes 0-13107199
Main->>Chunk2: Start: bytes 13107200-26214399
Main->>Chunk3: Start: bytes 26214400-39321599
Main->>Chunk4: Start: bytes 39321600-52428799
Chunk1->>Writer: Write bytes to temp/chunk_0
Chunk2->>Writer: Write bytes to temp/chunk_1
Chunk3->>Writer: Write bytes to temp/chunk_2
Chunk4->>Writer: Write bytes to temp/chunk_3
Main->>Writer: Merge chunks → final file
```
Each chunk downloads via a separate HTTP `Range` request:
```http
GET /large.iso HTTP/1.1
Host: example.com
Range: bytes=13107200-26214399
```
### Concurrency Control
- Max 4 parallel connections by default
- Customizable via `--parallel N` or config `parallel` key
- Min chunk size: 1 MB
- Max chunk size: 50 MB
## Phase 5: Post-Processing
### Decompression
If `auto_decompress` is enabled (default) and the server sends compressed content, the response body is transparently decompressed:
| Content-Encoding | Handler |
|---|---|
| `gzip` | `compress/gzip` |
| `deflate` | `compress/flate` |
| `zlib` | Internal |
| `bzip2` | Internal |
| `lzw` | Internal |
Use `--no-decompress` to preserve the compressed response.
### Checksum Verification
If `--checksum` or `--checksum-file` is provided, the downloaded file is hashed and compared:
```
Expected: a1b2c3d4...
Actual: a1b2c3d4...
Checksum OK
```
Supported algorithms: SHA-256, SHA-512, SHA3-256, SHA3-512, BLAKE2b, MD5.
### Atomic Rename
On successful completion, the temp file is atomically renamed:
```go
os.Rename("file.zip.goget.tmp", "file.zip")
```
If the download fails or is interrupted, the temp file remains for resume.
### PGP Verification and Decryption
If `--pgp-verify` or `--pgp-decrypt` is set, the downloaded file is processed via `golang.org/x/crypto/openpgp`:
- **Detached signature verification** — `--pgp-sig file.sig --pgp-key public.key`
- **Decryption** — `--pgp-decrypt --pgp-key private.key --pgp-passphrase "secret"`
- Auto-detect signature files: `.asc`, `.sig`
The decrypted file replaces the encrypted one (`.gpg` → stripped extension, or `.decrypted` suffix).
### HSTS Cache
After every successful HTTPS connection, the HSTS cache (`~/.config/goget/hsts`) is updated per RFC 6797. Expired entries are pruned on load. This ensures that future `http://` requests to known hosts are automatically upgraded to `https://`.
### Graceful Shutdown (SIGINT)
When the user sends Ctrl+C (SIGINT) during a download, goget persists partial progress to the `.goget.meta` sidecar for **all protocols** (HTTP, FTP, SFTP, WebDAV) and exits with code 130. A subsequent `goget --resume` picks up where it left off. The signal handler uses `signal.NotifyContext` and propagates cancellation through the download pipeline via `ctx.Err()` checks after every read.
## Smart Timeout
When no explicit `--timeout` is set, goget calculates a timeout based on file size:
```
timeout = (fileSize / minSpeed) × safetyFactor
```
Where:
- `minSpeed` = 10 KB/s (conservative minimum)
- `safetyFactor` = 3.0
- `minTimeout` = 30 seconds
- `maxTimeout` = 24 hours
A 1 GB file at 10 KB/s → 34 hours (capped to 24 hours max).
A 10 MB file at 10 KB/s → 51 minutes.
## Rate Limiting
`--rate-limit` uses a token bucket algorithm:
```go
bucket := transport.NewTokenBucket(rate)
// For each read:
bucket.Wait(n)
```
The token bucket allows short bursts above the limit while maintaining the average rate.
### Speed Format
```
--rate-limit 1MB/s # 1,000,000 bytes/sec
--rate-limit 500KB/s # 500,000 bytes/sec
--rate-limit 1GB/s # 1,000,000,000 bytes/sec
--rate-limit 100 # 100 bytes/sec (plain number)
```
## Retry Logic
When a download fails, goget retries with exponential backoff:
```
Attempt 1: wait 1s, retry
Attempt 2: wait 2s, retry
Attempt 3: wait 4s, retry
...
Attempt N: wait min(2^(N-1) × 1s, 30s), retry
```
Max retries default to 3. Use `--max-retries N` to increase. Use `--retry-all-errors` to retry on HTTP 4xx/5xx (by default, only 5xx server errors trigger retries).
docs/migration.md
+173
View File
@@ -0,0 +1,173 @@
# Migration from curl & wget
goget is designed as a drop-in replacement for `curl` and `wget` in most workflows. This guide helps you migrate existing scripts and commands.
## Quick Translation
### curl → goget
| curl | goget | Notes |
|---|---|---|
| `curl -O <URL>` | `goget --url <URL>` | Download with auto filename |
| `curl -o file.zip <URL>` | `goget --url <URL> --output file.zip` | Named output |
| `curl -C - <URL>` | `goget --url <URL> --resume` | Resume download |
| `curl -L <URL>` | `goget --url <URL>` | Redirects followed by default |
| `curl -H "K: V" <URL>` | `goget --header "K: V" --url <URL>` | Custom headers |
| `curl -d "data" <URL>` | `goget --data "data" --url <URL>` | POST data |
| `curl -F "f=@file" <URL>` | `goget --form "f=@file" --url <URL>` | Multipart upload |
| `curl -k <URL>` | `goget --insecure --url <URL>` | Skip TLS verify |
| `curl -x <proxy> <URL>` | `goget --proxy <proxy> --url <URL>` | HTTP proxy |
| `curl -I <URL>` | `goget --spider --url <URL>` | HEAD request only |
| `curl --create-dirs <URL>` | `goget --url <URL>` | Create dirs (default in goget) |
| `curl --max-time 30 <URL>` | `goget --max-time 30s --url <URL>` | Max transfer time |
| `curl --cert cert.pem --key key.pem` | `goget --cert cert.pem --key key.pem --url <URL>` | mTLS |
| `curl -w "%{http_code}" <URL>` | `goget --write-out "%{http_code}" --url <URL>` | Metadata output |
| `curl -s <URL>` | `goget --quiet --url <URL>` | Silent mode |
| `curl -v <URL>` | `goget --verbose --url <URL>` | Verbose mode |
| `curl -b cookies.txt <URL>` | `goget --cookie-jar cookies.txt --url <URL>` | Cookies |
| `curl --ssl-key-log-file keys.log` | `goget --ssl-key-log keys.log --url <URL>` | SSL key log |
| `curl --retry-all-errors <URL>` | `goget --retry-all-errors --url <URL>` | Retry on all errors |
### wget → goget
| wget | goget | Notes |
|---|---|---|
| `wget <URL>` | `goget --url <URL>` | Basic download |
| `wget -O file.zip <URL>` | `goget --url <URL> --output file.zip` | Named output |
| `wget -c <URL>` | `goget --url <URL> --resume` | Resume download |
| `wget -r <URL>` | `goget --url <URL> --recursive` | Recursive download |
| `wget -r -l 3 <URL>` | `goget --url <URL> --recursive --max-depth 3` | Depth limit |
| `wget -m <URL>` | `goget --url <URL> --mirror` | Full mirror |
| `wget --mirror --convert-links` | `goget --url <URL> --mirror` | Mirror + convert links |
| `wget -p <URL>` | `goget --url <URL> --page-requisites` | Page requisites |
| `wget -A "*.pdf" <URL>` | `goget --url <URL> --accept "*.pdf"` | Accept pattern |
| `wget -R "*.zip" <URL>` | `goget --url <URL> --reject "*.zip"` | Reject pattern |
| `wget -np <URL>` | `goget --url <URL> --no-parent` | No parent dir |
| `wget -H <URL>` | `goget --url <URL> --span-hosts` | Span hosts |
| `wget -D a.com,b.com <URL>` | `goget --url <URL> --domains a.com,b.com` | Limit domains |
| `wget -w 2 <URL>` | `goget --url <URL> --wait 2s` | Delay between requests |
| `wget --random-wait <URL>` | `goget --url <URL> --random-wait` | Random delay |
| `wget --limit-rate 1M <URL>` | `goget --url <URL> --rate-limit 1MB/s` | Rate limit |
| `wget --warc-file=archive` | `goget --url <URL> --warc-file archive.warc` | WARC output |
| `wget --progress=dot <URL>` | `goget --url <URL> --progress=dot` | Dot progress |
| `wget --adjust-extension <URL>` | `goget --url <URL> --adjust-extension` | Add .html extension |
| `wget --no-clobber <URL>` | `goget --url <URL> --no-clobber` | Skip existing files |
| `wget --timestamping <URL>` | `goget --url <URL> --timestamping` | Time-stamp check |
| `wget --continue <URL>` | `goget --url <URL> --continue` | Continue partial |
| `wget -nc <URL>` | `goget --url <URL> --no-clobber` | No clobber |
| `wget -nd <URL>` | `goget --url <URL> --cut-dirs 999` | No directories |
| `wget --no-directories <URL>` | `goget --url <URL>` | Create dirs (default in goget) |
| `wget -E <URL>` | `goget --url <URL> --adjust-extension` | Adjust extension |
## Behavior Differences
### Redirects
| Tool | Default Behavior |
|---|---|
| **curl** | Does **not** follow redirects by default |
| **wget** | Follows redirects by default |
| **goget** | Follows redirects by default (use `--no-redirect` to disable) |
### Output Filenames
| Tool | Default Output |
|---|---|
| **curl** | stdout |
| **wget** | Auto-detected from URL |
| **goget** | Auto-detected from URL (use `--output -` for stdout) |
### Verbosity
| Tool | Default |
|---|---|
| **curl** | Silent (progress bar) |
| **wget** | Verbose (progress + status) |
| **goget** | Verbose (progress bar + status, use `--quiet` for silent) |
## Features Unique to goget
These features have no direct curl/wget equivalent:
| Feature | Flag |
|---|---|
| **IPv6-first connections** | Always on (disable with `--no-ipv4`) |
| **Smart timeout** | `--timeout 0` (auto-calculates from file size) |
| **9 protocols** | HTTP, FTP, SFTP, WebDAV, Gemini, Gopher, data:, file:// |
| **Quantum-safe checksums** | `--checksum-algo sha3-256`, `--checksum-algo sha3-512`, `--checksum-algo blake2b` |
| **Download queue** | `--queue`, `--queue-list`, `--queue-process` |
| **Scheduling** | `--schedule "0 2 * * *"` |
| **Post-download hooks** | `--on-complete "script.sh"` |
| **JSON progress** | `--json --no-progress` |
| **Metalink** | `--metalink`, `--metalink-file` |
| **HSTS cache** | `--hsts-file` (RFC 6797) |
| **Go library API** | `import "codeberg.org/petrbalvin/goget/pkg/api"` |
| **`--create-dirs` (default true)** | Auto-creates missing parent directories of target file |
| **SOCKS5 proxy** | `socks5://` (local DNS) and `socks5h://` (remote DNS) |
| **Shell completion** | `--completion bash`, `--completion zsh`, `--completion fish` |
## Script Migration Patterns
### CI/CD Download Script
```bash
# Before (wget)
wget -q -O data.zip https://example.com/data.zip && unzip data.zip
# After (goget)
goget --quiet --url https://example.com/data.zip --output data.zip && unzip data.zip
```
### API Request
```bash
# Before (curl)
curl -s -H "Authorization: Bearer $TOKEN" -o output.json https://api.example.com/data
# After (goget)
goget --quiet --header "Authorization: Bearer $TOKEN" --url https://api.example.com/data --output output.json
```
### Mirror Website
```bash
# Before (wget)
wget --mirror --convert-links --adjust-extension --page-requisites --no-parent https://example.com
# After (goget)
goget --url https://example.com --mirror
```
### Scheduled Download
```bash
# Before (cron + wget)
# crontab: 0 2 * * * wget -q -O /backup/daily.zip https://example.com/daily.zip
# After (goget with built-in scheduling)
goget --url https://example.com/daily.zip --output /backup/daily.zip --schedule "0 2 * * *"
```
### Multi-File Download with Checksums
```bash
# Before (curl + sha256sum)
curl -O https://example.com/file.zip
curl -O https://example.com/SHA256SUMS
sha256sum -c SHA256SUMS
# After (goget)
goget --url https://example.com/file.zip --checksum-file SHA256SUMS
```
## What's Missing
Features from curl/wget not yet available in goget:
| Feature | Status |
|---|---|
| **Bandwidth throttling per-domain** | Global rate limit only |
| **DNS-over-HTTPS** | Not yet implemented |
| **Short flags** (`-O`, `-o`, `-s`, `-v`) | Long flags only (`--output`, `--quiet`, `--verbose`) |
| **MIME type detection for local files** | Not implemented |
| **Brotli decompression** | gzip/deflate only |
docs/protocols.md
+243
View File
@@ -0,0 +1,243 @@
# Protocol Support
goget supports 9 network protocols through a unified abstraction layer. Each protocol implements `core.Protocol` and registers with a global registry.
## Supported Protocols
| Scheme | Protocol | Resume | Parallel | Upload | Recursive | Library |
|---|---|---|---|---|---|---|
| `http://` `https://` | HTTP/HTTPS | ✓ | ✓ | ✓ | ✓ | Go stdlib `net/http` |
| `ftp://` `ftps://` | FTP/FTPS | ✓ | — | — | ✓ | Custom implementation |
| `sftp://` | SFTP | ✓ | ✓ | — | ✓ | `golang.org/x/crypto/ssh` |
| `file://` | Local file | — | — | — | ✓ | Go stdlib `os` |
| `data:` | Data URI | — | — | — | — | Go stdlib |
| `gopher://` | Gopher | — | — | — | — | Custom implementation |
| `webdav://` `webdavs://` | WebDAV | ✓ | — | ✓ | ✓ | HTTP (via delegation) |
| `gemini://` | Gemini | — | — | — | — | Custom implementation |
## HTTP/HTTPS (`http://`, `https://`)
The primary protocol handler with the most features.
### Features
- **HTTP/1.1 and HTTP/2** — Automatic protocol negotiation via ALPN
- **TLS 1.2 and 1.3** — With configurable cipher suites
- **IPv6-first** — Prefers IPv6 addresses; falls back to IPv4 only when no IPv6 record exists
- **Parallel chunked downloads** — Splits files over 100 MB into chunks and downloads them concurrently using `Range` requests
- **Resume** — Supports `Range`/`Accept-Ranges` for resuming interrupted downloads
- **Upload** — PUT and POST with `Content-Type` detection
- **Compression** — Automatic gzip, deflate, brotli (if available) decompression
- **Cookies** — Netscape-format cookie jar for session persistence
- **HSTS** — RFC 6797 cache for automatic HTTPS upgrades
- **OAuth 2.0** — Client credentials, authorization code, and refresh token flows
- **Digest authentication** — RFC 7616
- **Basic authentication** — RFC 7617
- **Rate limiting** — Token bucket bandwidth capping
- **Proxy** — HTTP CONNECT proxy support
- **Custom DNS** — Configurable DNS servers via `--dns-servers`
- **Interface binding** — Linux-specific `SO_BINDTODEVICE`
- **Timing breakdown** — DNS, TCP, TLS, TTFB timing via `--trace-time`
- **WARC output** — Web ARChive format for archival
### Configuration
```bash
# TLS settings
--insecure Skip TLS certificate verification
--cert client.pem Client certificate (mTLS)
--key client.key Client private key (mTLS)
--pinned-cert SHA256 Certificate pinning
--ssl-key-log file TLS key log for Wireshark
# Proxy
--proxy http://proxy:8080
# Rate limiting
--rate-limit 1MB/s
# Headers
--header "X-Custom: value"
# Authentication
--username user --password pass
--auth-type basic|digest|auto
# Timing
--trace-time
```
## FTP/FTPS (`ftp://`, `ftps://`)
Custom FTP implementation supporting active and passive modes.
### Features
- **Active and passive modes** — Auto-detection; passive preferred
- **Explicit TLS (FTPS)** — AUTH TLS command on port 21
- **Resume** — REST command for partial transfers
- **Directory listing** — For recursive operations
### Limitations
- No parallel downloads
- No upload support
### Configuration
```bash
# Explicit TLS (FTPS)
--url ftps://ftp.example.com/file.zip
# Username and password
--url ftp://ftp.example.com/file.zip --username user --password pass
# Anonymous (default)
--url ftp://anonymous@ftp.example.com/file.zip
```
## SFTP (`sftp://`)
SFTP over SSH protocol.
### Features
- **SSH key authentication** — Auto-detection from `~/.ssh/id_rsa`, `~/.ssh/id_ed25519`
- **Password authentication** — Via `--password` or `.netrc`
- **Known hosts verification** — Via `~/.ssh/known_hosts`
- **Resume** — Partial transfer support; parallel recursive downloads available via `--recursive-parallel N` with connection pooling
- **`known_hosts` verification** — Reads `~/.ssh/known_hosts`; does not write to it. First-connection accept via `--ssh-insecure`
### Configuration
```bash
# With known hosts verification
--url sftp://user@host:/path/to/file.zip
# Skip host key verification (first connection)
--url sftp://user@host:/path/to/file.zip --ssh-insecure
# Custom known hosts file
--url sftp://user@host:/path/to/file.zip --known-hosts ~/.ssh/known_hosts
# With password
--url sftp://user@host:/path/to/file.zip --password "secret"
```
## Local File (`file://`)
Reads or copies local files.
### Features
- **File copy** — Copies local files to the output path
- **Directory listing** — For recursive operations
- **No network required** — Works fully offline
### Usage
```bash
# Copy a local file
goget --url file:///home/user/document.pdf
# Recursive directory copy
goget --url file:///home/user/docs/ --recursive --output ./backup/
```
## Data URI (`data:`)
Handles inline data URIs as defined in RFC 2397.
### Features
- **Base64 decoding** — `data:application/zip;base64,UEsDB...`
- **Plain text** — `data:text/plain,Hello%20World`
- **No network required** — Data is embedded in the URI
### Usage
```bash
# Base64 encoded data
goget --url "data:application/zip;base64,UEsDB..." --output decoded.zip
# Plain text
goget --url "data:text/plain,Hello%20World" --output hello.txt
```
## Gopher (`gopher://`)
Implements the Gopher protocol as specified in RFC 1436.
### Features
- **Type parsing** — Respects Gopher type codes (0 = text, 1 = directory, 9 = binary)
- **Directory listing** — For recursive operations
- **Text and binary downloads**
### Limitations
- No resume
- No parallel downloads
- No encryption (Gopher has no TLS)
- No upload
### Usage
```bash
# Download a file from a Gopher server
goget --url gopher://gopher.example.com/9/file.zip
# Browse a Gopher directory
goget --url gopher://gopher.example.com/1/
```
## Gemini (`gemini://`)
Implements the Gemini protocol (a lightweight alternative to HTTP/HTTPS).
### Features
- **TLS** — Mandatory TLS connections with TLS 1.2 minimum
- **Redirect following** — Respects Gemini redirect status codes (3x), up to 10 redirects
- **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled
- **Text and binary downloads** — 2x success responses stream content directly
### Limitations
- No resume
- No parallel downloads
- No upload (Gemini is download-only)
### Usage
```bash
# Download a file
goget --url gemini://gemini.example.com/file.zip
# Browse a capsule
goget --url gemini://gemini.example.com/
```
## Protocol Resolution Flow
```mermaid
flowchart TD
URL[URL input]:::accent6
Parse["Parse URL"]:::accent7
Normalize["Normalize scheme\n(https→http, ftps→ftp)"]:::accent7
Registry["Look up in protocol registry"]:::accent1
Found{"Protocol found?"}:::accent4
Error["Unsupported protocol error"]:::accent4
Handle["Delegate to protocol handler"]:::accent1
URL --> Parse --> Normalize --> Registry
Registry --> Found
Found -->|Yes| Handle
Found -->|No| Error
classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff
classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff
classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff
classDef accent7 fill:#64748B,stroke:#475569,color:#fff
```
docs/recursive-mirror.md
+256
View File
@@ -0,0 +1,256 @@
# Recursive Downloading & Website Mirroring
goget can recursively download linked pages and mirror entire websites for offline viewing. This document explains the two modes, their configuration, and the underlying architecture.
## Quick Comparison
| Feature | Recursive (`--recursive`) | Mirror (`--mirror`) |
|---|---|---|
| Depth limit | By default (`--max-depth N`) | Infinite |
| Link conversion | Optional (`--convert-links`) | Automatic |
| Page requisites | Manual (`--page-requisites`) | Automatic |
| robots.txt | Respected by default | Respected by default |
| Use case | Download a subtree of a site | Full offline archive |
## Recursive Mode
Recursive mode follows links from HTML pages and downloads linked resources.
```bash
# Basic recursive download, depth 3
goget --url https://example.com/docs/ --recursive --max-depth 3
# Only PDF files
goget --recursive --accept "*.pdf" --url https://example.com/docs/
# Only images and HTML
goget --recursive --accept "image/*,text/html" --url https://example.com/
```
### Link Filtering
```mermaid
flowchart TD
HTML[Parse HTML page]:::accent1
Extract["Extract all links\n(a, img, link, script)"]:::accent1
FilterByDomain{"Domain match?"}:::accent7
FilterByPattern{"Accept pattern\nmatch?"}:::accent7
FilterByDepth{"Depth ≤ max?"}:::accent7
FilterByParent{"No-parent\ncheck?"}:::accent7
ExcludeFilter{"Not in exclude\nlist?"}:::accent7
Enqueue["Add to download queue"]:::accent1
Skip["Skip"]:::accent4
HTML --> Extract --> FilterByDomain
FilterByDomain -->|Yes| FilterByPattern
FilterByDomain -->|No, --span-hosts| FilterByPattern
FilterByDomain -->|No| Skip
FilterByPattern -->|Yes| FilterByDepth
FilterByPattern -->|No| Skip
FilterByDepth -->|Yes| FilterByParent
FilterByDepth -->|No| Skip
FilterByParent -->|Pass| ExcludeFilter
FilterByParent -->|Fail| Skip
ExcludeFilter -->|Pass| Enqueue
ExcludeFilter -->|Fail| Skip
classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff
classDef accent4 fill:#EF4444,stroke:#DC2626,color:#fff
classDef accent7 fill:#64748B,stroke:#475569,color:#fff
```
### Accept Patterns
The `--accept` flag supports two types:
| Type | Example | Matches |
|---|---|---|
| **Glob** | `*.pdf` | Filenames ending in `.pdf` |
| **Glob** | `*.html,*.css` | Multiple patterns (comma-separated) |
| **MIME** | `text/html` | Exact MIME type |
| **MIME** | `image/*` | Any MIME in the `image/` category |
### Domain Control
| Flag | Effect |
|---|---|
| (default) | Only follow links within the starting domain |
| `--span-hosts` | Follow links to any domain |
| `--domains a.com,b.com` | Restrict to specific domains |
| `--follow-external` | Follow external links (aliases for `--span-hosts`) |
| `--no-parent` | Don't ascendant above the starting URL path |
| `--recursive-parallel` | `<N>` | Number of concurrent file downloads in recursive mode across all protocols (`0` = sequential). HTTP, WebDAV, FTP, and SFTP each get a worker pool bounded by a semaphore; subdirectory recursion propagates the limit. Connection pooling (FTP, SFTP) opens N independent control connections |
### Request Throttling
```bash
# Fixed 2-second delay between requests
goget --recursive --wait 2s --url https://example.com/docs/
# Randomize delay (0.5x – 1.5x of --wait)
goget --recursive --wait 2s --random-wait --url https://example.com/docs/
```
### Page Requisites
`--page-requisites` downloads CSS, JavaScript, and images needed to render each HTML page:
```bash
goget --recursive --page-requisites --url https://example.com/page.html
```
This parses `<link>`, `<script>`, `<img>`, `<source>`, and `<video>` tags and downloads their `src`/`href` targets.
## Mirror Mode
| `--dry-run` | — | List all files that would be downloaded in recursive/mirror mode without saving to disk |
`--mirror` is a shorthand for `--recursive --convert-links --page-requisites --infinite-depth`:
```bash
# Full site mirror
goget --url https://example.com --mirror --output ./mirror
# With link conversion for offline viewing
goget --url https://example.com --mirror --convert-links --output ./mirror
```
### Link Conversion
When `--convert-links` is enabled, HTML links are rewritten for local offline viewing:
```
Before: <a href="https://example.com/about/">About</a>
After: <a href="./about/index.html">About</a>
Before: <img src="https://example.com/img/logo.png">
After: <img src="./img/logo.png">
```
The link rewriter handles:
- Absolute URLs → relative paths
- Protocol-relative URLs (`//example.com/...`)
- Root-relative URLs (`/about/`)
- CSS `url()` references
### Robots.txt
goget respects `robots.txt` by default. Use `--no-robots` to ignore it:
```bash
goget --mirror --no-robots --url https://example.com
```
### Asset Control
```bash
# Mirror but skip CSS/JS/images
goget --mirror --no-mirror-assets --url https://example.com
# Only mirror specific file types
goget --mirror --accept "*.html,*.jpg" --url https://example.com
```
## Output Structure
Mirrored sites preserve the URL path structure:
```
mirror/
├── index.html
├── about/
│ └── index.html
├── blog/
│ ├── index.html
│ └── post-1.html
├── css/
│ └── style.css
└── img/
└── logo.png
```
Use `--cut-dirs N` to strip directory components:
```bash
# Original: https://example.com/pub/docs/file.html
# Without cut: ./mirror/pub/docs/file.html
# With --cut-dirs 2: ./mirror/doc/file.html
goget --mirror --cut-dirs 2 --url https://example.com/pub/docs/
```
## Architecture
### Crawler (`internal/recursive`)
```go
type CrawlerConfig struct {
MaxDepth int
FollowExternal bool
ExcludePatterns []string
IncludePatterns []string
AcceptPatterns []string
NoParent bool
PageRequisites bool
ConvertLinks bool
OutputDir string
Delay time.Duration
RandomWait bool
MaxWorkers int // concurrent download workers (0 = sequential)
}
```
The crawler:
1. Downloads an HTML page
2. Parses it with `golang.org/x/net/html`
3. Extracts all links (`a[href]`, `img[src]`, `link[href]`, `script[src]`)
4. Filters against accept/reject patterns, domain rules, and depth limits
5. Enqueues matching URLs for download
6. Applies configurable delay between requests
### Mirror (`internal/mirror`)
The mirror wraps the crawler with additional configuration:
- Infinite depth (or configurable via `MirrorConfig.MaxDepth`)
- Automatic page requisites
- Automatic link conversion
- robots.txt compliance
### Link Rewriting (`internal/linkrewrite`)
After the download completes, links are rewritten in-place:
1. Read each HTML file
2. Parse with `golang.org/x/net/html`
3. Identify all `href` and `src` attributes
4. Convert absolute URLs to relative paths
5. Write back the modified HTML (backup `.orig` if `--backup-converted`)
## Timestamping
`--timestamping` downloads a file only if the server copy is newer than the local copy:
```bash
goget --timestamping --url https://example.com/file.zip
```
goget compares the `Last-Modified` header against the local file's modification time.
## WARC Archiving
For legal and archival compliance, output can be written in WARC format:
```bash
goget --warc-file archive.warc --url https://example.com/page.html
```
Each downloaded resource gets a separate WARC record with:
- Request metadata (URL, timestamp, headers)
- Response metadata (status code, content type, headers)
- Raw response body
## Performance Tips
- **Limit depth** — `--max-depth 3` prevents runaway recursion on large sites
- **Filter aggressively** — Use `--accept "*.pdf,*.html"` and `--reject "*.zip"` to narrow scope
- **Use delay** — `--wait 500ms` prevents server overload and IP blocks
- **Limit domains** — `--domains example.com` prevents crawling external CDNs
- **Skip assets** — `--no-mirror-assets` for text-only archives
docs/security.md
+258
View File
@@ -0,0 +1,258 @@
# Security
goget implements multiple security features to protect both the client and server during file transfers. This document covers TLS configuration, certificate handling, checksum verification, PGP support, HSTS, and SSRF protection.
## TLS / HTTPS
### Default Security
- **TLS 1.2 minimum** — TLS 1.0 and 1.1 are disabled
- **Certificate verification** — Enabled by default against system CA bundle
- **HTTP/2 support** — Automatic ALPN negotiation
### Certificate Pinning
Pin a specific certificate by its SHA-256 hash:
```bash
goget --pinned-cert "a1b2c3d4e5f6..." --url https://example.com
```
If the server certificate's SHA-256 doesn't match, the connection is aborted.
### Client Certificates (mTLS)
Authenticate the client to the server with mutual TLS:
```bash
goget --cert client.pem --key client.key --url https://mtls.example.com
```
### Custom CA Bundle
```bash
goget --cacert custom-ca.pem --url https://internal.example.com
```
### Insecure Mode
Disable all TLS verification (⚠ **for testing only**):
```bash
goget --insecure --url https://self-signed.local
```
### SSLKEYLOGFILE
Log TLS session keys for Wireshark debugging:
```bash
goget --ssl-key-log tls.keys --url https://example.com
# Then: Wireshark → Preferences → TLS → (Pre-)Master-Secret log filename → tls.keys
```
## HSTS (HTTP Strict Transport Security)
Per RFC 6797, goget maintains an HSTS cache that automatically upgrades HTTP connections to HTTPS for known hosts.
### How It Works
```mermaid
sequenceDiagram
participant Client as goget
participant Server as Server
Client->>Server: GET http://example.com
Server-->>Client: 200 OK + Strict-Transport-Security: max-age=31536000
Client->>Client: Store: example.com → 365 days
Client->>Server: GET http://example.com/page (later)
Client->>Client: HSTS cache hit → upgrade to HTTPS
Client->>Server: GET https://example.com/page
```
### Cache Location
```
~/.config/goget/hsts
```
Can be overridden with `GOGET_HSTS` environment variable or `--hsts-file`.
### Cache Format
```json
[
{
"host": "example.com",
"include_subdomains": true,
"expires": "2027-06-01T00:00:00Z"
}
]
```
Expired entries are automatically pruned on load.
## Checksum Verification
Verify file integrity after download:
### Direct Hash
```bash
goget --checksum "a1b2c3d4e5f6..." --checksum-algo sha256 --url https://example.com/file.zip
```
Supported algorithms:
| Algorithm | Flag Value | Hash Length |
|---|---|---|
| SHA-256 | `sha256` | 64 hex chars |
| SHA-512 | `sha512` | 128 hex chars |
| SHA3-256 | `sha3-256` | 64 hex chars |
| SHA3-512 | `sha3-512` | 128 hex chars |
| BLAKE2b | `blake2b` | 128 hex chars |
| MD5 | `md5` | 32 hex chars |
### Checksum File
Supports standard `SHA256SUMS` format:
```bash
# SHA256SUMS file:
# a1b2c3d4... file.zip
# e5f6a7b8... *file.tar.gz
goget --checksum-file SHA256SUMS --url https://example.com/file.zip
```
Both plain (`file.zip`) and binary-mode (`*file.tar.gz`) formats are supported.
### Error Handling
On mismatch:
```
Checksum mismatch
Expected: a1b2c3d4...
Actual: e5f6a7b8...
```
The error is a typed `core.GogetError` with `Type: "CHECKSUM"` and `Details["expected"]` / `Details["actual"]`.
## PGP / GPG
### Signature Verification
Verify PGP detached signatures:
```bash
# With explicit signature file
goget --pgp-verify --pgp-sig file.sig --pgp-key public.key --url https://example.com/file
# Auto-detect signature file (tries: file.asc, file.sig, file.gpg)
goget --pgp-verify --pgp-key public.key --url https://example.com/file
```
### Decryption
Decrypt PGP-encrypted files:
```bash
# Decrypt with key and passphrase
goget --pgp-decrypt --pgp-key private.key --pgp-passphrase "secret" --url https://example.com/file.gpg
# Read passphrase from file (safer)
goget --pgp-decrypt --pgp-key private.key --pgp-passphrase-file pass.txt --url https://example.com/file.gpg
```
The decrypted file replaces the encrypted one:
- `file.gpg` → `file`
- `file.pgp` → `file`
- `file.asc` → `file`
- Other → `filename.decrypted`
> **Note:** PGP functionality uses `golang.org/x/crypto/openpgp`, which is frozen. A replacement will be provided in a future release.
## SSRF Protection
By default, goget blocks connections to private and internal IP ranges:
| Range | CIDR | Description |
|---|---|---|
| `10.0.0.0/8` | Private | RFC 1918 |
| `172.16.0.0/12` | Private | RFC 1918 |
| `127.0.0.0/8` | Loopback | Internal |
| `169.254.0.0/16` | Link-local | Auto-configuration |
| `::1/128` | IPv6 loopback | Internal |
### IPv4-Mapped IPv6 Normalization
DNS-controlled IPv4-mapped IPv6 addresses (e.g. `::ffff:10.0.0.1`) are now normalized to plain IPv4 before the private-IP check. This prevents an attacker from reaching internal hosts through DNS responses that embed IPv4 addresses in IPv6 form. Covered by a 17-case unit test in the transport layer.
Override with `--allow-private`:
```bash
goget --allow-private --url https://internal.example.com
```
## DNS Security
### Custom Resolvers
Bypass system DNS for specific queries:
```bash
goget --dns-servers 1.1.1.1,8.8.8.8 --url https://example.com
```
### IPv6-First
The transport dialer prefers IPv6 and only falls back to IPv4 when no IPv6 address is available. This reduces NAT-related attack surface:
```bash
# IPv6 only (abort if no IPv6)
goget --no-ipv4 --url https://example.com
```
## Credential Safety
### Safe Error Logging
When a URL contains user credentials (`https://user:pass@example.com`), `core.SafeURL()` strips them from error messages and logs:
```go
// Original: https://admin:secret@api.example.com/data
// Logged: https://api.example.com/data
```
### OAuth Error Sanitization
Non-2xx OAuth responses are sanitized to extract only the structured `error` / `error_description` fields (RFC 6749 §5.2). Providers that echo `client_secret` or other secrets in their error response bodies can no longer leak them into goget logs or error displays.
### Config Sanitization
`config.Save()` strips sensitive fields before writing to disk:
- `auth.password`
- `auth.oauth.client_secret`
- `auth.oauth.access_token`
- `auth.oauth.refresh_token`
### File Permission Hardening
Configuration files (`config.toml`), resume metadata sidecars (`.goget.meta`), and atomic temp files are written with mode `0600` (owner-only), so OAuth tokens, source URLs, and downloaded payloads are never world-readable on multi-user systems.
### Environment Variables
- `GOGET_CONFIG` — alternative config file path
- `GOGET_PASSWORD` — password override (no file storage)
- `GOGET_HSTS` — alternative HSTS cache path
- `NO_COLOR` — disable color output
## Timeouts & Limits
| Feature | Default | Description |
|---|---|---|
| Connection timeout | 30s (auto) | Smart calculation based on file size |
| Max total time | — (`--max-time`) | Abort if exceeded |
| Max file size | — (`--max-filesize`) | Abort if server response exceeds limit |
| Max retries | 3 | Exponential backoff with 30s cap |
| Retry delay | 1s initial, 2× multiplier | Configurable via `--retry-delay` |
go.mod
+14
View File
@@ -0,0 +1,14 @@
module codeberg.org/petrbalvin/goget
go 1.26.3
require (
golang.org/x/crypto v0.51.0
golang.org/x/net v0.54.0
)
require (
codeberg.org/petrbalvin/interpres v1.1.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.37.0 // indirect
)
go.sum
+12
View File
@@ -0,0 +1,12 @@
codeberg.org/petrbalvin/interpres v1.1.0 h1:RK3shdI3sglNuck3Z4JKPvha3Rh0ckjuqxuji7Ds2DI=
codeberg.org/petrbalvin/interpres v1.1.0/go.mod h1:sWgi5ijKNNLu5sCtjix9jG/guhdnDRlROENrFhrIt90=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
internal/archive/archive_test.go
+1156
View File
File diff suppressed because it is too large Load Diff
internal/archive/auto.go
+43
View File
@@ -0,0 +1,43 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"fmt"
"os"
"path/filepath"
)
type ExtractConfig struct {
DestDir string
StripComponents int
IncludePatterns []string
ExcludePatterns []string
PreservePermissions bool
Overwrite bool
Verbose bool
}
func DefaultExtractConfig() *ExtractConfig {
return &ExtractConfig{
DestDir: ".", StripComponents: 0, IncludePatterns: []string{},
ExcludePatterns: []string{}, PreservePermissions: true, Overwrite: true, Verbose: false,
}
}
func ExtractFile(srcPath, destDir string, cfg *ExtractConfig) error {
if cfg == nil {
cfg = DefaultExtractConfig()
}
srcFile, err := os.Open(srcPath)
if err != nil {
return fmt.Errorf("failed to open source file: %w", err)
}
defer srcFile.Close()
return AutoExtract(filepath.Base(srcPath), srcFile, destDir)
}
func ShouldExtractFile(filename string) bool { return IsArchiveFormat(filename) }
func ListSupportedFormats() []string { return ListSupported() }
func SupportsFormat(format string) bool { return Supports(format) }
internal/archive/fuzz_test.go
+80
View File
@@ -0,0 +1,80 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"bytes"
"testing"
)
// FuzzTarExtract tests tar extraction with fuzzed input to catch path traversal
// and other security issues in header parsing.
func FuzzTarExtract(f *testing.F) {
// Seed corpus with valid tar data
f.Add([]byte{})
// Minimal valid tar header (512 bytes of zeros)
emptyHeader := make([]byte, 512)
f.Add(emptyHeader)
// Tar file with a ".." path traversal attempt
traversalTar := makeTarWithName("../../../etc/passwd")
f.Add(traversalTar)
// Tar file with absolute path
absTar := makeTarWithName("/etc/passwd")
f.Add(absTar)
// Tar file with symlink pointing outside
symlinkTar := makeSymlinkTar("link", "../../../etc/passwd")
f.Add(symlinkTar)
f.Fuzz(func(t *testing.T, data []byte) {
// Fuzz the tar extraction — it must never panic or escape the temp dir
dir := t.TempDir()
ext := NewTarExtractor()
reader := bytes.NewReader(data)
// Ignore errors — we only care about panics and path escapes
_ = ext.Extract(reader, dir)
})
}
func makeTarWithName(name string) []byte {
// Create a minimal tar file with a regular file at the given path
var buf bytes.Buffer
nameBytes := []byte(name)
if len(nameBytes) > 99 {
nameBytes = nameBytes[:99]
}
header := make([]byte, 512)
copy(header[0:99], nameBytes)
// Set type flag to regular file
header[156] = '0'
// Set size to zero
buf.Write(header)
// Two zero blocks to mark end of archive
buf.Write(make([]byte, 1024))
return buf.Bytes()
}
func makeSymlinkTar(name, target string) []byte {
var buf bytes.Buffer
nameBytes := []byte(name)
if len(nameBytes) > 99 {
nameBytes = nameBytes[:99]
}
header := make([]byte, 512)
copy(header[0:99], nameBytes)
// Set type flag to symlink
header[156] = '2'
// Write link target
targetBytes := []byte(target)
if len(targetBytes) > 99 {
targetBytes = targetBytes[:99]
}
copy(header[157:256], targetBytes)
buf.Write(header)
buf.Write(make([]byte, 1024))
return buf.Bytes()
}
internal/archive/interface.go
+84
View File
@@ -0,0 +1,84 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"io"
"strings"
)
// Extractor is the interface for archive extraction
type Extractor interface {
Name() string
Extensions() []string
Extract(r io.Reader, destDir string) error
CanHandle(filename string) bool
}
// BaseExtractor provides a basic implementation
type BaseExtractor struct {
name string
extensions []string
}
func NewBaseExtractor(name string, extensions []string) *BaseExtractor {
return &BaseExtractor{name: name, extensions: extensions}
}
func (b *BaseExtractor) Name() string { return b.name }
func (b *BaseExtractor) Extensions() []string { return b.extensions }
func (b *BaseExtractor) CanHandle(filename string) bool {
filename = strings.ToLower(filename)
for _, ext := range b.extensions {
if strings.HasSuffix(filename, ext) {
return true
}
}
return false
}
// DetectArchiveFormat detects the archive format by file extension
func DetectArchiveFormat(filename string) string {
filename = strings.ToLower(filename)
if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") {
return "tar.gz"
}
if strings.HasSuffix(filename, ".tar.bz2") || strings.HasSuffix(filename, ".tbz2") {
return "tar.bz2"
}
if strings.HasSuffix(filename, ".zip") {
return "zip"
}
if strings.HasSuffix(filename, ".tar") {
return "tar"
}
return ""
}
// IsArchiveFormat checks whether the filename looks like an archive
func IsArchiveFormat(filename string) bool { return DetectArchiveFormat(filename) != "" }
// GetArchiveAndCompression splits filename into archive + compression parts
func GetArchiveAndCompression(filename string) (archive, compression string) {
filename = strings.ToLower(filename)
if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") {
return "tar", "gzip"
}
if strings.HasSuffix(filename, ".tar.bz2") || strings.HasSuffix(filename, ".tbz2") {
return "tar", "bzip2"
}
if strings.HasSuffix(filename, ".zip") {
return "zip", ""
}
if strings.HasSuffix(filename, ".tar") {
return "tar", ""
}
if strings.HasSuffix(filename, ".gz") {
return "", "gzip"
}
if strings.HasSuffix(filename, ".bz2") {
return "", "bzip2"
}
return "", ""
}
internal/archive/registry.go
+87
View File
@@ -0,0 +1,87 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"fmt"
"io"
"strings"
"sync"
)
type registry struct {
mu sync.RWMutex
extractors map[string]Extractor
}
var globalRegistry = &registry{extractors: make(map[string]Extractor)}
func Register(e Extractor) error {
globalRegistry.mu.Lock()
defer globalRegistry.mu.Unlock()
name := strings.ToLower(e.Name())
if name == "" {
return fmt.Errorf("extractor name cannot be empty")
}
if _, exists := globalRegistry.extractors[name]; exists {
return fmt.Errorf("extractor already registered: %s", name)
}
globalRegistry.extractors[name] = e
return nil
}
func Get(format string) (Extractor, error) {
globalRegistry.mu.RLock()
defer globalRegistry.mu.RUnlock()
e, exists := globalRegistry.extractors[strings.ToLower(format)]
if !exists {
return nil, fmt.Errorf("unsupported archive format: %s", format)
}
return e, nil
}
func GetByFilename(filename string) (Extractor, error) {
format := DetectArchiveFormat(filename)
if format == "" {
return nil, fmt.Errorf("unknown archive format for: %s", filename)
}
return Get(format)
}
func Extract(format string, src io.Reader, destDir string) error {
e, err := Get(format)
if err != nil {
return err
}
return e.Extract(src, destDir)
}
func ExtractByFilename(filename string, src io.Reader, destDir string) error {
e, err := GetByFilename(filename)
if err != nil {
return err
}
return e.Extract(src, destDir)
}
func AutoExtract(filename string, src io.Reader, destDir string) error {
return ExtractByFilename(filename, src, destDir)
}
func Supports(format string) bool {
globalRegistry.mu.RLock()
defer globalRegistry.mu.RUnlock()
_, exists := globalRegistry.extractors[strings.ToLower(format)]
return exists
}
func ListSupported() []string {
globalRegistry.mu.RLock()
defer globalRegistry.mu.RUnlock()
names := make([]string, 0, len(globalRegistry.extractors))
for name := range globalRegistry.extractors {
names = append(names, name)
}
return names
}
internal/archive/tar.go
+134
View File
@@ -0,0 +1,134 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"archive/tar"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
)
type tarExtractor struct{ *BaseExtractor }
func NewTarExtractor() *tarExtractor {
return &tarExtractor{NewBaseExtractor("tar", []string{".tar"})}
}
// Extract extracts a tar archive into destDir (streaming-friendly)
func (t *tarExtractor) Extract(r io.Reader, destDir string) error {
tr := tar.NewReader(r)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("failed to read tar header: %w", err)
}
if err := t.extractEntry(header, tr, destDir); err != nil {
return err
}
}
return nil
}
// extractEntry extracts a single entry from a tar archive
func (t *tarExtractor) extractEntry(header *tar.Header, r io.Reader, destDir string) error {
// Sanitize path to prevent tar-slip attacks
cleanName := filepath.Clean(header.Name)
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
return fmt.Errorf("unsafe path in tar: %s", header.Name)
}
targetPath := filepath.Join(destDir, cleanName)
switch header.Typeflag {
case tar.TypeDir:
// Create directory with original permissions
return os.MkdirAll(targetPath, os.FileMode(header.Mode))
case tar.TypeReg:
// Create parent directories
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
// Create file with original permissions (mask setuid/setgid/sticky)
mode := os.FileMode(header.Mode) & 0777
file, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return err
}
defer file.Close()
// Copy contents
if _, err := io.Copy(file, r); err != nil {
return err
}
// Restore timestamps
atime := header.AccessTime
if atime.IsZero() {
atime = time.Now()
}
mtime := header.ModTime
if mtime.IsZero() {
mtime = time.Now()
}
if err := os.Chtimes(targetPath, atime, mtime); err != nil {
// Non-fatal
}
return nil
case tar.TypeSymlink:
// Validate symlink target against path traversal
linkTarget := filepath.Clean(header.Linkname)
if filepath.IsAbs(linkTarget) || strings.HasPrefix(linkTarget, "..") {
return fmt.Errorf("symlink target outside extraction directory: %s", header.Linkname)
}
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
os.Remove(targetPath)
return os.Symlink(linkTarget, targetPath)
case tar.TypeLink:
// Hard link - validate target stays within destDir to prevent path traversal
cleanTarget := filepath.Clean(header.Linkname)
if filepath.IsAbs(cleanTarget) || strings.HasPrefix(cleanTarget, "..") {
return fmt.Errorf("hard link target outside extraction directory: %s", header.Linkname)
}
linkTarget := filepath.Join(destDir, cleanTarget)
// Ensure resolved path does not escape destDir
if !strings.HasPrefix(linkTarget, destDir+string(filepath.Separator)) && linkTarget != destDir {
return fmt.Errorf("hard link target outside extraction directory: %s", header.Linkname)
}
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
return os.Link(linkTarget, targetPath)
case tar.TypeChar, tar.TypeBlock:
// Device files - skip with warning (requires root)
return nil
case tar.TypeFifo:
// FIFO - skip (platform-specific)
return nil
default:
// Unknown type - skip
return nil
}
}
func init() { Register(NewTarExtractor()) }
internal/archive/tarbz2.go
+25
View File
@@ -0,0 +1,25 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"compress/bzip2"
"io"
)
type tarBz2Extractor struct {
*BaseExtractor
tar *tarExtractor
}
func NewTarBz2Extractor() *tarBz2Extractor {
return &tarBz2Extractor{NewBaseExtractor("tar.bz2", []string{".tar.bz2", ".tbz2"}), NewTarExtractor()}
}
func (t *tarBz2Extractor) Extract(r io.Reader, destDir string) error {
bz2r := bzip2.NewReader(r)
return t.tar.Extract(bz2r, destDir)
}
func init() { Register(NewTarBz2Extractor()) }
internal/archive/targz.go
+42
View File
@@ -0,0 +1,42 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"compress/gzip"
"fmt"
"io"
)
type tarGzExtractor struct {
*BaseExtractor
tar *tarExtractor
}
func NewTarGzExtractor() *tarGzExtractor {
return &tarGzExtractor{
BaseExtractor: NewBaseExtractor("tar.gz", []string{".tar.gz", ".tgz"}),
tar: NewTarExtractor(),
}
}
// Extract extracts a tar.gz archive into destDir
func (t *tarGzExtractor) Extract(r io.Reader, destDir string) error {
// Step 1: Create gzip reader for decompression
gzr, err := gzip.NewReader(r)
if err != nil {
return fmt.Errorf("failed to create gzip reader: %w", err)
}
defer func() {
if cerr := gzr.Close(); cerr != nil && err == nil {
err = cerr
}
}()
// Step 2: Extract tar from decompressed stream
// gzr implements io.Reader that returns DECOMPRESSED data
return t.tar.Extract(gzr, destDir)
}
func init() { Register(NewTarGzExtractor()) }
internal/archive/zip.go
+141
View File
@@ -0,0 +1,141 @@
//go:build linux || freebsd
// +build linux freebsd
package archive
import (
"archive/zip"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
type zipExtractor struct{ *BaseExtractor }
// MaxZipInputSize is the maximum size in bytes of a single file extracted
// from a zip archive. Files whose declared uncompressed size exceeds this
// limit are rejected outright; the limit is also enforced at copy time as
// a safety net in case the central directory lies (a known zip-bomb
// technique). Defaults to 1 GiB.
var MaxZipInputSize int64 = 1 << 30
// MaxZipBufferSize is the maximum size in bytes of the buffered
// (compressed) zip archive before it is opened. Archives larger than this
// are rejected to prevent the on-disk temp file from growing without
// bound. Defaults to 4 GiB.
var MaxZipBufferSize int64 = 4 << 30
func NewZipExtractor() *zipExtractor {
return &zipExtractor{NewBaseExtractor("zip", []string{".zip"})}
}
// Extract extracts a zip archive into destDir
func (z *zipExtractor) Extract(r io.Reader, destDir string) error {
// Zip.NewReader requires io.ReaderAt + size
// Buffer input into a temp file, bounded by MaxZipBufferSize to
// prevent a zip bomb from filling the disk with the (still
// compressed) archive while we try to open it.
tmpFile, err := os.CreateTemp("", "goget-zip-*.tmp")
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
defer os.Remove(tmpPath) // Cleanup temp file after completion
// LimitReader returns EOF after MaxZipBufferSize+1 bytes, which lets
// us detect "input exceeded the limit" by stat'ing the temp file.
if _, err := io.Copy(tmpFile, io.LimitReader(r, MaxZipBufferSize+1)); err != nil {
tmpFile.Close()
return fmt.Errorf("failed to buffer zip: %w", err)
}
if err := tmpFile.Close(); err != nil {
return fmt.Errorf("failed to close temp file: %w", err)
}
info, err := os.Stat(tmpPath)
if err != nil {
return fmt.Errorf("failed to stat temp file: %w", err)
}
if info.Size() > MaxZipBufferSize {
return fmt.Errorf("zip archive exceeds maximum size of %d bytes (possible zip bomb)", MaxZipBufferSize)
}
// Open zip reader (zip.OpenReader detects size automatically)
zipReader, err := zip.OpenReader(tmpPath)
if err != nil {
return fmt.Errorf("failed to open zip archive: %w", err)
}
defer zipReader.Close()
// Extract each file from archive
for _, file := range zipReader.File {
if err := z.extractFile(file, destDir); err != nil {
return fmt.Errorf("failed to extract %s: %w", file.Name, err)
}
}
return nil
}
// extractFile extracts a single file from zip archive
func (z *zipExtractor) extractFile(file *zip.File, destDir string) error {
// Sanitize path to prevent zip-slip attacks
cleanName := filepath.Clean(file.Name)
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
return fmt.Errorf("unsafe path in zip: %s", file.Name)
}
targetPath := filepath.Join(destDir, cleanName)
// Handle directories
if file.FileInfo().IsDir() {
return os.MkdirAll(targetPath, 0755)
}
// Reject files whose declared uncompressed size exceeds the limit.
// The central directory declares this for every file; a value larger
// than MaxZipInputSize is a strong zip-bomb signal and we refuse the
// whole archive (caller can decide whether to retry with a different
// policy).
if file.UncompressedSize64 > uint64(MaxZipInputSize) {
return fmt.Errorf("file %s too large: %d bytes uncompressed (max %d)", file.Name, file.UncompressedSize64, MaxZipInputSize)
}
// Create parent directories
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
// Open file in archive (automatically decompresses if compressed)
rc, err := file.Open()
if err != nil {
return err
}
defer rc.Close()
// Create output file with original permissions (mask setuid/setgid/sticky)
mode := file.Mode() & 0777
outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return err
}
defer outFile.Close()
// io.LimitReader is a safety net for archives whose central directory
// lies about UncompressedSize64 (or leaves it at 0). The primary
// defence is the UncompressedSize64 check above.
if _, err := io.Copy(outFile, io.LimitReader(rc, MaxZipInputSize)); err != nil {
return err
}
// Restore timestamps
if err := os.Chtimes(targetPath, file.Modified, file.Modified); err != nil {
// Non-fatal, ignore error
}
return nil
}
func init() { Register(NewZipExtractor()) }
internal/auth/auth_test.go
+1588
View File
File diff suppressed because it is too large Load Diff
internal/auth/basic.go
+76
View File
@@ -0,0 +1,76 @@
//go:build linux || freebsd
// +build linux freebsd
package auth
import (
"encoding/base64"
"net/http"
"net/url"
"strings"
)
// BasicAuth adds Basic Authentication header to the request
func BasicAuth(req *http.Request, username, password string) {
if username == "" {
return
}
auth := username + ":" + password
encoded := base64.StdEncoding.EncodeToString([]byte(auth))
req.Header.Set("Authorization", "Basic "+encoded)
}
// ParseURLAuth extracts username/password from URL (https://user:pass@host)
func ParseURLAuth(u *url.URL) (username, password string, hasAuth bool) {
if u == nil || u.User == nil {
return "", "", false
}
username = u.User.Username()
if username == "" {
return "", "", false
}
password, hasAuth = u.User.Password()
return username, password, true
}
// StripAuthFromURL removes credentials from URL for safe logging
func StripAuthFromURL(u *url.URL) *url.URL {
if u == nil {
return nil
}
// Clone URL to avoid modifying original
clean := *u
clean.User = nil
return &clean
}
// MaskPassword returns a masked version of the password for logging
func MaskPassword(password string) string {
if password == "" {
return ""
}
if len(password) <= 4 {
return "****"
}
return password[:2] + strings.Repeat("*", len(password)-2)
}
// GetAuthForURL gets credentials for a given URL (priority: CLI > URL > config)
func GetAuthForURL(cliUsername, cliPassword, configUsername, configPassword string, u *url.URL) (username, password string) {
// 1. CLI flags have highest priority
if cliUsername != "" {
return cliUsername, cliPassword
}
// 2. URL-embedded credentials
if urlUser, urlPass, ok := ParseURLAuth(u); ok {
return urlUser, urlPass
}
// 3. Config file credentials
if configUsername != "" {
return configUsername, configPassword
}
return "", ""
}
internal/auth/digest.go
+206
View File
@@ -0,0 +1,206 @@
// Package auth handles HTTP authentication schemes (Basic, Digest, OAuth 2.0).
//go:build linux || freebsd
// +build linux freebsd
package auth
import (
"crypto/md5"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strings"
)
// DigestAuthHandler handles HTTP Digest Authentication
type DigestAuthHandler struct {
username string
password string
}
// NewDigestAuthHandler creates new handler
func NewDigestAuthHandler(username, password string) *DigestAuthHandler {
return &DigestAuthHandler{username, password}
}
// HandleResponse processes a 401 response and returns a modified request
func (h *DigestAuthHandler) HandleResponse(req *http.Request, resp *http.Response) (*http.Request, error) {
// Parse WWW-Authenticate header
authHeader := resp.Header.Get("WWW-Authenticate")
if !strings.HasPrefix(authHeader, "Digest ") {
return nil, fmt.Errorf("not a digest auth challenge")
}
// Parse digest parameters
params := parseDigestParams(authHeader[7:])
// Validate required parameters
requiredParams := []string{"realm", "nonce"}
for _, param := range requiredParams {
if _, exists := params[param]; !exists {
return nil, fmt.Errorf("missing required digest parameter: %s", param)
}
}
// Calculate response hash according to RFC 2617 / RFC 7616
qop := params.get("qop")
nonceCount := params.get("nc")
cnonce := params.get("cnonce")
opaque := params.get("opaque")
// HA1 = MD5(username:realm:password) or MD5(username:realm:password):nonce:cnonce
ha1 := h.calculateHA1(params["realm"], params["nonce"], cnonce, qop)
// HA2 = MD5(method:uri) or MD5(method:uri:response-body) for auth-int.
// RFC 7616 §3.4 (and RFC 2617 §3.2.2.3) require uri to be the
// request-uri — the path *and* the query string, exactly as it
// appears in the request line. Using req.URL.Path strips the
// query, which causes digest auth to fail for any URL that carries
// a token or other parameter in the query string.
ha2 := h.calculateHA2(req.Method, req.URL.RequestURI(), qop)
// Response calculation based on qop
var response string
if qop == "auth" || qop == "auth-int" {
response = md5Sum(ha1 + ":" + params["nonce"] + ":" + nonceCount + ":" + cnonce + ":" + qop + ":" + ha2)
} else {
// No qop (old RFC 2617 style)
response = md5Sum(ha1 + ":" + params["nonce"] + ":" + ha2)
}
// Build Authorization header — see note above about request-uri
// vs path.
auth := h.buildAuthHeader(params, req.URL.RequestURI(), response, nonceCount, cnonce, qop, opaque)
// Clone request and add header
newReq := req.Clone(req.Context())
newReq.Header.Set("Authorization", auth)
return newReq, nil
}
func (h *DigestAuthHandler) calculateHA1(realm, nonce, cnonce, qop string) string {
a1 := h.username + ":" + realm + ":" + h.password
ha1 := md5Sum(a1)
// If qop is specified, we need to include nonce and cnonce
if qop == "auth" || qop == "auth-int" {
ha1 = md5Sum(ha1 + ":" + nonce + ":" + cnonce)
}
return ha1
}
func (h *DigestAuthHandler) calculateHA2(method, uri, qop string) string {
a2 := method + ":" + uri
// For auth-int, we would need to include body hash, but that's complex
// For now, we just support auth and no-qop modes
return md5Sum(a2)
}
func (h *DigestAuthHandler) buildAuthHeader(params map[string]string, uri, response, nc, cnonce, qop, opaque string) string {
parts := []string{
fmt.Sprintf(`username="%s"`, h.username),
fmt.Sprintf(`realm="%s"`, params["realm"]),
fmt.Sprintf(`nonce="%s"`, params["nonce"]),
fmt.Sprintf(`uri="%s"`, uri),
fmt.Sprintf(`response="%s"`, response),
}
if qop != "" {
parts = append(parts,
fmt.Sprintf(`qop=%s`, qop),
fmt.Sprintf(`nc=%s`, nc),
fmt.Sprintf(`cnonce="%s"`, cnonce),
)
}
if opaque != "" {
parts = append(parts, fmt.Sprintf(`opaque="%s"`, opaque))
}
return "Digest " + strings.Join(parts, ", ")
}
// GenerateCnonce generates a cryptographically random client nonce
func GenerateCnonce() string {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
// Fallback should never happen on a functioning system
// but we still produce a value rather than crash
for i := range b {
b[i] = byte(i)
}
}
return hex.EncodeToString(b)
}
// parseDigestParams parses the WWW-Authenticate header parameters
func parseDigestParams(header string) digestParams {
params := make(digestParams)
// Handle quoted and unquoted values
currentKey := ""
currentVal := ""
inQuotes := false
hasSeenEquals := false
for i := 0; i < len(header); i++ {
char := header[i]
switch {
case char == '=' && !inQuotes:
currentKey = strings.TrimSpace(currentKey)
currentVal = ""
hasSeenEquals = true
case char == '"' && (i == 0 || header[i-1] != '\\'):
inQuotes = !inQuotes
case char == ',' && !inQuotes:
if currentKey != "" {
params[currentKey] = strings.Trim(strings.TrimSpace(currentVal), `"`)
}
currentKey = ""
currentVal = ""
hasSeenEquals = false
case (char == ' ' || char == '\t') && !inQuotes && ((!hasSeenEquals && currentKey == "") || (hasSeenEquals && currentVal == "")):
// Skip leading whitespace before a key or after '='
default:
if !hasSeenEquals {
currentKey += string(char)
} else {
currentVal += string(char)
}
}
}
// Don't forget the last parameter
if currentKey != "" {
params[currentKey] = strings.Trim(strings.TrimSpace(currentVal), `"`)
}
return params
}
// digestParams is a map of digest authentication parameters
type digestParams map[string]string
func (p digestParams) get(key string) string {
if val, ok := p[key]; ok {
return val
}
return ""
}
func md5Sum(s string) string {
h := md5.Sum([]byte(s))
return hex.EncodeToString(h[:])
}
// SHA256 sum for future use (RFC 7616 supports SHA-256)
func sha256Sum(s string) string {
h := sha256.Sum256([]byte(s))
return hex.EncodeToString(h[:])
}
internal/auth/netrc.go
+103
View File
@@ -0,0 +1,103 @@
//go:build linux || freebsd
// +build linux freebsd
package auth
import (
"bufio"
"os"
"path/filepath"
"strings"
)
// NetrcEntry holds credentials for a single machine entry in .netrc.
type NetrcEntry struct {
Machine string
Login string
Password string
}
// LoadNetrc reads ~/.netrc and returns entries keyed by machine name.
func LoadNetrc() (map[string]*NetrcEntry, error) {
home, err := os.UserHomeDir()
if err != nil {
return nil, nil
}
path := filepath.Join(home, ".netrc")
file, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer file.Close()
entries := make(map[string]*NetrcEntry)
var current *NetrcEntry
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// Skip comments and empty lines
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
// "default" is also valid but we treat it as a machine
if len(fields) >= 2 && (fields[0] == "machine" || fields[0] == "default") {
if current != nil && current.Machine != "" {
entries[current.Machine] = current
}
name := fields[1]
if fields[0] == "default" {
name = "default"
}
current = &NetrcEntry{Machine: name}
// Rest of line may have login/password
parseNetrcLine(fields[2:], current)
} else if current != nil {
parseNetrcLine(fields, current)
}
}
if current != nil && current.Machine != "" {
entries[current.Machine] = current
}
return entries, scanner.Err()
}
func parseNetrcLine(fields []string, entry *NetrcEntry) {
for i := 0; i < len(fields); i++ {
switch fields[i] {
case "login":
if i+1 < len(fields) {
entry.Login = fields[i+1]
i++
}
case "password":
if i+1 < len(fields) {
entry.Password = fields[i+1]
i++
}
}
}
}
// LookupNetrc returns credentials from .netrc for a given hostname.
func LookupNetrc(host string) (login, password string, found bool) {
entries, err := LoadNetrc()
if err != nil || entries == nil {
return "", "", false
}
// Exact match first
if e, ok := entries[host]; ok {
return e.Login, e.Password, true
}
// Default entry
if e, ok := entries["default"]; ok {
return e.Login, e.Password, true
}
return "", "", false
}
internal/auth/oauth.go
+293
View File
@@ -0,0 +1,293 @@
//go:build linux || freebsd
// +build linux freebsd
package auth
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// OAuthConfig configuration for OAuth 2.0
type OAuthConfig struct {
ClientID string `json:"client_id,omitempty" toml:"client_id,omitempty"`
ClientSecret string `json:"client_secret,omitempty" toml:"client_secret,omitempty"`
TokenURL string `json:"token_url,omitempty" toml:"token_url,omitempty"`
AuthURL string `json:"auth_url,omitempty" toml:"auth_url,omitempty"`
RedirectURI string `json:"redirect_uri,omitempty" toml:"redirect_uri,omitempty"`
Scopes []string `json:"scopes,omitempty" toml:"scopes,omitempty"`
AccessToken string `json:"access_token,omitempty" toml:"access_token,omitempty"`
RefreshToken string `json:"refresh_token,omitempty" toml:"refresh_token,omitempty"`
TokenType string `json:"token_type,omitempty" toml:"token_type,omitempty"`
Expiry time.Time `json:"expiry,omitempty" toml:"expiry,omitempty"`
GrantType string `json:"grant_type,omitempty" toml:"grant_type,omitempty"` // authorization_code, client_credentials, password, refresh_token
}
// TokenResponse response from a token endpoint
type TokenResponse struct {
AccessToken string `json:"access_token"`
TokenType string `json:"token_type"`
RefreshToken string `json:"refresh_token,omitempty"`
ExpiresIn int `json:"expires_in,omitempty"`
Scope string `json:"scope,omitempty"`
Error string `json:"error,omitempty"`
ErrorDesc string `json:"error_description,omitempty"`
}
// OAuthClient client for OAuth 2.0 authentication
type OAuthClient struct {
config *OAuthConfig
client *http.Client
}
// NewOAuthClient creates a new OAuth client
func NewOAuthClient(cfg *OAuthConfig) *OAuthClient {
if cfg.TokenType == "" {
cfg.TokenType = "Bearer"
}
return &OAuthClient{
config: cfg,
client: &http.Client{Timeout: 30 * time.Second},
}
}
// IsTokenValid checks whether the token is valid
func (c *OAuthClient) IsTokenValid() bool {
if c.config.AccessToken == "" {
return false
}
if c.config.Expiry.IsZero() {
return true // No expiry, assume valid
}
// Token is valid if it expires in more than 30 seconds
return time.Now().Add(30 * time.Second).Before(c.config.Expiry)
}
// RequestToken gets new access token
func (c *OAuthClient) RequestToken(ctx context.Context) (*TokenResponse, error) {
if c.config.TokenURL != "" {
u, err := url.Parse(c.config.TokenURL)
if err != nil {
return nil, fmt.Errorf("invalid oauth token URL: %w", err)
}
if u.Scheme != "https" && u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1" && u.Hostname() != "::1" {
return nil, fmt.Errorf("oauth token URL must use HTTPS for security")
}
}
formData := url.Values{}
grantType := c.config.GrantType
if grantType == "" {
grantType = "client_credentials"
}
formData.Set("grant_type", grantType)
switch grantType {
case "client_credentials":
formData.Set("client_id", c.config.ClientID)
formData.Set("client_secret", c.config.ClientSecret)
if len(c.config.Scopes) > 0 {
formData.Set("scope", strings.Join(c.config.Scopes, " "))
}
case "password":
formData.Set("client_id", c.config.ClientID)
formData.Set("client_secret", c.config.ClientSecret)
// Username and password would need to be passed separately
// This is handled by caller
case "authorization_code":
formData.Set("client_id", c.config.ClientID)
formData.Set("client_secret", c.config.ClientSecret)
// Code and redirect_uri would need to be passed separately
case "refresh_token":
formData.Set("client_id", c.config.ClientID)
formData.Set("client_secret", c.config.ClientSecret)
formData.Set("refresh_token", c.config.RefreshToken)
}
req, err := http.NewRequestWithContext(ctx, "POST", c.config.TokenURL, strings.NewReader(formData.Encode()))
if err != nil {
return nil, fmt.Errorf("failed to create token request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("token request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read token response: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("token endpoint returned %s", formatOAuthError(resp.StatusCode, body))
}
var tokenResp TokenResponse
if err := json.Unmarshal(body, &tokenResp); err != nil {
return nil, fmt.Errorf("failed to parse token response: %w", err)
}
if tokenResp.Error != "" {
return nil, fmt.Errorf("oauth error: %s - %s", tokenResp.Error, tokenResp.ErrorDesc)
}
return &tokenResp, nil
}
// ApplyToken applies the token to the HTTP request
func (c *OAuthClient) ApplyToken(req *http.Request) {
if c.config.AccessToken == "" {
return
}
tokenType := c.config.TokenType
if tokenType == "" {
tokenType = "Bearer"
}
req.Header.Set("Authorization", fmt.Sprintf("%s %s", tokenType, c.config.AccessToken))
}
// RefreshToken refreshes access token using a refresh token
func (c *OAuthClient) RefreshToken(ctx context.Context) error {
if c.config.RefreshToken == "" {
return fmt.Errorf("no refresh token available")
}
oldGrantType := c.config.GrantType
c.config.GrantType = "refresh_token"
defer func() { c.config.GrantType = oldGrantType }()
tokenResp, err := c.RequestToken(ctx)
if err != nil {
return err
}
c.UpdateToken(tokenResp)
return nil
}
// UpdateToken updates internal token from response
func (c *OAuthClient) UpdateToken(tokenResp *TokenResponse) {
c.config.AccessToken = tokenResp.AccessToken
c.config.TokenType = tokenResp.TokenType
if tokenResp.RefreshToken != "" {
c.config.RefreshToken = tokenResp.RefreshToken
}
if tokenResp.ExpiresIn > 0 {
c.config.Expiry = time.Now().Add(time.Duration(tokenResp.ExpiresIn) * time.Second)
}
}
// GetAuthorizationURL returns the authorization URL (for authorization_code flow)
func (c *OAuthClient) GetAuthorizationURL(state string) string {
if c.config.AuthURL == "" {
return ""
}
params := url.Values{}
params.Set("client_id", c.config.ClientID)
params.Set("redirect_uri", c.config.RedirectURI)
params.Set("response_type", "code")
params.Set("state", state)
if len(c.config.Scopes) > 0 {
params.Set("scope", strings.Join(c.config.Scopes, " "))
}
baseURL := c.config.AuthURL
if strings.Contains(baseURL, "?") {
return baseURL + "&" + params.Encode()
}
return baseURL + "?" + params.Encode()
}
// ExchangeCode exchanges an authorization code for an access token
func (c *OAuthClient) ExchangeCode(ctx context.Context, code string) (*TokenResponse, error) {
formData := url.Values{}
formData.Set("grant_type", "authorization_code")
formData.Set("code", code)
formData.Set("redirect_uri", c.config.RedirectURI)
formData.Set("client_id", c.config.ClientID)
formData.Set("client_secret", c.config.ClientSecret)
req, err := http.NewRequestWithContext(ctx, "POST", c.config.TokenURL, strings.NewReader(formData.Encode()))
if err != nil {
return nil, fmt.Errorf("failed to create code exchange request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("code exchange request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read code exchange response: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("code exchange endpoint returned %s", formatOAuthError(resp.StatusCode, body))
}
var tokenResp TokenResponse
if err := json.Unmarshal(body, &tokenResp); err != nil {
return nil, fmt.Errorf("failed to parse code exchange response: %w", err)
}
if tokenResp.Error != "" {
return nil, fmt.Errorf("oauth error: %s - %s", tokenResp.Error, tokenResp.ErrorDesc)
}
return &tokenResp, nil
}
// formatOAuthError converts a non-2xx OAuth response into a human-readable
// error string while avoiding leaking sensitive data. The OAuth provider's
// raw response body is never returned verbatim: we first try to extract
// the structured `error` and `error_description` fields defined in RFC
// 6749 §5.2 (which is what well-behaved providers return), and only fall
// back to a 512-byte truncated snippet of the raw body if the body is
// not structured. This prevents upstream providers that echo back
// secrets, stack traces, or other sensitive payloads in their error
// bodies from leaking those payloads into our logs and error displays.
func formatOAuthError(statusCode int, body []byte) string {
if len(body) == 0 {
return fmt.Sprintf("status %d with empty body", statusCode)
}
var structured struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
if err := json.Unmarshal(body, &structured); err == nil && structured.Error != "" {
if structured.ErrorDescription != "" {
return fmt.Sprintf("status %d: %s - %s", statusCode, structured.Error, structured.ErrorDescription)
}
return fmt.Sprintf("status %d: %s", statusCode, structured.Error)
}
const maxSnippet = 512
snippet := string(body)
if len(snippet) > maxSnippet {
snippet = snippet[:maxSnippet] + "... (truncated)"
}
return fmt.Sprintf("status %d: %s", statusCode, snippet)
}
internal/cli/cli_test.go
+1154
View File
File diff suppressed because it is too large Load Diff
internal/cli/flags.go
+101
View File
@@ -0,0 +1,101 @@
//go:build linux || freebsd
// +build linux freebsd
package cli
import (
_ "embed"
"fmt"
"io"
"os"
"strings"
)
//go:embed help.txt
var helpText string
// ANSI color codes
const (
ColorReset = "\033[0m"
ColorGreen = "\033[32m"
ColorCyan = "\033[36m"
ColorYellow = "\033[33m"
ColorBlue = "\033[34m"
ColorRed = "\033[31m"
ColorBold = "\033[1m"
ColorMagenta = "\033[35m"
)
// isColorTerminal checks if writer supports ANSI colors
func isColorTerminal(w io.Writer) bool {
if os.Getenv("NO_COLOR") != "" {
return false
}
if _, ok := w.(*os.File); ok {
return os.Getenv("TERM") != "dumb"
}
return false
}
// PrintUsage prints formatted help from embedded help.txt
func PrintUsage(w io.Writer) {
useColors := isColorTerminal(w)
if useColors {
bold := ColorBold
reset := ColorReset
green := ColorGreen
yellow := ColorYellow
magenta := ColorMagenta
_ = bold
_ = reset
_ = green
_ = yellow
_ = magenta
// Print help text with ANSI colors for section headers
for _, line := range strings.Split(helpText, "\n") {
if strings.HasPrefix(line, "NAME") || strings.HasPrefix(line, "SYNOPSIS") ||
strings.HasPrefix(line, "DESCRIPTION") || strings.HasSuffix(strings.TrimRight(line, " "), ":") {
fmt.Fprintf(w, "%s%s%s\n", ColorBold, line, ColorReset)
} else {
fmt.Fprintln(w, line)
}
}
} else {
fmt.Fprint(w, helpText)
}
}
// PrintCategoryHelp prints help filtered by category.
// Categories: all, target, network, auth, mirror, recursive, verify, queue, metalink, upload, pgp, extraction, config
func PrintCategoryHelp(w io.Writer, category string) {
if category == "" || category == "all" {
PrintUsage(w)
return
}
categories := map[string]string{
"target": "TARGETS",
"network": "NETWORK",
"auth": "AUTHENTICATION",
"mirror": "MIRROR MODE",
"recursive": "RECURSIVE DOWNLOAD",
"verify": "VERIFICATION",
"queue": "DOWNLOAD QUEUE",
"metalink": "METALINK",
"upload": "BASIC OPTIONS",
"pgp": "PGP/GPG",
"extraction": "EXTRACTION",
"config": "CONFIGURATION",
}
title, ok := categories[strings.ToLower(category)]
if !ok {
fmt.Fprintf(w, "Unknown category: %s\nAvailable: %s\n", category,
"target, network, auth, mirror, recursive, verify, queue, metalink, upload, pgp, extraction, config, all")
return
}
fmt.Fprintf(w, "Showing category: %s\nUse --help for full help.\n\n", title)
PrintUsage(w) // For now, print full help — future: filter
}
internal/cli/help.go
+85
View File
@@ -0,0 +1,85 @@
//go:build linux || freebsd
// +build linux freebsd
package cli
import (
"fmt"
"io"
"os"
"strings"
)
// PrintError prints an error with color styling
func PrintError(w io.Writer, err error) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorRed + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%s✗ Error:%s %v\n", prefix, suffix, err)
}
// PrintWarning prints a warning with color styling
func PrintWarning(w io.Writer, msg string) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorYellow + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%s⚠ Warning:%s %s\n", prefix, suffix, msg)
}
// PrintSuccess prints a success message with color styling
func PrintSuccess(w io.Writer, msg string) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorGreen + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%s✓%s %s\n", prefix, suffix, msg)
}
// PrintInfo prints an info message with color styling
func PrintInfo(w io.Writer, msg string) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorCyan + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%sℹ%s %s\n", prefix, suffix, msg)
}
// FormatError formats an error with context
func FormatError(err error, context string) error {
if context == "" {
return err
}
return fmt.Errorf("%s: %w", context, err)
}
// IsTerminal checks if the writer is a terminal
func IsTerminal(w io.Writer) bool {
_, ok := w.(*os.File)
return ok && os.Getenv("TERM") != "dumb"
}
// Colorize wraps text in ANSI color codes if the writer supports it
func Colorize(w io.Writer, colorCode, text string) string {
if !isColorTerminal(w) {
return text
}
return colorCode + text + ColorReset
}
// Indent returns text indented by the specified number of spaces
func Indent(text string, spaces int) string {
indent := strings.Repeat(" ", spaces)
lines := strings.Split(text, "\n")
for i, line := range lines {
if line != "" {
lines[i] = indent + line
}
}
return strings.Join(lines, "\n")
}
internal/cli/help.txt
+161
View File
@@ -0,0 +1,161 @@
NAME
goget - Modern IPv6-First Downloader
SYNOPSIS
goget --url <URL> [OPTIONS]
DESCRIPTION
curl/wget replacement in Go. IPv6-first, parallel downloads, resume,
checksum verification, recursive mirroring, and 8 protocols.
TARGET
--url <URL> URL to download (repeatable)
OUTPUT
--output <FILE> Output file (default: auto from URL)
--restart Delete existing file and restart download
--overwrite Overwrite existing file (no atomic rename)
--content-disposition Use server-provided filename
--keep-timestamps Set file modification time from Last-Modified
PROGRESS
--verbose Verbose output with progress bar (default)
--no-progress Hide progress bar
--progress=dot Wget-style dot progress
--quiet Suppress all output except errors
--json Output progress as JSON-lines (for scripting)
--debug Debug mode with detailed info
NETWORK
--timeout <DURATION> Connection timeout (0 = auto)
--max-time <DURATION> Maximum total transfer time
--max-retries <N> Maximum retry attempts on failure
--max-filesize <SIZE> Abort if file exceeds size (e.g., 100MB)
--proxy <URL> Proxy server URL
--no-ipv4 Disable IPv4 fallback
--no-redirect Disable following HTTP redirects
--dns-servers <IPs> Custom DNS servers (comma-separated)
--bind-interface <IFACE> Bind to specific network interface
--allow-private Allow connections to private/internal IPs
--pinned-cert <SHA256> Certificate pinning (SHA-256 hash)
CURL-COMPATIBLE
--header "K: V" Custom HTTP header (repeatable)
--insecure Skip TLS certificate verification
--cert <FILE> Client certificate (PEM)
--key <FILE> Client private key (PEM)
--data "key=value" Send POST data
--form "field=value" Multipart form data (repeatable)
--form "field=@file" Upload file via multipart form
--spider Check URL existence only (HEAD request)
--write-out FORMAT Print metadata (curl format)
--trace-time Print HTTP timing breakdown
WGET-COMPATIBLE
--accept "*.pdf" Accept patterns (filename glob or MIME)
--no-parent Do not ascend to parent directory
--adjust-extension Append .html extension to HTML files
--wait <DURATION> Delay between requests (recursive)
--random-wait Randomize wait time (0.5x - 1.5x)
--page-requisites Download CSS/JS/images for pages
--span-hosts Follow links to external domains
--convert-links Convert links for local viewing
--warc-file <FILE> Write WARC output for archiving
--progress=dot Wget-style dot progress
RECURSIVE / MIRROR
--recursive Enable recursive downloading
--mirror Mirror entire website (infinite depth)
--max-depth <N> Maximum recursion depth
--follow-external Follow external domains
--exclude-pattern <P> Exclude URL patterns (comma-separated)
--no-convert-links Don't convert links in mirror mode
--no-mirror-assets Don't download CSS/JS/images in mirror
--no-robots Don't respect robots.txt in mirror
--rate-limit <RATE> Rate limit (e.g., 1MB/s)
DOWNLOAD MANAGEMENT
--queue Add to download queue
--queue-list List queue items
--queue-process Process download queue
--queue-clear Clear completed items from queue
--queue-file <PATH> Custom queue file path
--queue-priority <N> Priority (1-10)
--batch-file <FILE> Download URLs from file (one per line)
--schedule <TIME> Schedule download (datetime or cron)
--on-complete <CMD> Run command after successful download
AUTHENTICATION
--username <USER> Username for HTTP Basic/Digest Auth
--password <PASS> Password (prefer --password-file for security)
--password-file <FILE> Read password from file
--auth-type <TYPE> Auth type: basic, digest, auto
--cookie-jar <FILE> Load/save cookies (Netscape format)
--oauth-client-id <ID> OAuth 2.0 Client ID
--oauth-client-secret <S>OAuth 2.0 Client Secret
--oauth-token-url <URL> OAuth 2.0 Token URL
--oauth-access-token <T> OAuth 2.0 Access Token
--oauth-refresh-token <T>OAuth 2.0 Refresh Token
UPLOAD
--upload Upload file instead of downloading
--upload-method <M> PUT or POST (default: PUT)
--upload-file <FILE> File to upload
--form-data "k=v" Form data (comma-separated)
--file-field <NAME> Form field name (default: file)
VERIFICATION
--checksum <HASH> Expected checksum
--checksum-algo <ALG> Algorithm: sha256, sha512, blake2b, sha3-256, sha3-512
--checksum-file <FILE> Read checksum from SHA256SUMS file
PGP / GPG
--pgp-decrypt Decrypt PGP file after download
--pgp-verify Verify PGP signature
--pgp-sig <FILE> Path to signature file (.asc, .sig)
--pgp-key <FILE> Path to key file
--pgp-passphrase <PASS> Passphrase for private key
--pgp-passphrase-file F Read passphrase from file
METALINK
--metalink Treat URL as Metalink (multi-source)
--metalink-file <PATH> Path to local Metalink file
CONFIGURATION
--init-config Generate default config file
--config-get <KEY> Get config value
--config-set <K> <V> Set config value
--config-list List all config values
--config-unset <KEY> Unset config value
INFORMATION
--info <URL> Show file info without downloading
--benchmark <URL> Benchmark download speed
--generate-man-page Generate man page and exit
--completion <SHELL> Generate shell completion (bash, zsh, fish)
--help Show this help
--version Show version
PROTOCOLS
http:// https:// HTTP/1.1, HTTP/2, TLS 1.2/1.3
ftp:// ftps:// FTP (active/passive, explicit TLS)
file:// Local file copy + recursive directories
data: Inline data (base64 + plain text)
gopher:// RFC 1436 with type parsing
gemini:// Gemini protocol (TLS, redirect)
EXAMPLES
goget --url https://example.com/file.zip
goget --url https://a.com/1.zip --url https://b.com/2.zip
goget -H "Authorization: Bearer TOKEN" --url https://api.example.com
goget --form "name=value" --form "file=@./data.csv" --url https://httpbin.org/post
goget --recursive --accept "*.pdf" --url https://example.com/docs/
goget --mirror --convert-links --url https://example.com --output ./mirror
goget --schedule "0 2 * * *" --url https://example.com/daily.zip
goget --json --no-progress --url https://example.com/file.zip
FILES
~/.config/goget/config.json Configuration
<file>.goget.meta Resume metadata
More: https://codeberg.org/petrbalvin/goget
internal/cli/parser.go
+80
View File
@@ -0,0 +1,80 @@
//go:build linux || freebsd
// +build linux freebsd
package cli
import (
"flag"
"fmt"
"os"
"strings"
)
// Parser helps with argument parsing
type Parser struct {
fs *flag.FlagSet
}
// NewParser creates new parser
func NewParser(name string) *Parser {
return &Parser{
fs: flag.NewFlagSet(name, flag.ExitOnError),
}
}
// Parse parses arguments
func (p *Parser) Parse(args []string) error {
return p.fs.Parse(args)
}
// GetBool returns a bool flag
func (p *Parser) GetBool(name string) bool {
return p.fs.Lookup(name).Value.String() == "true"
}
// GetString returns a string flag
func (p *Parser) GetString(name string) string {
return p.fs.Lookup(name).Value.String()
}
// ParseArgs parses command-line arguments
func ParseArgs(args []string) (map[string]string, error) {
result := make(map[string]string)
for i := 0; i < len(args); i++ {
arg := args[i]
if strings.HasPrefix(arg, "--") {
key := strings.TrimPrefix(arg, "--")
if i+1 < len(args) && !strings.HasPrefix(args[i+1], "--") {
result[key] = args[i+1]
i++
} else {
result[key] = "true"
}
}
}
return result, nil
}
// ValidateRequiredFlags validates required flags
func ValidateRequiredFlags(flags map[string]string, required []string) error {
var missing []string
for _, req := range required {
if _, exists := flags[req]; !exists {
missing = append(missing, req)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing required flags: %s", strings.Join(missing, ", "))
}
return nil
}
// ExitWithError prints an error and exits the program
func ExitWithError(err error) {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
internal/cli/progress.go
+357
View File
@@ -0,0 +1,357 @@
//go:build linux || freebsd
// +build linux freebsd
package cli
import (
"fmt"
"io"
"os"
"strings"
"sync"
"time"
format "codeberg.org/petrbalvin/goget/internal/format"
)
// ProgressBarConfig configures the progress bar
type ProgressBarConfig struct {
Total int64
Width int
Colors bool
OnComplete func()
Writer io.Writer
}
// DefaultProgressBarConfig returns the default configuration
func DefaultProgressBarConfig() *ProgressBarConfig {
return &ProgressBarConfig{
Total: -1,
Width: 0,
Colors: true,
Writer: os.Stderr,
}
}
// ProgressBar represents a visual progress bar
type ProgressBar struct {
config *ProgressBarConfig
current int64
startTime time.Time
lastUpdate time.Time
barWidth int
mu sync.Mutex
speedHistory []float64
speedMax float64
}
// NewProgressBar creates a new progress bar
func NewProgressBar(cfg *ProgressBarConfig) *ProgressBar {
if cfg == nil {
cfg = DefaultProgressBarConfig()
}
pb := &ProgressBar{
config: cfg,
current: 0,
startTime: time.Now(),
lastUpdate: time.Now(),
barWidth: 40,
}
if cfg.Width == 0 {
tw := getTerminalWidth()
pb.barWidth = max(20, tw-45)
} else {
pb.barWidth = cfg.Width
}
return pb
}
// SetTotal sets the total value in a thread-safe way
func (pb *ProgressBar) SetTotal(total int64) {
pb.mu.Lock()
pb.config.Total = total
pb.mu.Unlock()
}
// Update updates progress
func (pb *ProgressBar) Update(current int64) {
pb.mu.Lock()
pb.current = current
pb.renderLocked()
pb.mu.Unlock()
}
// Add adds to the current progress
func (pb *ProgressBar) Add(n int64) {
pb.mu.Lock()
pb.current += n
pb.renderLocked()
pb.mu.Unlock()
}
// Finish completes the progress bar
func (pb *ProgressBar) Finish() {
pb.mu.Lock()
if pb.config.Total > 0 {
pb.current = pb.config.Total
}
pb.renderFinalLocked()
pb.mu.Unlock()
if pb.config.OnComplete != nil {
pb.config.OnComplete()
}
}
// Cancel cancels the progress bar with error state
func (pb *ProgressBar) Cancel() {
pb.mu.Lock()
pb.renderCancelledLocked()
pb.mu.Unlock()
}
// renderLocked renders the current state; must be called with pb.mu locked
func (pb *ProgressBar) renderLocked() {
now := time.Now()
if now.Sub(pb.lastUpdate) < 50*time.Millisecond {
return
}
pb.lastUpdate = now
w := pb.config.Writer
total := pb.config.Total
current := pb.current
elapsed := now.Sub(pb.startTime)
// Calculate speed
speed := float64(0)
if elapsed.Seconds() > 0 {
speed = float64(current) / elapsed.Seconds()
}
// Record speed history (keep last 15 samples)
pb.speedHistory = append(pb.speedHistory, speed)
if len(pb.speedHistory) > 15 {
pb.speedHistory = pb.speedHistory[len(pb.speedHistory)-15:]
}
// Update max speed
if speed > pb.speedMax {
pb.speedMax = speed
}
percent := float64(0)
showPercent := total > 0
if showPercent {
percent = float64(current) / float64(total) * 100
}
var percentStr string
if showPercent {
percentStr = fmt.Sprintf(" %5.1f%%", percent)
} else {
percentStr = " ??%"
}
filled := 0
empty := pb.barWidth
if showPercent {
filled = int(float64(pb.barWidth) * percent / 100)
empty = pb.barWidth - filled
}
barStart := ""
barEnd := ""
if pb.config.Colors {
barStart = ColorGreen
barEnd = ColorReset
}
bar := barStart + strings.Repeat("█", filled) + barEnd + strings.Repeat("░", empty)
currentStr := format.Bytes(current)
totalStr := "?"
if total > 0 {
totalStr = format.Bytes(total)
}
speedStr := format.Speed(speed)
etaStr := ""
if showPercent && current < total && speed > 0 {
remaining := float64(total-current) / speed
etaStr = formatDuration(time.Duration(remaining * float64(time.Second)))
}
etaPrefix := ""
etaSuffix := ""
if etaStr != "" && pb.config.Colors {
etaPrefix = ColorCyan
etaSuffix = ColorReset
}
// Build speed sparkline
sparkline := pb.renderSparkline()
var line strings.Builder
if pb.config.Colors {
line.WriteString(ColorBold)
}
line.WriteString("[")
line.WriteString(bar)
line.WriteString("]")
if pb.config.Colors {
line.WriteString(ColorReset)
}
line.WriteString(percentStr)
line.WriteString(fmt.Sprintf(" %s/%s", currentStr, totalStr))
line.WriteString(fmt.Sprintf(" %s", speedStr))
if sparkline != "" {
line.WriteString(fmt.Sprintf(" %s", sparkline))
}
if etaStr != "" {
line.WriteString(fmt.Sprintf(" %sETA: %s%s", etaPrefix, etaStr, etaSuffix))
}
fmt.Fprintf(w, "\r%s\033[K", line.String())
}
// renderFinalLocked renders the completed state; must be called with pb.mu locked
func (pb *ProgressBar) renderFinalLocked() {
w := pb.config.Writer
elapsed := time.Since(pb.startTime)
prefix := ""
suffix := ""
if pb.config.Colors {
prefix = ColorGreen + ColorBold
suffix = ColorReset
}
line := fmt.Sprintf("%s✓ Download complete%s | %s | %s elapsed",
prefix, suffix, format.Bytes(pb.current), formatDuration(elapsed))
fmt.Fprintf(w, "\r%s\n", line)
}
// renderCancelledLocked renders the cancelled state; must be called with pb.mu locked
func (pb *ProgressBar) renderCancelledLocked() {
w := pb.config.Writer
prefix := ""
suffix := ""
if pb.config.Colors {
prefix = ColorRed + ColorBold
suffix = ColorReset
}
line := fmt.Sprintf("%s✗ Download cancelled%s", prefix, suffix)
fmt.Fprintf(w, "\r%s\n", line)
}
// renderSparkline returns a mini ASCII sparkline of recent speed history.
// Uses Unicode block elements: ▁▂▃▄▅▆▇█
func (pb *ProgressBar) renderSparkline() string {
if len(pb.speedHistory) < 2 {
return ""
}
chars := []string{"▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"}
maxSpeed := pb.speedMax
if maxSpeed == 0 {
for _, s := range pb.speedHistory {
if s > maxSpeed {
maxSpeed = s
}
}
}
if maxSpeed == 0 {
return ""
}
// Take up to 10 most recent samples
history := pb.speedHistory
if len(history) > 10 {
history = history[len(history)-10:]
}
var sparkline strings.Builder
for _, speed := range history {
ratio := speed / maxSpeed
if ratio > 1.0 {
ratio = 1.0
}
idx := int(ratio * float64(len(chars)-1))
if idx < 0 {
idx = 0
}
if idx >= len(chars) {
idx = len(chars) - 1
}
sparkline.WriteString(chars[idx])
}
return sparkline.String()
}
// getTerminalWidth gets the terminal width
func getTerminalWidth() int {
if ws := os.Getenv("COLUMNS"); ws != "" {
if w := parseInt(ws); w > 0 {
return w
}
}
return 80
}
// parseInt helper
func parseInt(s string) int {
var n int
for _, c := range s {
if c >= '0' && c <= '9' {
n = n*10 + int(c-'0')
} else {
break
}
}
return n
}
// formatDuration formats a duration
func formatDuration(d time.Duration) string {
d = d.Round(time.Second)
h := d / time.Hour
d -= h * time.Hour
m := d / time.Minute
d -= m * time.Minute
s := d / time.Second
if h > 0 {
return fmt.Sprintf("%dh%dm", h, m)
}
if m > 0 {
return fmt.Sprintf("%dm%ds", m, s)
}
return fmt.Sprintf("%ds", s)
}
// PrintProgressInfo prints an info message with progress styling
func PrintProgressInfo(w io.Writer, msg string) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorCyan + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%s• %s%s\n", prefix, msg, suffix)
}
// PrintProgressSuccess prints a success message with progress styling
func PrintProgressSuccess(w io.Writer, msg string) {
prefix, suffix := "", ""
if isColorTerminal(w) {
prefix = ColorGreen + ColorBold
suffix = ColorReset
}
fmt.Fprintf(w, "%s✓ %s%s\n", prefix, msg, suffix)
}
internal/compression/bzip2.go
+35
View File
@@ -0,0 +1,35 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"compress/bzip2"
"fmt"
"io"
)
type bzip2Decompressor struct{ *BaseDecompressor }
func NewBzip2Decompressor() *bzip2Decompressor {
return &bzip2Decompressor{NewBaseDecompressor("bzip2", []string{"bzip2"}, []string{".bz2", ".tbz", ".tbz2"})}
}
// Reader returns a bzip2 decompression reader wrapped in a NopCloser.
// compress/bzip2.NewReader returns a plain io.Reader (no Close), so we
// wrap it to satisfy the io.ReadCloser contract of the Decompressor
// interface — the bzip2 reader has no internal state that needs
// releasing.
func (b *bzip2Decompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return io.NopCloser(bzip2.NewReader(r)), nil
}
type bzip2Compressor struct{ name string }
func NewBzip2Compressor() *bzip2Compressor { return &bzip2Compressor{name: "bzip2"} }
func (b *bzip2Compressor) Name() string { return b.name }
func (b *bzip2Compressor) Writer(w io.Writer) (io.WriteCloser, error) {
return nil, fmt.Errorf("bzip2 compression not supported in stdlib")
}
func init() { Register(NewBzip2Decompressor()) }
internal/compression/compression_test.go
+1081
View File
File diff suppressed because it is too large Load Diff
internal/compression/flate.go
+45
View File
@@ -0,0 +1,45 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"compress/flate"
"fmt"
"io"
)
type flateDecompressor struct{ *BaseDecompressor }
func NewFlateDecompressor() *flateDecompressor {
return &flateDecompressor{NewBaseDecompressor("flate", []string{}, []string{".deflate", ".fl"})}
}
// ✅ Oprava: flate.NewReader returns io.ReadCloser, ale my returnsme io.Reader
func (f *flateDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return flate.NewReader(r), nil
}
type flateCompressor struct {
name string
level int
}
func NewFlateCompressor() *flateCompressor {
return &flateCompressor{name: "flate", level: flate.DefaultCompression}
}
func NewFlateCompressorWithLevel(level int) (*flateCompressor, error) {
if level < flate.HuffmanOnly || level > flate.BestCompression {
return nil, fmt.Errorf("invalid compression level: %d (must be %d-%d)",
level, flate.HuffmanOnly, flate.BestCompression)
}
return &flateCompressor{name: "flate", level: level}, nil
}
func (f *flateCompressor) Name() string { return f.name }
func (f *flateCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return flate.NewWriter(w, f.level)
}
func init() { Register(NewFlateDecompressor()) }
internal/compression/gzip.go
+30
View File
@@ -0,0 +1,30 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"compress/gzip"
"io"
)
type gzipDecompressor struct{ *BaseDecompressor }
func NewGzipDecompressor() *gzipDecompressor {
return &gzipDecompressor{NewBaseDecompressor("gzip", []string{"gzip"}, []string{".gz"})}
}
// ✅ Oprava: returns (io.Reader, error)
func (g *gzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return gzip.NewReader(r)
}
type gzipCompressor struct{ name string }
func NewGzipCompressor() *gzipCompressor { return &gzipCompressor{name: "gzip"} }
func (g *gzipCompressor) Name() string { return g.name }
func (g *gzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return gzip.NewWriter(w), nil
}
func init() { Register(NewGzipDecompressor()) }
internal/compression/interface.go
+150
View File
@@ -0,0 +1,150 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"io"
"strings"
"sync"
)
// Decompressor defines the interface for decompression
type Decompressor interface {
Name() string
CanHandle(contentEncoding string) bool
CanHandleFile(filename string) bool
// Reader returns an io.ReadCloser (not just an io.Reader) so
// callers can release the decompressor's internal state — decoder
// tables, dictionaries, etc. — when finished. Forgetting to call
// Close on a long-lived mirror scan would let decoder state
// accumulate without bound.
Reader(r io.Reader) (io.ReadCloser, error)
}
// Compressor defines the interface for compression
type Compressor interface {
Name() string
Writer(w io.Writer) (io.WriteCloser, error)
}
// BaseDecompressor is the base implementation
type BaseDecompressor struct {
name string
contentEncodings []string
fileExtensions []string
}
func NewBaseDecompressor(name string, contentEncodings, fileExtensions []string) *BaseDecompressor {
return &BaseDecompressor{
name: name,
contentEncodings: contentEncodings,
fileExtensions: fileExtensions,
}
}
func (bd *BaseDecompressor) Name() string { return bd.name }
func (bd *BaseDecompressor) Extensions() []string { return bd.fileExtensions }
func (bd *BaseDecompressor) CanHandle(contentEncoding string) bool {
for _, e := range bd.contentEncodings {
if strings.EqualFold(e, contentEncoding) {
return true
}
}
return false
}
func (bd *BaseDecompressor) CanHandleFile(filename string) bool {
filename = strings.ToLower(filename)
for _, e := range bd.fileExtensions {
if strings.HasSuffix(filename, e) {
return true
}
}
return false
}
// Registry for decompressors
type registry struct {
mu sync.RWMutex
items map[string]Decompressor
}
var globalRegistry = &registry{items: make(map[string]Decompressor)}
func Register(d Decompressor) {
globalRegistry.mu.Lock()
defer globalRegistry.mu.Unlock()
globalRegistry.items[strings.ToLower(d.Name())] = d
}
func Get(name string) (Decompressor, bool) {
globalRegistry.mu.RLock()
defer globalRegistry.mu.RUnlock()
d, ok := globalRegistry.items[strings.ToLower(name)]
return d, ok
}
func GetByFilename(filename string) (Decompressor, bool) {
format := DetectCompressionByFilename(filename)
if format == "" || format == "identity" {
return nil, false
}
return Get(format)
}
// GetCompressionFromHeader detects compression from the Content-Encoding header
func GetCompressionFromHeader(contentEncoding string) string {
if contentEncoding == "" {
return "identity"
}
encodings := ParseContentEncoding(contentEncoding)
return GetEffectiveEncoding(encodings)
}
func ParseContentEncoding(header string) []string {
if header == "" {
return []string{"identity"}
}
parts := strings.Split(header, ",")
result := make([]string, 0, len(parts))
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
result = append(result, strings.ToLower(p))
}
}
if len(result) == 0 {
return []string{"identity"}
}
return result
}
func GetEffectiveEncoding(encodings []string) string {
for _, e := range encodings {
if e != "identity" {
return e
}
}
return "identity"
}
func DetectCompressionByFilename(filename string) string {
filename = strings.ToLower(filename)
switch {
case strings.HasSuffix(filename, ".gz"):
return "gzip"
case strings.HasSuffix(filename, ".zlib"), strings.HasSuffix(filename, ".zz"):
return "zlib"
case strings.HasSuffix(filename, ".deflate"), strings.HasSuffix(filename, ".fl"):
return "flate"
case strings.HasSuffix(filename, ".bz2"), strings.HasSuffix(filename, ".tbz"), strings.HasSuffix(filename, ".tbz2"):
return "bzip2"
case strings.HasSuffix(filename, ".z"), strings.HasSuffix(filename, ".lzw"):
return "lzw"
}
return "identity"
}
func IsCompressedFile(filename string) bool {
return DetectCompressionByFilename(filename) != "identity"
}
internal/compression/lzw.go
+52
View File
@@ -0,0 +1,52 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"compress/lzw"
"io"
)
const lzwLitWidth = 8
type lzwDecompressor struct {
*BaseDecompressor
order lzw.Order
}
func NewLzwDecompressor() *lzwDecompressor { return NewLzwDecompressorWithOrder(lzw.LSB) }
func NewLzwDecompressorWithOrder(order lzw.Order) *lzwDecompressor {
name := "lzw"
if order == lzw.MSB {
name = "lzw-msb"
}
return &lzwDecompressor{NewBaseDecompressor(name, []string{}, []string{".z", ".lzw"}), order}
}
func (l *lzwDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return lzw.NewReader(r, l.order, lzwLitWidth), nil
}
type lzwCompressor struct {
name string
order lzw.Order
}
func NewLzwCompressor() *lzwCompressor { return NewLzwCompressorWithOrder(lzw.LSB) }
func NewLzwCompressorWithOrder(order lzw.Order) *lzwCompressor {
name := "lzw"
if order == lzw.MSB {
name = "lzw-msb"
}
return &lzwCompressor{name: name, order: order}
}
func (l *lzwCompressor) Name() string { return l.name }
func (l *lzwCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return lzw.NewWriter(w, l.order, lzwLitWidth), nil
}
func init() { Register(NewLzwDecompressor()) }
internal/compression/zlib.go
+30
View File
@@ -0,0 +1,30 @@
//go:build linux || freebsd
// +build linux freebsd
package compression
import (
"compress/zlib"
"io"
)
type zlibDecompressor struct{ *BaseDecompressor }
func NewZlibDecompressor() *zlibDecompressor {
return &zlibDecompressor{NewBaseDecompressor("zlib", []string{"deflate"}, []string{".zlib", ".zz"})}
}
// ✅ Oprava: returns (io.Reader, error)
func (z *zlibDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return zlib.NewReader(r)
}
type zlibCompressor struct{ name string }
func NewZlibCompressor() *zlibCompressor { return &zlibCompressor{name: "zlib"} }
func (z *zlibCompressor) Name() string { return z.name }
func (z *zlibCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return zlib.NewWriter(w), nil
}
func init() { Register(NewZlibDecompressor()) }
internal/config/config.go
+356
View File
@@ -0,0 +1,356 @@
//go:build linux || freebsd
// +build linux freebsd
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/auth"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/interpres"
)
// AuthConfig holds authentication configuration
type AuthConfig struct {
Username string `json:"username,omitempty" toml:"username,omitempty"`
Password string `json:"password,omitempty" toml:"password,omitempty"`
PasswordFile string `json:"password_file,omitempty" toml:"password_file,omitempty"`
AuthType string `json:"auth_type,omitempty" toml:"auth_type,omitempty"` // basic, digest, auto
OAuth auth.OAuthConfig `json:"oauth,omitempty" toml:"oauth,omitempty"`
}
// RecursiveConfig holds recursive download configuration
type RecursiveConfig struct {
Enabled bool `json:"enabled,omitempty" toml:"enabled,omitempty"`
MaxDepth int `json:"max_depth,omitempty" toml:"max_depth,omitempty"`
FollowExternal bool `json:"follow_external,omitempty" toml:"follow_external,omitempty"`
ExcludePatterns []string `json:"exclude_patterns,omitempty" toml:"exclude_patterns,omitempty"`
IncludePatterns []string `json:"include_patterns,omitempty" toml:"include_patterns,omitempty"`
Delay string `json:"delay,omitempty" toml:"delay,omitempty"`
Parallel int `json:"parallel,omitempty" toml:"parallel,omitempty"`
}
// ExtractConfig holds archive extraction configuration
type ExtractConfig struct {
Enabled bool `json:"enabled,omitempty" toml:"enabled,omitempty"`
OutputDir string `json:"output_dir,omitempty" toml:"output_dir,omitempty"`
StripComponents int `json:"strip_components,omitempty" toml:"strip_components,omitempty"`
ExcludePatterns []string `json:"exclude_patterns,omitempty" toml:"exclude_patterns,omitempty"`
PreservePermissions bool `json:"preserve_permissions,omitempty" toml:"preserve_permissions,omitempty"`
AutoDetect bool `json:"auto_detect,omitempty" toml:"auto_detect,omitempty"`
}
// Config represents the application configuration
type Config struct {
Timeout time.Duration `json:"timeout,omitempty" toml:"timeout,omitempty"`
Parallel int `json:"parallel,omitempty" toml:"parallel,omitempty"`
OutputDir string `json:"output_dir,omitempty" toml:"output_dir,omitempty"`
Verbose bool `json:"verbose,omitempty" toml:"verbose,omitempty"`
Debug bool `json:"debug,omitempty" toml:"debug,omitempty"`
IPv4Fallback bool `json:"ipv4_fallback,omitempty" toml:"ipv4_fallback,omitempty"`
Proxy string `json:"proxy,omitempty" toml:"proxy,omitempty"`
UserAgent string `json:"user_agent,omitempty" toml:"user_agent,omitempty"`
AutoDecompress bool `json:"auto_decompress,omitempty" toml:"auto_decompress,omitempty"`
InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty" toml:"insecure_skip_verify,omitempty"`
ChecksumAlgo string `json:"checksum_algo,omitempty" toml:"checksum_algo,omitempty"`
Auth AuthConfig `json:"auth,omitempty" toml:"auth,omitempty"`
MaxSpeed int64 `json:"max_speed,omitempty" toml:"max_speed,omitempty"`
AutoResume bool `json:"auto_resume,omitempty" toml:"auto_resume,omitempty"`
KeepMetadata bool `json:"keep_metadata,omitempty" toml:"keep_metadata,omitempty"`
Color string `json:"color,omitempty" toml:"color,omitempty"`
ProgressStyle string `json:"progress_style,omitempty" toml:"progress_style,omitempty"`
ShowETA bool `json:"show_eta,omitempty" toml:"show_eta,omitempty"`
Recursive RecursiveConfig `json:"recursive,omitempty" toml:"recursive,omitempty"`
Extract ExtractConfig `json:"extract,omitempty" toml:"extract,omitempty"`
CookieJar string `json:"cookie_jar,omitempty" toml:"cookie_jar,omitempty"`
DNSServers []string `json:"dns_servers,omitempty" toml:"dns_servers,omitempty"`
MaxRetries int `json:"max_retries,omitempty" toml:"max_retries,omitempty"`
configPath string
}
// DefaultConfig returns the default configuration
func DefaultConfig() *Config {
return &Config{
Timeout: 30 * time.Minute,
Parallel: 0,
OutputDir: ".",
Verbose: true,
Debug: false,
IPv4Fallback: true,
Proxy: "",
UserAgent: core.Name + "/" + core.Version,
AutoDecompress: true,
InsecureSkipVerify: false,
ChecksumAlgo: "sha256",
Auth: AuthConfig{},
MaxSpeed: 0,
AutoResume: true,
KeepMetadata: false,
Color: "auto",
ProgressStyle: "unicode",
ShowETA: true,
Recursive: RecursiveConfig{
Enabled: false,
MaxDepth: 3,
FollowExternal: false,
ExcludePatterns: []string{},
IncludePatterns: []string{},
Delay: "100ms",
},
Extract: ExtractConfig{
Enabled: false,
OutputDir: "",
StripComponents: 0,
ExcludePatterns: []string{},
PreservePermissions: true,
AutoDetect: true,
},
CookieJar: "",
}
}
// Load loads the configuration from file
func Load(customPath string) (*Config, error) {
cfg := DefaultConfig()
configPath := customPath
if configPath == "" {
configPath = getConfigPath()
}
if configPath == "" {
return cfg, nil
}
cfg.configPath = configPath
data, err := os.ReadFile(configPath)
if err != nil {
if os.IsNotExist(err) {
return cfg, nil
}
return nil, fmt.Errorf("failed to read config: %w", err)
}
if err := interpres.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("failed to parse config %s: %w", configPath, err)
}
if cfg.Auth.PasswordFile != "" && cfg.Auth.Password == "" {
passwordPath := expandTilde(cfg.Auth.PasswordFile)
if data, err := os.ReadFile(passwordPath); err == nil {
cfg.Auth.Password = strings.TrimSpace(string(data))
}
}
return cfg, nil
}
// Save writes the configuration to disk
func (c *Config) Save() error {
if c.configPath == "" {
c.configPath = getConfigPath()
if c.configPath == "" {
return fmt.Errorf("cannot determine config path")
}
}
if err := os.MkdirAll(filepath.Dir(c.configPath), 0755); err != nil {
return err
}
cfgCopy := *c
cfgCopy.Auth.Password = ""
cfgCopy.Auth.OAuth.ClientSecret = ""
cfgCopy.Auth.OAuth.AccessToken = ""
cfgCopy.Auth.OAuth.RefreshToken = ""
data, err := interpres.Marshal(cfgCopy)
if err != nil {
return err
}
return os.WriteFile(c.configPath, data, 0600)
}
// Path returns the config file path
func (c *Config) Path() string {
return c.configPath
}
// CLIFlags holds CLI overrides to merge into the configuration.
type CLIFlags struct {
Timeout time.Duration
Parallel int
Verbose bool
Debug bool
MaxSpeed int64
Proxy string
NoIPv4 bool
NoDecompress bool
Username string
Password string
CookieJar string
Recursive bool
MaxDepth int
FollowExternal bool
ExcludePatterns []string
Extract bool
ExtractDir string
StripComponents int
}
// Merge merges CLI overrides into the configuration.
func (c *Config) Merge(cli *CLIFlags) {
if cli.Timeout > 0 {
c.Timeout = cli.Timeout
}
if cli.Parallel >= 0 {
c.Parallel = cli.Parallel
}
if cli.Verbose {
c.Verbose = true
}
if cli.Debug {
c.Debug = true
}
if cli.MaxSpeed >= 0 {
c.MaxSpeed = cli.MaxSpeed
}
if cli.Proxy != "" {
c.Proxy = cli.Proxy
}
if cli.NoIPv4 {
c.IPv4Fallback = false
}
if cli.NoDecompress {
c.AutoDecompress = false
}
if cli.Username != "" {
c.Auth.Username = cli.Username
c.Auth.Password = cli.Password
}
if cli.CookieJar != "" {
c.CookieJar = cli.CookieJar
}
if cli.Recursive {
c.Recursive.Enabled = true
}
if cli.MaxDepth > 0 {
c.Recursive.MaxDepth = cli.MaxDepth
}
if cli.FollowExternal {
c.Recursive.FollowExternal = true
}
if len(cli.ExcludePatterns) > 0 {
c.Recursive.ExcludePatterns = cli.ExcludePatterns
}
if cli.Extract {
c.Extract.Enabled = true
}
if cli.ExtractDir != "" {
c.Extract.OutputDir = cli.ExtractDir
}
if cli.StripComponents >= 0 {
c.Extract.StripComponents = cli.StripComponents
}
}
func getConfigPath() string {
if path := os.Getenv("GOGET_CONFIG"); path != "" {
return path
}
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
return filepath.Join(xdg, "goget", "config.toml")
}
if home, err := os.UserHomeDir(); err == nil {
return filepath.Join(home, ".config", "goget", "config.toml")
}
return ""
}
func expandTilde(path string) string {
if strings.HasPrefix(path, "~/") {
if home, err := os.UserHomeDir(); err == nil {
return filepath.Join(home, path[2:])
}
}
return path
}
func (c *Config) GetEffectiveTimeout(fileSize int64, userOverride time.Duration) time.Duration {
tc := core.DefaultTimeoutConfig()
return tc.CalculateTimeout(fileSize, userOverride)
}
func (c *Config) GetParallelConnections(fileSize int64) int {
if c.Parallel > 0 {
return c.Parallel
}
if fileSize >= 100*1024*1024 {
return 4
}
return 1
}
func (c *Config) ShouldUseColors() bool {
switch c.Color {
case "always":
return true
case "never":
return false
default:
if os.Getenv("NO_COLOR") != "" {
return false
}
return true
}
}
func (c *Config) IsExcluded(url string) bool {
for _, pattern := range c.Recursive.ExcludePatterns {
if strings.Contains(url, pattern) {
return true
}
}
return false
}
func (c *Config) GetRecursiveDelay() time.Duration {
if c.Recursive.Delay == "" {
return 100 * time.Millisecond
}
d, err := time.ParseDuration(c.Recursive.Delay)
if err != nil {
return 100 * time.Millisecond
}
return d
}
func (c *Config) GetExtractOutputDir(defaultDir string) string {
if c.Extract.OutputDir != "" {
return c.Extract.OutputDir
}
return defaultDir
}
func ParseSpeed(s string) int64 {
if s == "" || s == "0" {
return 0
}
s = strings.TrimSpace(strings.ToLower(s))
var multiplier int64 = 1
switch {
case strings.HasSuffix(s, "tb/s"):
multiplier = 1000 * 1000 * 1000 * 1000
s = strings.TrimSuffix(s, "tb/s")
case strings.HasSuffix(s, "gb/s"):
multiplier = 1000 * 1000 * 1000
s = strings.TrimSuffix(s, "gb/s")
case strings.HasSuffix(s, "mb/s"):
multiplier = 1000 * 1000
s = strings.TrimSuffix(s, "mb/s")
case strings.HasSuffix(s, "kb/s"):
multiplier = 1000
s = strings.TrimSuffix(s, "kb/s")
case strings.HasSuffix(s, "/s"):
s = strings.TrimSuffix(s, "/s")
}
var value float64
fmt.Sscanf(s, "%f", &value)
return int64(value * float64(multiplier))
}
internal/config/config_test.go
+338
View File
@@ -0,0 +1,338 @@
//go:build linux || freebsd
// +build linux freebsd
package config
import (
"strings"
"testing"
"time"
"codeberg.org/petrbalvin/interpres"
)
func TestDefaultConfig(t *testing.T) {
cfg := DefaultConfig()
if cfg.Timeout != 30*time.Minute {
t.Errorf("Expected default timeout 30m, got %v", cfg.Timeout)
}
if cfg.Parallel != 0 {
t.Errorf("Expected default parallel 0, got %d", cfg.Parallel)
}
if cfg.Verbose != true {
t.Errorf("Expected default verbose true, got %v", cfg.Verbose)
}
if cfg.AutoDecompress != true {
t.Errorf("Expected default auto decompress true, got %v", cfg.AutoDecompress)
}
if cfg.ChecksumAlgo != "sha256" {
t.Errorf("Expected default checksum algo sha256, got %s", cfg.ChecksumAlgo)
}
}
func TestConfigMerge(t *testing.T) {
cfg := DefaultConfig()
cfg.Merge(&CLIFlags{
Timeout: 1 * time.Hour,
Parallel: 4,
Verbose: true,
Debug: false,
MaxSpeed: 1024 * 1024,
Proxy: "http://proxy",
NoIPv4: false,
NoDecompress: false,
Username: "user",
Password: "pass",
CookieJar: "",
Recursive: false,
MaxDepth: 0,
FollowExternal: false,
ExcludePatterns: nil,
Extract: false,
ExtractDir: "",
StripComponents: 0,
})
if cfg.Timeout != 1*time.Hour {
t.Errorf("Expected timeout 1h, got %v", cfg.Timeout)
}
if cfg.Parallel != 4 {
t.Errorf("Expected parallel 4, got %d", cfg.Parallel)
}
if cfg.Proxy != "http://proxy" {
t.Errorf("Expected proxy http://proxy, got %s", cfg.Proxy)
}
if cfg.Auth.Username != "user" {
t.Errorf("Expected username user, got %s", cfg.Auth.Username)
}
}
func TestGetEffectiveTimeout(t *testing.T) {
cfg := DefaultConfig()
// User override
timeout := cfg.GetEffectiveTimeout(1000000, 1*time.Hour)
if timeout != 1*time.Hour {
t.Errorf("Expected 1h with user override, got %v", timeout)
}
// Auto calculation for known size
timeout = cfg.GetEffectiveTimeout(100*1024*1024, 0)
if timeout < 30*time.Second {
t.Errorf("Expected at least 30s for auto timeout, got %v", timeout)
}
// Unknown size
timeout = cfg.GetEffectiveTimeout(-1, 0)
if timeout != 30*time.Minute {
t.Errorf("Expected 30m for unknown size, got %v", timeout)
}
}
func TestGetParallelConnections(t *testing.T) {
cfg := DefaultConfig()
// Explicit parallel
cfg.Parallel = 8
if cfg.GetParallelConnections(1000) != 8 {
t.Errorf("Expected 8 explicit connections, got %d", cfg.GetParallelConnections(1000))
}
// Auto for large file
cfg.Parallel = 0
if cfg.GetParallelConnections(200*1024*1024) != 4 {
t.Errorf("Expected 4 auto connections for large file, got %d", cfg.GetParallelConnections(200*1024*1024))
}
// Auto for small file
if cfg.GetParallelConnections(1024) != 1 {
t.Errorf("Expected 1 connection for small file, got %d", cfg.GetParallelConnections(1024))
}
}
func TestShouldUseColors(t *testing.T) {
cfg := DefaultConfig()
// Always
cfg.Color = "always"
if !cfg.ShouldUseColors() {
t.Error("Expected colors with always setting")
}
// Never
cfg.Color = "never"
if cfg.ShouldUseColors() {
t.Error("Expected no colors with never setting")
}
// Auto (default)
cfg.Color = "auto"
// Can't test environment in unit tests, but verify it doesn't panic
_ = cfg.ShouldUseColors()
}
func TestIsExcluded(t *testing.T) {
cfg := DefaultConfig()
cfg.Recursive.ExcludePatterns = []string{"example.com", "test.org"}
if !cfg.IsExcluded("https://example.com/file.zip") {
t.Error("Expected example.com to be excluded")
}
if !cfg.IsExcluded("https://test.org/file.zip") {
t.Error("Expected test.org to be excluded")
}
if cfg.IsExcluded("https://other.com/file.zip") {
t.Error("Expected other.com to not be excluded")
}
}
func TestGetRecursiveDelay(t *testing.T) {
cfg := DefaultConfig()
delay := cfg.GetRecursiveDelay()
if delay != 100*time.Millisecond {
t.Errorf("Expected default delay 100ms, got %v", delay)
}
cfg.Recursive.Delay = "500ms"
delay = cfg.GetRecursiveDelay()
if delay != 500*time.Millisecond {
t.Errorf("Expected delay 500ms, got %v", delay)
}
// Invalid delay should fall back to default
cfg.Recursive.Delay = "invalid"
delay = cfg.GetRecursiveDelay()
if delay != 100*time.Millisecond {
t.Errorf("Expected default delay for invalid value, got %v", delay)
}
}
func TestGetExtractOutputDir(t *testing.T) {
cfg := DefaultConfig()
// Empty config should return default
dir := cfg.GetExtractOutputDir("/default")
if dir != "/default" {
t.Errorf("Expected /default, got %s", dir)
}
// Config with output dir
cfg.Extract.OutputDir = "/custom"
dir = cfg.GetExtractOutputDir("/default")
if dir != "/custom" {
t.Errorf("Expected /custom, got %s", dir)
}
}
func TestParseSpeed(t *testing.T) {
tests := []struct {
input string
expected int64
}{
{"", 0},
{"0", 0},
{"100", 100},
{"100B/s", 100},
{"1KB/s", 1000},
{"1MB/s", 1000000},
{"1GB/s", 1000000000},
{"10MB/s", 10000000},
{"100 KB/s", 100000},
}
for _, test := range tests {
result := ParseSpeed(test.input)
if result != test.expected {
t.Errorf("ParseSpeed(%s) = %d, expected %d", test.input, result, test.expected)
}
}
}
func TestExpandTilde(t *testing.T) {
// Test with tilde
path := expandTilde("~/test/file.txt")
if path == "~/test/file.txt" {
t.Error("Expected tilde to be expanded")
}
// Test without tilde
path = expandTilde("/absolute/path/file.txt")
if path != "/absolute/path/file.txt" {
t.Errorf("Expected unchanged path, got %s", path)
}
}
func TestConfigSave(t *testing.T) {
cfg := DefaultConfig()
cfg.Verbose = true
cfg.Parallel = 4
// Save should not fail (may create directories)
err := cfg.Save()
// We can't test the actual file creation in unit tests without a temp dir
// but we verify the method doesn't panic
_ = err
}
func TestLoadNonExistentConfig(t *testing.T) {
cfg, err := Load("/nonexistent/path/config.toml")
if err != nil {
t.Fatalf("Expected no error for non-existent config, got %v", err)
}
if cfg == nil {
t.Fatal("Expected default config for non-existent file")
}
}
func TestMarshalUnmarshalTOML(t *testing.T) {
cfg := DefaultConfig()
cfg.Parallel = 4
cfg.Verbose = true
cfg.Timeout = 5 * time.Minute
cfg.Auth.Username = "testuser"
cfg.Auth.AuthType = "basic"
cfg.Recursive.Enabled = true
cfg.Recursive.MaxDepth = 5
cfg.DNSServers = []string{"1.1.1.1", "8.8.8.8"}
data, err := interpres.Marshal(*cfg)
if err != nil {
t.Fatalf("interpres.Marshal failed: %v", err)
}
tomlStr := string(data)
// Verify key elements are present
if !strings.Contains(tomlStr, "parallel = 4") {
t.Errorf("expected 'parallel = 4' in TOML output, got:\n%s", tomlStr)
}
if !strings.Contains(tomlStr, "verbose = true") {
t.Errorf("expected 'verbose = true' in TOML output, got:\n%s", tomlStr)
}
if !strings.Contains(tomlStr, "[auth]") {
t.Errorf("expected '[auth]' section in TOML output, got:\n%s", tomlStr)
}
if !strings.Contains(tomlStr, "username = \"testuser\"") {
t.Errorf("expected 'username = \"testuser\"' in TOML output, got:\n%s", tomlStr)
}
if !strings.Contains(tomlStr, "[recursive]") {
t.Errorf("expected '[recursive]' section in TOML output, got:\n%s", tomlStr)
}
if !strings.Contains(tomlStr, "dns_servers = [") {
t.Errorf("expected 'dns_servers' array in TOML output, got:\n%s", tomlStr)
}
// Roundtrip: unmarshal back and verify
var cfg2 Config
if err := interpres.Unmarshal(data, &cfg2); err != nil {
t.Fatalf("interpres.Unmarshal failed: %v", err)
}
if cfg2.Parallel != 4 {
t.Errorf("roundtrip: expected Parallel 4, got %d", cfg2.Parallel)
}
if cfg2.Verbose != true {
t.Errorf("roundtrip: expected Verbose true, got %v", cfg2.Verbose)
}
if cfg2.Auth.Username != "testuser" {
t.Errorf("roundtrip: expected Auth.Username testuser, got %s", cfg2.Auth.Username)
}
if cfg2.Recursive.MaxDepth != 5 {
t.Errorf("roundtrip: expected Recursive.MaxDepth 5, got %d", cfg2.Recursive.MaxDepth)
}
if len(cfg2.DNSServers) != 2 || cfg2.DNSServers[0] != "1.1.1.1" {
t.Errorf("roundtrip: expected DNSServers [1.1.1.1, 8.8.8.8], got %v", cfg2.DNSServers)
}
}
func TestMarshalTOMLDefaultConfig(t *testing.T) {
cfg := DefaultConfig()
data, err := interpres.Marshal(*cfg)
if err != nil {
t.Fatalf("interpres.Marshal with default config failed: %v", err)
}
if len(data) == 0 {
t.Error("expected non-empty TOML output for default config")
}
// Verify it can be parsed back
var cfg2 Config
if err := interpres.Unmarshal(data, &cfg2); err != nil {
t.Fatalf("interpres.Unmarshal of default output failed: %v", err)
}
if cfg2.OutputDir != "." {
t.Errorf("expected OutputDir '.', got %s", cfg2.OutputDir)
}
if cfg2.ChecksumAlgo != "sha256" {
t.Errorf("expected ChecksumAlgo sha256, got %s", cfg2.ChecksumAlgo)
}
}
func TestConfigPath(t *testing.T) {
cfg := DefaultConfig()
path := cfg.Path()
// Path should be empty when loaded with DefaultConfig
_ = path
}
internal/cookie/jar.go
+201
View File
@@ -0,0 +1,201 @@
//go:build linux || freebsd
// +build linux freebsd
package cookie
import (
"bufio"
"fmt"
"net/http"
"net/http/cookiejar"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"golang.org/x/net/publicsuffix"
)
// Jar wraps http.CookieJar with Netscape format import/export.
type Jar struct {
*cookiejar.Jar
mu sync.Mutex
cookies map[string][]*http.Cookie // domain -> cookies (for export)
}
// NewJar creates a new cookie jar with public suffix support.
func NewJar() (*Jar, error) {
jar, err := cookiejar.New(&cookiejar.Options{
PublicSuffixList: publicsuffix.List,
})
if err != nil {
return nil, err
}
return &Jar{
Jar: jar,
cookies: make(map[string][]*http.Cookie),
}, nil
}
// SetCookies stores cookies in the jar and tracks them for export.
func (j *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) {
j.Jar.SetCookies(u, cookies)
j.mu.Lock()
defer j.mu.Unlock()
for _, c := range cookies {
// Key the export map by the cookie's effective domain, not the
// URL hostname. A cookie set for "example.com" via a request to
// "www.example.com" must round-trip through SaveToFile/LoadFromFile
// keyed by "example.com", otherwise the second SetCookies call
// (from LoadFromFile) lands under a different map key and the
// cookies appear to be silently lost between sessions.
key := cookieStorageKey(c, u)
j.cookies[key] = append(j.cookies[key], c)
}
}
// cookieStorageKey returns the canonical (no leading dot) domain used as
// the storage key for the export map. It prefers the cookie's own Domain
// attribute (which reflects the cookie's actual scope) and falls back to
// the URL hostname for host-only cookies (those without a Domain attr).
func cookieStorageKey(c *http.Cookie, u *url.URL) string {
if c.Domain != "" {
return strings.TrimPrefix(c.Domain, ".")
}
return u.Hostname()
}
// LoadFromFile loads cookies from a file in Netscape format.
func (j *Jar) LoadFromFile(path string) error {
if path == "" {
return nil
}
file, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to open cookie jar: %w", err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Split(line, "\t")
if len(fields) < 7 {
continue
}
domain := fields[0]
path := fields[2]
expiresStr := fields[4]
name := fields[5]
value := fields[6]
expires, err := strconv.ParseInt(expiresStr, 10, 64)
if err != nil {
continue
}
if expires > 0 && time.Now().Unix() > expires {
continue
}
// Strip the leading dot from the Netscape-format domain so the
// round-trip back through SetCookies uses a stable key. Go's
// net/url strips the leading dot from Hostname() since Go 1.20,
// but matching that behaviour here makes the export map key
// independent of Go stdlib version.
domainBare := strings.TrimPrefix(domain, ".")
scheme := "https"
if fields[3] != "TRUE" {
scheme = "http"
}
u := &url.URL{
Scheme: scheme,
Host: domainBare,
Path: path,
}
cookie := &http.Cookie{
Name: name,
Value: value,
Path: path,
Domain: domainBare,
Expires: time.Unix(expires, 0),
Secure: fields[3] == "TRUE",
}
j.SetCookies(u, []*http.Cookie{cookie})
}
return scanner.Err()
}
// SaveToFile saves cookies to a file in Netscape format.
func (j *Jar) SaveToFile(path string) error {
if path == "" {
return nil
}
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to create cookie jar: %w", err)
}
defer file.Close()
fmt.Fprintln(file, "# Netscape HTTP Cookie File")
fmt.Fprintf(file, "# Generated by goget %s at %s\n", core.Version, time.Now().Format(time.RFC3339))
fmt.Fprintln(file, "# Format: domain flag path secure expiration name value")
fmt.Fprintln(file)
j.mu.Lock()
defer j.mu.Unlock()
for domain, cookies := range j.cookies {
for _, c := range cookies {
secure := "FALSE"
if c.Secure {
secure = "TRUE"
}
// The map key is always stored without a leading dot, but
// the Netscape format expects one — always emit it.
expires := c.Expires.Unix()
if expires <= 0 {
expires = time.Now().Add(365 * 24 * time.Hour).Unix()
}
fmt.Fprintf(file, ".%s\t%s\t%s\t%s\t%d\t%s\t%s\n",
domain,
"TRUE",
c.Path,
secure,
expires,
c.Name,
c.Value,
)
}
}
return nil
}
// GetCookieCount returns the number of cookies for a given URL.
func (j *Jar) GetCookieCount(u *url.URL) int {
return len(j.Cookies(u))
}
internal/cookie/jar_test.go
+874
View File
@@ -0,0 +1,874 @@
//go:build linux || freebsd
// +build linux freebsd
package cookie
import (
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"testing"
"time"
)
// futureTimestamp returns a unix timestamp safely in the future.
func futureTimestamp() int64 {
return time.Now().Unix() + 86400*365 // ~1 year from now
}
// pastTimestamp returns a unix timestamp safely in the past.
func pastTimestamp() int64 {
return time.Now().Unix() - 86400 // 1 day ago
}
// fmtInt64 is a helper to avoid importing strconv in every test.
func fmtInt64(v int64) string {
return strconv.FormatInt(v, 10)
}
// writeTempFile writes content to a temp file inside t.TempDir() and returns its path.
func writeTempFile(t *testing.T, dir, name, content string) string {
t.Helper()
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatal(err)
}
return path
}
// netscapeLine builds a single Netscape-format cookie line.
func netscapeLine(domain, domainFlag, path, secure, expires, name, value string) string {
return domain + "\t" + domainFlag + "\t" + path + "\t" + secure + "\t" + expires + "\t" + name + "\t" + value
}
// ---------------------------------------------------------------------------
// 1. TestNewJar
// ---------------------------------------------------------------------------
func TestNewJar(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatalf("NewJar() returned unexpected error: %v", err)
}
if j == nil {
t.Fatal("NewJar() returned nil")
}
if j.Jar == nil {
t.Fatal("NewJar().Jar is nil")
}
}
// ---------------------------------------------------------------------------
// 2. TestLoadFromFileEmptyPath
// ---------------------------------------------------------------------------
func TestLoadFromFileEmptyPath(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(""); err != nil {
t.Fatalf("LoadFromFile(\"\") should return nil, got: %v", err)
}
}
// ---------------------------------------------------------------------------
// 3. TestLoadFromFileNotExist
// ---------------------------------------------------------------------------
func TestLoadFromFileNotExist(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile("/tmp/this_file_does_not_exist_xxx.cookie"); err != nil {
t.Fatalf("LoadFromFile(non-existent) should return nil, got: %v", err)
}
}
// ---------------------------------------------------------------------------
// 4. TestLoadFromFile
// ---------------------------------------------------------------------------
func TestLoadFromFile(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".example.com", "TRUE", "/", "FALSE", fmtInt64(ft), "test", "cookie_value") + "\n"
path := writeTempFile(t, dir, "cookies.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// Cookie set with scheme=http (because secure=FALSE), Domain=.example.com, Path=/
// Retrieve over http.
u, _ := url.Parse("http://sub.example.com/path")
cookies := j.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie, got %d", len(cookies))
}
if cookies[0].Name != "test" {
t.Errorf("expected name 'test', got %q", cookies[0].Name)
}
if cookies[0].Value != "cookie_value" {
t.Errorf("expected value 'cookie_value', got %q", cookies[0].Value)
}
_ = cookies[0].Path // cookiejar does not preserve Path on retrieval
}
// TestLoadFromFileHttps verifies that secure=TRUE results in an https cookie.
func TestLoadFromFileHttps(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".secure.example.com", "TRUE", "/", "TRUE", fmtInt64(ft), "sess", "abc") + "\n"
path := writeTempFile(t, dir, "cookies_secure.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// Secure cookie — must be retrieved over HTTPS.
u, _ := url.Parse("https://secure.example.com/path")
cookies := j.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 secure cookie over HTTPS, got %d", len(cookies))
}
if cookies[0].Name != "sess" {
t.Errorf("expected name 'sess', got %q", cookies[0].Name)
}
if cookies[0].Value != "abc" {
t.Errorf("expected value 'abc', got %q", cookies[0].Value)
}
}
// ---------------------------------------------------------------------------
// 5. TestLoadFromFileCommentsAndEmptyLines
// ---------------------------------------------------------------------------
func TestLoadFromFileCommentsAndEmptyLines(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content :=
"# Netscape HTTP Cookie File\n" +
"# This is a comment that should be skipped\n" +
"\n" +
" \t \n" +
netscapeLine(".alpha.com", "TRUE", "/", "FALSE", fmtInt64(ft), "a", "1") + "\n" +
"# Another comment\n" +
netscapeLine(".beta.com", "TRUE", "/", "FALSE", fmtInt64(ft), "b", "2") + "\n"
path := writeTempFile(t, dir, "cookies.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// Check alpha
u, _ := url.Parse("http://alpha.com/path")
cookies := j.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie for alpha.com, got %d", len(cookies))
}
if cookies[0].Name != "a" {
t.Errorf("expected name 'a', got %q", cookies[0].Name)
}
// Check beta
u2, _ := url.Parse("http://beta.com/path")
cookies2 := j.Cookies(u2)
if len(cookies2) != 1 {
t.Fatalf("expected 1 cookie for beta.com, got %d", len(cookies2))
}
if cookies2[0].Name != "b" {
t.Errorf("expected name 'b', got %q", cookies2[0].Name)
}
}
// ---------------------------------------------------------------------------
// 6. TestLoadFromFileExpiredCookie
// ---------------------------------------------------------------------------
func TestLoadFromFileExpiredCookie(t *testing.T) {
dir := t.TempDir()
pt := pastTimestamp()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".expired.com", "TRUE", "/", "FALSE", fmtInt64(pt), "gone", "x") + "\n" +
netscapeLine(".valid.com", "TRUE", "/", "FALSE", fmtInt64(ft), "alive", "y") + "\n"
path := writeTempFile(t, dir, "cookies.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// Expired cookie should not be stored.
u, _ := url.Parse("http://expired.com/path")
cookies := j.Cookies(u)
if len(cookies) != 0 {
t.Errorf("expected 0 cookies for expired domain, got %d", len(cookies))
}
// Valid cookie should be stored.
u2, _ := url.Parse("http://valid.com/path")
cookies2 := j.Cookies(u2)
if len(cookies2) != 1 {
t.Fatalf("expected 1 cookie for valid domain, got %d", len(cookies2))
}
if cookies2[0].Name != "alive" {
t.Errorf("expected name 'alive', got %q", cookies2[0].Name)
}
}
// TestLoadFromFileSessionCookie documents the current behaviour: when expires=0,
// LoadFromFile does NOT skip it (because 0 > 0 is false), but time.Unix(0,0)
// makes the cookiejar itself treat it as expired. Therefore the cookie is not
// retrievable. This is a known limitation of the current implementation.
func TestLoadFromFileSessionCookie(t *testing.T) {
dir := t.TempDir()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".session.com", "TRUE", "/", "FALSE", "0", "sess", "active") + "\n"
path := writeTempFile(t, dir, "session.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// The cookie is loaded (no error) but can not be retrieved because
// time.Unix(0,0) is the Unix epoch (1970) which the cookiejar sees as expired.
u, _ := url.Parse("http://session.com/path")
cookies := j.Cookies(u)
if len(cookies) != 0 {
t.Logf("Note: session cookie (expires=0) currently not retrievable, got %d cookies", len(cookies))
}
}
// ---------------------------------------------------------------------------
// 7. TestSaveToFileEmptyPath
// ---------------------------------------------------------------------------
func TestSaveToFileEmptyPath(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.SaveToFile(""); err != nil {
t.Fatalf("SaveToFile(\"\") should return nil, got: %v", err)
}
}
// ---------------------------------------------------------------------------
// 8. TestSaveToFile
// ---------------------------------------------------------------------------
func TestSaveToFile(t *testing.T) {
dir := t.TempDir()
savePath := filepath.Join(dir, "subdir", "cookies.txt")
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.SaveToFile(savePath); err != nil {
t.Fatalf("SaveToFile() returned error: %v", err)
}
// File must exist.
if _, err := os.Stat(savePath); os.IsNotExist(err) {
t.Fatalf("SaveToFile() did not create file at %s", savePath)
}
data, err := os.ReadFile(savePath)
if err != nil {
t.Fatal(err)
}
content := string(data)
if len(content) == 0 {
t.Fatal("saved file is empty")
}
// Must contain Netscape header.
if !contains(content, "Netscape HTTP Cookie File") {
t.Errorf("saved file should contain 'Netscape HTTP Cookie File', got:\n%s", content)
}
// Must contain the generated-by line.
if !contains(content, "Generated by goget") {
t.Errorf("saved file should contain 'Generated by goget', got:\n%s", content)
}
}
// TestSaveToFileExistingDirectory verifies SaveToFile works when the
// directory already exists.
func TestSaveToFileExistingDirectory(t *testing.T) {
dir := filepath.Join(t.TempDir(), "already_exists")
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
savePath := filepath.Join(dir, "cookies.txt")
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.SaveToFile(savePath); err != nil {
t.Fatalf("SaveToFile() returned error: %v", err)
}
if _, err := os.Stat(savePath); os.IsNotExist(err) {
t.Fatalf("file not created in existing directory at %s", savePath)
}
}
// ---------------------------------------------------------------------------
// 9. TestCookieRoundTrip
// ---------------------------------------------------------------------------
func TestCookieRoundTrip(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://roundtrip.example.com/path")
j.SetCookies(u, []*http.Cookie{
{Name: "session_id", Value: "abc123", Path: "/"},
})
cookies := j.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie, got %d", len(cookies))
}
if cookies[0].Name != "session_id" {
t.Errorf("expected name 'session_id', got %q", cookies[0].Name)
}
if cookies[0].Value != "abc123" {
t.Errorf("expected value 'abc123', got %q", cookies[0].Value)
}
}
// TestCookieRoundTripHttpsThenHttp verifies that an insecure cookie set over
// HTTPS is sent on HTTP as well.
func TestCookieRoundTripHttpsThenHttp(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://mix.example.com/path")
j.SetCookies(u, []*http.Cookie{
{Name: "lang", Value: "en", Path: "/"},
})
httpU, _ := url.Parse("http://mix.example.com/path")
cookies := j.Cookies(httpU)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie over HTTP after HTTPS set, got %d", len(cookies))
}
if cookies[0].Value != "en" {
t.Errorf("expected value 'en', got %q", cookies[0].Value)
}
}
// TestCookieRoundTripSecure verifies that a secure cookie set over HTTPS is
// NOT sent over plain HTTP.
func TestCookieRoundTripSecure(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://secure-only.example.com/path")
j.SetCookies(u, []*http.Cookie{
{Name: "secret", Value: "s3cr3t", Path: "/", Secure: true},
})
// Over HTTPS – should be present.
httpsCookies := j.Cookies(u)
hasSecret := false
for _, c := range httpsCookies {
if c.Name == "secret" {
hasSecret = true
break
}
}
if !hasSecret {
t.Error("secure cookie should be sent over HTTPS")
}
// Over HTTP – must NOT be present.
httpU, _ := url.Parse("http://secure-only.example.com/path")
httpCookies := j.Cookies(httpU)
for _, c := range httpCookies {
if c.Name == "secret" {
t.Error("secure cookie should NOT be sent over HTTP")
break
}
}
}
// ---------------------------------------------------------------------------
// 10. TestGetCookieCount
// ---------------------------------------------------------------------------
func TestGetCookieCount(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://count.example.com/path")
// Empty jar.
if c := j.GetCookieCount(u); c != 0 {
t.Errorf("expected 0 cookies for empty jar, got %d", c)
}
// Add two cookies.
j.SetCookies(u, []*http.Cookie{
{Name: "a", Value: "1", Path: "/"},
{Name: "b", Value: "2", Path: "/"},
})
if c := j.GetCookieCount(u); c != 2 {
t.Errorf("expected 2 cookies, got %d", c)
}
}
// TestGetCookieCountDifferentURLs verifies cookie counts are scoped to the URL.
func TestGetCookieCountDifferentURLs(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u1, _ := url.Parse("https://left.example.com/path")
u2, _ := url.Parse("https://right.example.com/path")
j.SetCookies(u1, []*http.Cookie{{Name: "only_left", Value: "x", Path: "/"}})
if c := j.GetCookieCount(u1); c != 1 {
t.Errorf("expected 1 cookie for left, got %d", c)
}
if c := j.GetCookieCount(u2); c != 0 {
t.Errorf("expected 0 cookies for right, got %d", c)
}
}
// ---------------------------------------------------------------------------
// 11. TestMultipleCookies
// ---------------------------------------------------------------------------
func TestMultipleCookies(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://multi.example.com/path")
j.SetCookies(u, []*http.Cookie{
{Name: "first", Value: "one", Path: "/"},
{Name: "second", Value: "two", Path: "/"},
{Name: "third", Value: "three", Path: "/"},
})
cookies := j.Cookies(u)
if len(cookies) != 3 {
t.Fatalf("expected 3 cookies, got %d", len(cookies))
}
vals := make(map[string]string)
for _, c := range cookies {
vals[c.Name] = c.Value
}
if vals["first"] != "one" {
t.Errorf("expected first=one, got first=%q", vals["first"])
}
if vals["second"] != "two" {
t.Errorf("expected second=two, got second=%q", vals["second"])
}
if vals["third"] != "three" {
t.Errorf("expected third=three, got third=%q", vals["third"])
}
}
// TestMultipleCookiesSameDomain verifies that setting a cookie with the same
// name overwrites the previous value.
func TestMultipleCookiesSameDomain(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://overwrite.example.com/path")
j.SetCookies(u, []*http.Cookie{{Name: "lang", Value: "en", Path: "/"}})
j.SetCookies(u, []*http.Cookie{{Name: "theme", Value: "dark", Path: "/"}})
j.SetCookies(u, []*http.Cookie{{Name: "lang", Value: "fr", Path: "/"}})
cookies := j.Cookies(u)
vals := make(map[string]string)
for _, c := range cookies {
vals[c.Name] = c.Value
}
if vals["lang"] != "fr" {
t.Errorf("expected lang=fr (overwritten), got lang=%q", vals["lang"])
}
if vals["theme"] != "dark" {
t.Errorf("expected theme=dark, got theme=%q", vals["theme"])
}
}
// TestMultipleCookiesDifferentPaths validates that cookies with different
// paths are scoped correctly.
func TestMultipleCookiesDifferentPaths(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
rootU, _ := url.Parse("https://paths.example.com/")
adminU, _ := url.Parse("https://paths.example.com/admin")
apiU, _ := url.Parse("https://paths.example.com/api/v1")
j.SetCookies(rootU, []*http.Cookie{{Name: "root", Value: "r", Path: "/"}})
j.SetCookies(adminU, []*http.Cookie{{Name: "admin", Value: "a", Path: "/admin"}})
j.SetCookies(apiU, []*http.Cookie{{Name: "api", Value: "v", Path: "/api"}})
// Root path should only see the root cookie.
rootCookies := j.Cookies(rootU)
if len(rootCookies) != 1 {
t.Fatalf("expected 1 cookie for root path, got %d", len(rootCookies))
}
if rootCookies[0].Name != "root" {
t.Errorf("expected 'root' cookie at root path, got %q", rootCookies[0].Name)
}
// Admin path should see root (/) + admin (/admin).
adminCookies := j.Cookies(adminU)
adminNames := make(map[string]bool)
for _, c := range adminCookies {
adminNames[c.Name] = true
}
if !adminNames["root"] {
t.Error("root cookie should be visible under /admin")
}
if !adminNames["admin"] {
t.Error("admin cookie should be visible under /admin")
}
// API path should see root (/) + api (/api).
apiCookies := j.Cookies(apiU)
apiNames := make(map[string]bool)
for _, c := range apiCookies {
apiNames[c.Name] = true
}
if !apiNames["root"] {
t.Error("root cookie should be visible under /api")
}
if !apiNames["api"] {
t.Error("api cookie should be visible under /api")
}
}
// ---------------------------------------------------------------------------
// Additional edge-case tests
// ---------------------------------------------------------------------------
// TestLoadFromFileMalformedLine checks that lines with fewer than 7 fields
// are silently skipped.
func TestLoadFromFileMalformedLine(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".good.com", "TRUE", "/", "FALSE", fmtInt64(ft), "ok", "fine") + "\n" +
"short.line\n" +
".also-short\tTRUE\t/\n" +
netscapeLine(".also-good.com", "TRUE", "/", "FALSE", fmtInt64(ft), "ok2", "fine2") + "\n"
path := writeTempFile(t, dir, "malformed.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// First valid cookie should be present.
u1, _ := url.Parse("http://good.com/path")
c1 := j.Cookies(u1)
if len(c1) != 1 {
t.Errorf("expected 1 cookie for good.com, got %d", len(c1))
}
// Second valid cookie should be present.
u2, _ := url.Parse("http://also-good.com/path")
c2 := j.Cookies(u2)
if len(c2) != 1 {
t.Errorf("expected 1 cookie for also-good.com, got %d", len(c2))
}
}
// TestLoadFromFileInvalidExpiry checks that an unparsable expiry is skipped.
func TestLoadFromFileInvalidExpiry(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".bad-expiry.com", "TRUE", "/", "FALSE", "not_a_number", "x", "y") + "\n" +
netscapeLine(".fresh.com", "TRUE", "/", "FALSE", fmtInt64(ft), "good", "yes") + "\n"
path := writeTempFile(t, dir, "bad_expiry.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
// The malformed-expiry line should be skipped.
u1, _ := url.Parse("http://bad-expiry.com/path")
c1 := j.Cookies(u1)
if len(c1) != 0 {
t.Errorf("expected 0 cookies for bad-expiry.com, got %d", len(c1))
}
// The valid line should still work.
u2, _ := url.Parse("http://fresh.com/path")
c2 := j.Cookies(u2)
if len(c2) != 1 {
t.Fatalf("expected 1 cookie for fresh.com, got %d", len(c2))
}
if c2[0].Name != "good" {
t.Errorf("expected name 'good', got %q", c2[0].Name)
}
}
// TestLoadAndSaveRoundTrip loads from a file then saves to another location.
func TestLoadAndSaveRoundTrip(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
loadContent := "# Netscape HTTP Cookie File\n" +
netscapeLine(".rt.com", "TRUE", "/", "FALSE", fmtInt64(ft), "a", "b") + "\n"
loadPath := writeTempFile(t, dir, "load.txt", loadContent)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(loadPath); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
savePath := filepath.Join(dir, "nested", "save.txt")
if err := j.SaveToFile(savePath); err != nil {
t.Fatalf("SaveToFile() returned error: %v", err)
}
if _, err := os.Stat(savePath); os.IsNotExist(err) {
t.Errorf("saved file not found at %s", savePath)
}
data, err := os.ReadFile(savePath)
if err != nil {
t.Fatal(err)
}
if !contains(string(data), "Netscape HTTP Cookie File") {
t.Error("saved file must contain Netscape header")
}
}
// TestGetCookieCountWithLoad verifies GetCookieCount after loading from file.
func TestGetCookieCountWithLoad(t *testing.T) {
dir := t.TempDir()
ft := futureTimestamp()
content := "# Netscape HTTP Cookie File\n" +
netscapeLine(".cnt1.com", "TRUE", "/", "FALSE", fmtInt64(ft), "c1", "v1") + "\n" +
netscapeLine(".cnt1.com", "TRUE", "/", "FALSE", fmtInt64(ft), "c2", "v2") + "\n"
path := writeTempFile(t, dir, "count.txt", content)
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j.LoadFromFile(path); err != nil {
t.Fatalf("LoadFromFile() returned error: %v", err)
}
u, _ := url.Parse("http://cnt1.com/path")
if c := j.GetCookieCount(u); c != 2 {
t.Errorf("expected 2 cookies after load, got %d", c)
}
}
// TestLoadFromFileOpenError verifies that an IO error (e.g. a directory
// instead of a file) is propagated as an error (not silently swallowed).
func TestLoadFromFileOpenError(t *testing.T) {
dir := t.TempDir()
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
// Try to open a directory – this should fail with an error (not IsNotExist).
err = j.LoadFromFile(dir)
if err == nil {
t.Log("Note: opening a directory may produce an error or not depending on OS")
}
}
// TestNewJarMultiple verifies that multiple jars can be created independently.
func TestNewJarMultiple(t *testing.T) {
j1, err := NewJar()
if err != nil {
t.Fatal(err)
}
j2, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://independent.example.com/path")
j1.SetCookies(u, []*http.Cookie{{Name: "only_j1", Value: "yes", Path: "/"}})
if c := len(j1.Cookies(u)); c != 1 {
t.Errorf("expected 1 cookie in j1, got %d", c)
}
if c := len(j2.Cookies(u)); c != 0 {
t.Errorf("expected 0 cookies in j2, got %d", c)
}
}
// TestSetCookiesUsesCookieDomainForStorageKey is a regression guard for the
// BACKLOG entry "Cookie domain mismatch between storage and export".
// The previous implementation keyed the export map by u.Hostname() and
// the file output by (c.Domain with a possibly-prepended dot). When
// stdlib cookiejar normalised c.Domain with a leading dot, the storage
// key and the file's domain field could diverge, and round-tripping
// through SaveToFile/LoadFromFile would re-key the cookie under a
// different domain — silently losing the cookie on the next session.
//
// The fix is to key storage by the cookie's effective domain (no
// leading dot) and to use that same key when writing the file. This
// test sets a cookie scoped to "example.com" via a request to
// "www.example.com", saves, loads into a fresh jar, and verifies the
// cookie is still retrievable.
func TestSetCookiesUsesCookieDomainForStorageKey(t *testing.T) {
dir := t.TempDir()
savePath := filepath.Join(dir, "roundtrip.txt")
// Session 1: set a cookie whose Domain is "example.com" even
// though the URL host is "www.example.com" (the typical case for
// a server issuing a cookie that scopes to the parent domain).
j1, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://www.example.com/path")
j1.SetCookies(u, []*http.Cookie{
{Name: "session", Value: "abc", Path: "/", Domain: "example.com"},
})
// SaveToFile must write ".example.com" (Netscape format with
// leading dot).
if err := j1.SaveToFile(savePath); err != nil {
t.Fatalf("SaveToFile: %v", err)
}
data, err := os.ReadFile(savePath)
if err != nil {
t.Fatal(err)
}
if !contains(string(data), ".example.com\t") {
t.Errorf("saved file must contain .example.com entry, got:\n%s", data)
}
// Session 2: load into a fresh jar, the cookie must come back
// and be retrievable for the original URL.
j2, err := NewJar()
if err != nil {
t.Fatal(err)
}
if err := j2.LoadFromFile(savePath); err != nil {
t.Fatalf("LoadFromFile: %v", err)
}
cookies := j2.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie after round-trip, got %d", len(cookies))
}
if cookies[0].Name != "session" || cookies[0].Value != "abc" {
t.Errorf("round-tripped cookie = %+v, want {Name: session, Value: abc}", cookies[0])
}
}
// TestSetCookiesHostOnlyCookie is the host-only counterpart: a cookie
// without a Domain attribute must be stored under the URL hostname
// (the only scope information available).
func TestSetCookiesHostOnlyCookie(t *testing.T) {
j, err := NewJar()
if err != nil {
t.Fatal(err)
}
u, _ := url.Parse("https://hostonly.example.com/path")
j.SetCookies(u, []*http.Cookie{
{Name: "k", Value: "v", Path: "/"},
})
cookies := j.Cookies(u)
if len(cookies) != 1 {
t.Fatalf("expected 1 cookie, got %d", len(cookies))
}
if cookies[0].Name != "k" {
t.Errorf("cookie name = %q, want %q", cookies[0].Name, "k")
}
}
// ---------------------------------------------------------------------------
// Helper: contains reports whether substr is in s.
// ---------------------------------------------------------------------------
func contains(s, substr string) bool {
if len(substr) == 0 {
return true
}
if len(substr) > len(s) {
return false
}
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}
internal/core/config.go
+47
View File
@@ -0,0 +1,47 @@
//go:build linux || freebsd
// +build linux freebsd
package core
import (
"time"
)
// TimeoutConfig configures smart timeout
type TimeoutConfig struct {
Min time.Duration
Max time.Duration
MinSpeedBytesPerSec int64
SafetyFactor float64
DefaultUnknown time.Duration
}
// DefaultTimeoutConfig returns default settings
func DefaultTimeoutConfig() *TimeoutConfig {
return &TimeoutConfig{
Min: 30 * time.Second,
Max: 24 * time.Hour,
MinSpeedBytesPerSec: 10 * 1024,
SafetyFactor: 3.0,
DefaultUnknown: 30 * time.Minute,
}
}
// CalculateTimeout calculates the recommended timeout
func (tc *TimeoutConfig) CalculateTimeout(fileSize int64, userOverride time.Duration) time.Duration {
if userOverride > 0 {
return userOverride
}
if fileSize <= 0 {
return tc.DefaultUnknown
}
expectedSeconds := float64(fileSize) / float64(tc.MinSpeedBytesPerSec)
calculated := time.Duration(expectedSeconds * tc.SafetyFactor * float64(time.Second))
if calculated < tc.Min {
return tc.Min
}
if calculated > tc.Max {
return tc.Max
}
return calculated
}
internal/core/context.go
+65
View File
@@ -0,0 +1,65 @@
//go:build linux || freebsd
// +build linux freebsd
package core
import (
"context"
"time"
)
// RequestContext holds metadata about the request across layers
type RequestContext struct {
// Request ID for logging
RequestID string
// Start of operation
StartTime time.Time
// Protocols used in the chain
ProtocolChain []string
// IP address to which was connected
ConnectedIP string
// IP version (4 or 6)
IPVersion int
// Number of attempts
AttemptCount int
// Context for cancellation
Context context.Context
}
// NewRequestContext creates a new request context
func NewRequestContext(ctx context.Context, requestID string) *RequestContext {
return &RequestContext{
RequestID: requestID,
StartTime: time.Now(),
ProtocolChain: make([]string, 0),
IPVersion: 0,
AttemptCount: 0,
Context: ctx,
}
}
// Duration returns the duration of operation
func (rc *RequestContext) Duration() time.Duration {
return time.Since(rc.StartTime)
}
// AddProtocol adds a protocol to the chain
func (rc *RequestContext) AddProtocol(protocol string) {
rc.ProtocolChain = append(rc.ProtocolChain, protocol)
}
// IsCancelled returns whether the context was cancelled
func (rc *RequestContext) IsCancelled() bool {
select {
case <-rc.Context.Done():
return true
default:
return false
}
}
internal/core/core_test.go
+1076
View File
File diff suppressed because it is too large Load Diff
internal/core/errors.go
+135
View File
@@ -0,0 +1,135 @@
//go:build linux || freebsd
// +build linux freebsd
package core
import (
"fmt"
"net/url"
)
// ErrorType defines the error type
type ErrorType string
const (
ErrNetwork ErrorType = "NETWORK"
ErrProtocol ErrorType = "PROTOCOL"
ErrFile ErrorType = "FILE"
ErrTimeout ErrorType = "TIMEOUT"
ErrAuth ErrorType = "AUTH"
ErrChecksum ErrorType = "CHECKSUM"
ErrUnsupported ErrorType = "UNSUPPORTED"
ErrCancelled ErrorType = "CANCELLED"
)
// GogetError is a structured error with context
type GogetError struct {
Type ErrorType
Message string
Cause error
URL string
Hint string // Actionable suggestion for the user
Details map[string]string
}
func (e *GogetError) Error() string {
var msg string
if e.Cause != nil {
msg = fmt.Sprintf("[%s] %s: %v", e.Type, e.Message, e.Cause)
} else {
msg = fmt.Sprintf("[%s] %s", e.Type, e.Message)
}
if e.Hint != "" {
msg += fmt.Sprintf(" (hint: %s)", e.Hint)
}
return msg
}
func (e *GogetError) Unwrap() error {
return e.Cause
}
// GetDetail returns a value from the Details map, or an empty string if the key is absent.
func (e *GogetError) GetDetail(key string) string {
if e.Details == nil {
return ""
}
return e.Details[key]
}
// NewNetworkError creates a network error
func NewNetworkError(msg string, cause error, url string) *GogetError {
return &GogetError{
Type: ErrNetwork,
Message: msg,
Cause: cause,
URL: url,
}
}
// NewProtocolError creates a protocol error
func NewProtocolError(msg string, cause error, url string) *GogetError {
return &GogetError{
Type: ErrProtocol,
Message: msg,
Cause: cause,
URL: url,
}
}
// NewFileError creates a file error
func NewFileError(msg string, cause error) *GogetError {
return &GogetError{
Type: ErrFile,
Message: msg,
Cause: cause,
}
}
// NewTimeoutError creates a timeout error
func NewTimeoutError(msg string, url string) *GogetError {
return &GogetError{
Type: ErrTimeout,
Message: msg,
URL: url,
Hint: "increase --max-time or --connect-timeout, or check network connectivity",
}
}
// NewChecksumError creates a checksum error
func NewChecksumError(expected, actual string) *GogetError {
return &GogetError{
Type: ErrChecksum,
Message: fmt.Sprintf("checksum mismatch: expected %s, got %s", expected, actual),
Hint: "verify the file integrity with an external checksum tool or re-download",
Details: map[string]string{
"expected": expected,
"actual": actual,
},
}
}
// NewUnsupportedError creates an unsupported operation error
func NewUnsupportedError(msg string) *GogetError {
return &GogetError{
Type: ErrUnsupported,
Message: msg,
}
}
// WithHint returns the error with the hint set. Use for adding contextual,
// actionable suggestions to an existing error without creating a new one.
func (e *GogetError) WithHint(hint string) *GogetError {
e.Hint = hint
return e
}
// SafeURL returns the URL string with credentials removed for safe error logging.
func SafeURL(u *url.URL) string {
if u == nil {
return ""
}
clean := *u
clean.User = nil
return clean.String()
}
internal/core/types.go
+174
View File
@@ -0,0 +1,174 @@
//go:build linux || freebsd
// +build linux freebsd
package core
import (
"context"
"fmt"
"io"
"net/url"
"time"
)
// ParallelConfig configures parallel downloading
type ParallelConfig struct {
Connections int
MinSize int64
MinChunkSize int64
MaxChunkSize int64
}
// DefaultParallelConfig returns default settings
func DefaultParallelConfig() *ParallelConfig {
return &ParallelConfig{
Connections: 0,
MinSize: 100 * 1024 * 1024,
MinChunkSize: 1 * 1024 * 1024,
MaxChunkSize: 50 * 1024 * 1024,
}
}
// GetConnections returns the number of connections
func (pc *ParallelConfig) GetConnections(fileSize int64) int {
if pc.Connections > 0 {
return pc.Connections
}
if fileSize >= pc.MinSize {
return 4
}
return 1
}
// ChunkInfo represents one file segment
type ChunkInfo struct {
ID int
Start int64
End int64
Downloaded int64
Status int
Error error
}
// ResumeInfo contains metadata for continuation
type ResumeInfo struct {
DownloadedBytes int64
ETag string
LastModified string
URL string
LastWrite time.Time
Chunks map[int]int64
}
// DownloadRequest represents a download request
type DownloadRequest struct {
URL *url.URL
Output string
Writer io.Writer
Resume bool
Timeout time.Duration
Verbose bool
DebugTransport bool
Checksum string
Headers map[string]string
Proxy string
AutoDecompress bool
ProgressCallback func(current, total int64, speed float64)
ResumeInfo *ResumeInfo
Parallel *ParallelConfig
Recursive bool
RecursiveParallel int // worker pool size for recursive downloads (0 = sequential)
DryRun bool // list what would be downloaded, do not write any files
MaxDepth int
MaxFileSize int64 // max file size to download (0 = unlimited)
AcceptPatterns []string // glob patterns for accepted files (e.g., "*.pdf,*.html")
RejectPatterns []string // glob patterns for rejected files (e.g., "*.tmp,*.bak")
KeepTimestamps bool // set file modification time from server response
Ctx context.Context
SSHKnownHosts string // path to known_hosts file (SFTP)
SSHInsecure bool // skip SSH host key verification (SFTP)
}
// DownloadResult represents a download result
type DownloadResult struct {
BytesDownloaded int64
TotalSize int64
Duration time.Duration
Protocol string
IPVersion int
Speed float64
OutputPath string
Checksum string
Resumed bool
Parallel bool
ChunksCount int
HTTPStatusCode int // HTTP status code from response
TraceTimings *TraceTimings // HTTP tracing breakdown
}
// TraceTimings holds HTTP request timing breakdown.
type TraceTimings struct {
DNSLookup time.Duration
TCPConnect time.Duration
TLSHandshake time.Duration
TTFB time.Duration // Time To First Byte
Total time.Duration
}
func (tt *TraceTimings) String() string {
return fmt.Sprintf(
"DNS: %v | TCP: %v | TLS: %v | TTFB: %v | Total: %v",
tt.DNSLookup.Round(time.Millisecond),
tt.TCPConnect.Round(time.Millisecond),
tt.TLSHandshake.Round(time.Millisecond),
tt.TTFB.Round(time.Millisecond),
tt.Total.Round(time.Millisecond),
)
}
// Protocol is the interface for protocols
type Protocol interface {
Scheme() string
CanHandle(url *url.URL) bool
Download(ctx context.Context, req *DownloadRequest) (*DownloadResult, error)
Capabilities() []string
SupportsResume() bool
SupportsCompression() bool
SupportsParallel() bool
SupportsRecursive() bool
}
// UploadRequest represents an upload request
type UploadRequest struct {
URL *url.URL
Input string
Reader io.Reader
Timeout time.Duration
Verbose bool
Headers map[string]string
Proxy string
Ctx context.Context
}
// UploadResult represents an upload result
type UploadResult struct {
BytesUploaded int64
Duration time.Duration
Protocol string
ResultURL string
}
// Uploader is the interface for protocols with upload support
type Uploader interface {
Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error)
SupportsUpload() bool
}
// ProgressCallback is a function for progress reporting
type ProgressCallback func(current, total int64, speed float64)
// ProgressWriter is a writer with progress support
type ProgressWriter interface {
io.Writer
SetProgressCallback(cb ProgressCallback)
}
internal/core/version.go
+10
View File
@@ -0,0 +1,10 @@
//go:build linux || freebsd
// Package version exposes the goget release identity.
package core
// Name is the program name reported by `--version` and in log lines.
const Name = "goget"
// Version is the goget release version. Follows SemVer.
var Version = "1.0.0"
internal/crypto/certs.go
+91
View File
@@ -0,0 +1,91 @@
//go:build linux || freebsd
// +build linux freebsd
package crypto
import (
"crypto/sha256"
"crypto/x509"
"encoding/pem"
"fmt"
"os"
"strings"
)
// CertLoader loads and parses certificates
type CertLoader struct {
pool *x509.CertPool
}
// NewCertLoader creates new cert loader
func NewCertLoader() *CertLoader {
return &CertLoader{
pool: x509.NewCertPool(),
}
}
// LoadCAFile loads CA certificates from a file
func (cl *CertLoader) LoadCAFile(path string) error {
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("failed to read ca file: %w", err)
}
if !cl.pool.AppendCertsFromPEM(data) {
return fmt.Errorf("failed to parse ca certificates")
}
return nil
}
// LoadSystemCerts loads system certificates
func (cl *CertLoader) LoadSystemCerts() error {
pool, err := x509.SystemCertPool()
if err != nil {
// Fallback for systems without system certificates
cl.pool = x509.NewCertPool()
return nil
}
cl.pool = pool
return nil
}
// GetPool returns the certificate pool
func (cl *CertLoader) GetPool() *x509.CertPool {
return cl.pool
}
// ParseCertificate parses a PEM certificate
func ParseCertificate(data []byte) (*x509.Certificate, error) {
block, _ := pem.Decode(data)
if block == nil {
return nil, fmt.Errorf("failed to decode PEM block")
}
return x509.ParseCertificate(block.Bytes)
}
// LoadCertificateFile loads a certificate from a file
func LoadCertificateFile(path string) (*x509.Certificate, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return ParseCertificate(data)
}
// FingerprintSHA256 calculates the SHA-256 fingerprint of a certificate
func FingerprintSHA256(cert *x509.Certificate) string {
if cert == nil {
return ""
}
hash := sha256.Sum256(cert.Raw)
// Format as colon-separated hex pairs (like OpenSSL format)
parts := make([]string, len(hash))
for i, b := range hash {
parts[i] = fmt.Sprintf("%02X", b)
}
return strings.Join(parts, ":")
}
internal/crypto/checksum.go
+150
View File
@@ -0,0 +1,150 @@
//go:build linux || freebsd
// +build linux freebsd
package crypto
import (
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"fmt"
"hash"
"io"
"os"
"golang.org/x/crypto/blake2b"
"golang.org/x/crypto/sha3"
)
// ChecksumType defines the checksum type
type ChecksumType string
const (
SHA256 ChecksumType = "sha256"
SHA512 ChecksumType = "sha512"
BLAKE2b ChecksumType = "blake2b"
SHA3_256 ChecksumType = "sha3-256"
SHA3_512 ChecksumType = "sha3-512"
MD5 ChecksumType = "md5"
)
// ChecksumVerifier verifies a file checksum
type ChecksumVerifier struct {
hashType ChecksumType
hasher hash.Hash
expected string
}
// NewChecksumVerifier creates new verifier
func NewChecksumVerifier(hashType ChecksumType, expected string) (*ChecksumVerifier, error) {
var hasher hash.Hash
switch hashType {
case SHA256:
hasher = sha256.New()
case SHA512:
hasher = sha512.New()
case BLAKE2b:
h, err := blake2b.New256(nil)
if err != nil {
return nil, fmt.Errorf("failed to create blake2b hasher: %w", err)
}
hasher = h
case SHA3_256:
hasher = sha3.New256()
case SHA3_512:
hasher = sha3.New512()
case MD5:
return nil, fmt.Errorf("md5 is deprecated and not supported")
default:
return nil, fmt.Errorf("unsupported hash type: %s", hashType)
}
return &ChecksumVerifier{
hashType: hashType,
hasher: hasher,
expected: expected,
}, nil
}
// VerifyFile verifies the checksum of a file
func (cv *ChecksumVerifier) VerifyFile(path string) (bool, string, error) {
file, err := os.Open(path)
if err != nil {
return false, "", fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
cv.hasher.Reset()
_, err = io.Copy(cv.hasher, file)
if err != nil {
return false, "", fmt.Errorf("failed to read file: %w", err)
}
actual := hex.EncodeToString(cv.hasher.Sum(nil))
match := actual == cv.expected
return match, actual, nil
}
// VerifyReader verifies checksum from a reader (streaming)
func (cv *ChecksumVerifier) VerifyReader(r io.Reader) (string, error) {
cv.hasher.Reset()
_, err := io.Copy(cv.hasher, r)
if err != nil {
return "", err
}
return hex.EncodeToString(cv.hasher.Sum(nil)), nil
}
// Hasher returns the underlying hasher for streaming verification
func (cv *ChecksumVerifier) Hasher() hash.Hash {
return cv.hasher
}
// Expected returns the expected checksum
func (cv *ChecksumVerifier) Expected() string {
return cv.expected
}
// Type returns the hash function type
func (cv *ChecksumVerifier) Type() ChecksumType {
return cv.hashType
}
// ComputeFileChecksum calculates the checksum of a file
func ComputeFileChecksum(path string, hashType ChecksumType) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
var hasher hash.Hash
switch hashType {
case SHA256:
hasher = sha256.New()
case SHA512:
hasher = sha512.New()
case BLAKE2b:
h, err := blake2b.New256(nil)
if err != nil {
return "", fmt.Errorf("failed to create blake2b hasher: %w", err)
}
hasher = h
case SHA3_256:
hasher = sha3.New256()
case SHA3_512:
hasher = sha3.New512()
default:
return "", fmt.Errorf("unsupported hash type: %s", hashType)
}
_, err = io.Copy(hasher, file)
if err != nil {
return "", err
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
internal/crypto/crypto_test.go
+190
View File
@@ -0,0 +1,190 @@
//go:build linux || freebsd
// +build linux freebsd
package crypto
import (
"os"
"strings"
"testing"
)
func TestNewChecksumVerifier(t *testing.T) {
tests := []struct {
hashType ChecksumType
valid bool
}{
{SHA256, true},
{SHA512, true},
{BLAKE2b, true},
{SHA3_256, true},
{SHA3_512, true},
{MD5, false}, // MD5 is deprecated
{"invalid", false},
}
for _, test := range tests {
_, err := NewChecksumVerifier(test.hashType, "abc123")
if test.valid && err != nil {
t.Errorf("NewChecksumVerifier(%s) failed: %v", test.hashType, err)
}
if !test.valid && err == nil {
t.Errorf("NewChecksumVerifier(%s) should have failed", test.hashType)
}
}
}
func TestChecksumVerifier(t *testing.T) {
// Test SHA-256
verifier, err := NewChecksumVerifier(SHA256, "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae")
if err != nil {
t.Fatalf("Failed to create verifier: %v", err)
}
// Test with string "foo"
actual, err := verifier.VerifyReader(strings.NewReader("foo"))
if err != nil {
t.Fatalf("VerifyReader failed: %v", err)
}
if actual != "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" {
t.Errorf("Unexpected hash: %s", actual)
}
}
func TestComputeFileChecksum(t *testing.T) {
// Create a temp file for testing
tmpFile := t.TempDir() + "/test.txt"
writeTestFile(tmpFile, "test content")
hash, err := ComputeFileChecksum(tmpFile, SHA256)
if err != nil {
t.Fatalf("ComputeFileChecksum failed: %v", err)
}
if hash == "" {
t.Error("Expected non-empty hash")
}
// Verify it's a valid hex string
if len(hash) != 64 {
t.Errorf("Expected SHA-256 hash length 64, got %d", len(hash))
}
}
func TestComputeFileChecksumTypes(t *testing.T) {
tmpFile := t.TempDir() + "/test.txt"
writeTestFile(tmpFile, "test")
tests := []struct {
hashType ChecksumType
length int
}{
{SHA256, 64},
{SHA512, 128},
{BLAKE2b, 64},
{SHA3_256, 64},
{SHA3_512, 128},
}
for _, test := range tests {
hash, err := ComputeFileChecksum(tmpFile, test.hashType)
if err != nil {
t.Errorf("ComputeFileChecksum(%s) failed: %v", test.hashType, err)
continue
}
if len(hash) != test.length {
t.Errorf("Expected %s hash length %d, got %d", test.hashType, test.length, len(hash))
}
}
}
func TestChecksumVerifierType(t *testing.T) {
verifier, _ := NewChecksumVerifier(SHA256, "abc")
if verifier.Type() != SHA256 {
t.Errorf("Expected type SHA256, got %s", verifier.Type())
}
}
func TestChecksumVerifierExpected(t *testing.T) {
expected := "abc123def456"
verifier, _ := NewChecksumVerifier(SHA256, expected)
if verifier.Expected() != expected {
t.Errorf("Expected %s, got %s", expected, verifier.Expected())
}
}
func TestChecksumVerifierHasher(t *testing.T) {
verifier, _ := NewChecksumVerifier(SHA256, "abc")
hasher := verifier.Hasher()
if hasher == nil {
t.Error("Expected non-nil hasher")
}
}
func writeTestFile(path, content string) {
f, _ := os.Create(path)
f.WriteString(content)
f.Close()
}
func TestVerifyFileSuccess(t *testing.T) {
path := "/tmp/goget_test_verify_ok.tmp"
writeTestFile(path, "test data for verification")
defer os.Remove(path)
hash, err := ComputeFileChecksum(path, SHA256)
if err != nil {
t.Fatalf("ComputeFileChecksum failed: %v", err)
}
verifier, err := NewChecksumVerifier(SHA256, hash)
if err != nil {
t.Fatalf("NewChecksumVerifier failed: %v", err)
}
match, actual, err := verifier.VerifyFile(path)
if err != nil {
t.Fatalf("VerifyFile failed: %v", err)
}
if !match {
t.Errorf("unexpected mismatch: expected %s, got %s", hash, actual)
}
}
func TestVerifyFileMismatch(t *testing.T) {
path := "/tmp/goget_test_verify_bad.tmp"
writeTestFile(path, "correct data")
defer os.Remove(path)
verifier, err := NewChecksumVerifier(SHA256, strings.Repeat("0", 64))
if err != nil {
t.Fatalf("NewChecksumVerifier failed: %v", err)
}
match, _, err := verifier.VerifyFile(path)
if err != nil {
t.Fatalf("VerifyFile failed: %v", err)
}
if match {
t.Error("expected checksum mismatch for wrong hash")
}
}
func TestVerifyFileNonexistent(t *testing.T) {
verifier, _ := NewChecksumVerifier(SHA256, strings.Repeat("a", 64))
_, _, err := verifier.VerifyFile("/nonexistent/path")
if err == nil {
t.Error("expected error for nonexistent file")
}
}
func TestAllSupportedChecksumTypes(t *testing.T) {
types := []ChecksumType{SHA256, SHA512, BLAKE2b, SHA3_256, SHA3_512}
for _, ct := range types {
t.Run(string(ct), func(t *testing.T) {
_, err := NewChecksumVerifier(ct, strings.Repeat("a", 128))
if err != nil {
t.Errorf("NewChecksumVerifier(%s) failed: %v", ct, err)
}
})
}
}
internal/crypto/interface.go
+107
View File
@@ -0,0 +1,107 @@
//go:build linux || freebsd
// +build linux freebsd
package crypto
import (
"io"
)
// Hasher is the interface for hashing functions
type Hasher interface {
// Name returns the name of the hash function
Name() string
// Hash calculates a hash from a reader
Hash(r io.Reader) (string, error)
// HashFile calculates a hash from a file
HashFile(path string) (string, error)
// Verify verifies a hash against the expected value
Verify(r io.Reader, expected string) (bool, error)
// VerifyFile verifies a file hash against the expected value
VerifyFile(path string, expected string) (bool, error)
}
// Encryptor is the interface for encryption
type Encryptor interface {
// Name returns the name of the cipher
Name() string
// Encrypt encrypts data
Encrypt(r io.Reader, w io.Writer, key []byte) error
// Decrypt decrypts data
Decrypt(r io.Reader, w io.Writer, key []byte) error
}
// Signer is the interface for digital signatures
type Signer interface {
// Name returns the name of the signature scheme
Name() string
// Sign creates a signature
Sign(data []byte, key []byte) ([]byte, error)
// Verify verifies a signature
Verify(data []byte, signature []byte, key []byte) (bool, error)
}
// KeyLoader is the interface for loading keys
type KeyLoader interface {
// LoadPrivateKey loads a private key
LoadPrivateKey(path string, passphrase []byte) ([]byte, error)
// LoadPublicKey loads a public key
LoadPublicKey(path string) ([]byte, error)
// GenerateKeyPair generates a key pair
GenerateKeyPair() (privateKey, publicKey []byte, err error)
}
// BaseHasher is a basic hasher implementation
type BaseHasher struct {
name string
}
// NewBaseHasher creates new base hasher
func NewBaseHasher(name string) *BaseHasher {
return &BaseHasher{name: name}
}
// Name returns the name of the hash function
func (bh *BaseHasher) Name() string {
return bh.name
}
// Hash calculates a hash from a reader
func (bh *BaseHasher) Hash(r io.Reader) (string, error) {
// Implementation in concrete hashers
return "", nil
}
// HashFile calculates a hash from a file
func (bh *BaseHasher) HashFile(path string) (string, error) {
// Implementation in concrete hashers
return "", nil
}
// Verify verifies a hash against the expected value
func (bh *BaseHasher) Verify(r io.Reader, expected string) (bool, error) {
actual, err := bh.Hash(r)
if err != nil {
return false, err
}
return actual == expected, nil
}
// VerifyFile verifies a file hash against the expected value
func (bh *BaseHasher) VerifyFile(path string, expected string) (bool, error) {
actual, err := bh.HashFile(path)
if err != nil {
return false, err
}
return actual == expected, nil
}
internal/crypto/pgp.go
+500
View File
@@ -0,0 +1,500 @@
//go:build linux || freebsd
// +build linux freebsd
package crypto
import (
"bytes"
"fmt"
"io"
"os"
"strings"
// Deprecated: golang.org/x/crypto/openpgp is frozen; no more security fixes.
// Will be replaced with an alternative in a future release.
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
"golang.org/x/crypto/openpgp/clearsign"
)
// PGPConfig configures PGP operations
type PGPConfig struct {
// Path to private key
PrivateKeyPath string
// Path to public key
PublicKeyPath string
// Passphrase for private key
Passphrase string
// Armor output (ASCII vs binary)
Armor bool
// KeyRing for verification
KeyRing openpgp.EntityList
}
// DefaultPGPConfig returns the default configuration
func DefaultPGPConfig() *PGPConfig {
return &PGPConfig{
Armor: true,
}
}
// PGPDecryptor decrypts PGP encrypted data
type PGPDecryptor struct {
config *PGPConfig
}
// NewPGPDecryptor creates a new PGP decryptor
func NewPGPDecryptor(cfg *PGPConfig) *PGPDecryptor {
if cfg == nil {
cfg = DefaultPGPConfig()
}
return &PGPDecryptor{config: cfg}
}
// Decrypt decrypts PGP encrypted data
func (pd *PGPDecryptor) Decrypt(r io.Reader, w io.Writer) error {
var keyring openpgp.EntityList
var err error
// Load private key if configured
if pd.config.PrivateKeyPath != "" {
keyring, err = loadPrivateKeyRing(pd.config.PrivateKeyPath, []byte(pd.config.Passphrase))
if err != nil {
return fmt.Errorf("failed to load private key: %w", err)
}
}
// Check if input is armored
buf := make([]byte, 64)
n, err := io.ReadFull(r, buf)
if err != nil && err != io.ErrUnexpectedEOF {
return fmt.Errorf("failed to read input: %w", err)
}
input := io.MultiReader(bytes.NewReader(buf[:n]), r)
// Try to decode armor
if bytes.HasPrefix(bytes.TrimSpace(buf[:n]), []byte("-----BEGIN PGP")) {
armoredBlock, err := armor.Decode(input)
if err != nil {
return fmt.Errorf("failed to decode pgp armor: %w", err)
}
input = armoredBlock.Body
}
// Decrypt the data
md, err := openpgp.ReadMessage(input, keyring, pd.config.passphrasePrompt(), nil)
if err != nil {
return fmt.Errorf("failed to decrypt message: %w", err)
}
// Read all decrypted data
_, err = io.Copy(w, md.UnverifiedBody)
readErr := err
// Close the body (this also verifies the signature if present)
// UnverifiedBody is actually a ReadCloser, but declared as io.Reader
if closer, ok := md.UnverifiedBody.(io.Closer); ok {
closeErr := closer.Close()
if closeErr != nil {
if readErr == nil {
return fmt.Errorf("signature verification failed: %w", closeErr)
}
}
}
if readErr != nil {
return fmt.Errorf("failed to read decrypted data: %w", readErr)
}
// Verify signature if present
if md.IsSigned && md.SignedBy != nil {
// Message was signed and verified
}
return nil
}
// DecryptFile decrypts a PGP encrypted file
func (pd *PGPDecryptor) DecryptFile(inputPath, outputPath string) error {
inputFile, err := os.Open(inputPath)
if err != nil {
return fmt.Errorf("failed to open input file: %w", err)
}
defer inputFile.Close()
outputFile, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer outputFile.Close()
return pd.Decrypt(inputFile, outputFile)
}
// PGPEncryptor encrypts data using PGP
type PGPEncryptor struct {
config *PGPConfig
}
// NewPGPEncryptor creates a new PGP encryptor
func NewPGPEncryptor(cfg *PGPConfig) *PGPEncryptor {
if cfg == nil {
cfg = DefaultPGPConfig()
}
return &PGPEncryptor{config: cfg}
}
// Encrypt encrypts data using PGP
func (pe *PGPEncryptor) Encrypt(r io.Reader, w io.Writer) error {
// Load public key
if pe.config.PublicKeyPath == "" {
return fmt.Errorf("public key path is required for encryption")
}
entity, err := loadPublicKey(pe.config.PublicKeyPath)
if err != nil {
return fmt.Errorf("failed to load public key: %w", err)
}
// Prepare hints
hints := &openpgp.FileHints{
IsBinary: true,
FileName: "", // Don't store filename
}
var out io.WriteCloser
if pe.config.Armor {
out, err = armor.Encode(w, "PGP MESSAGE", nil)
if err != nil {
return fmt.Errorf("failed to create armor encoder: %w", err)
}
} else {
out = nopWriteCloser{w}
}
// Encrypt the data
plaintext, err := openpgp.Encrypt(out, []*openpgp.Entity{entity}, nil, hints, nil)
if err != nil {
if out != nil {
out.Close()
}
return fmt.Errorf("failed to encrypt: %w", err)
}
_, err = io.Copy(plaintext, r)
if err != nil {
plaintext.Close()
if out != nil {
out.Close()
}
return fmt.Errorf("failed to write encrypted data: %w", err)
}
plaintext.Close()
if pe.config.Armor && out != nil {
out.Close()
}
return nil
}
// EncryptFile encrypts a file using PGP
func (pe *PGPEncryptor) EncryptFile(inputPath, outputPath string) error {
inputFile, err := os.Open(inputPath)
if err != nil {
return fmt.Errorf("failed to open input file: %w", err)
}
defer inputFile.Close()
outputFile, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer outputFile.Close()
return pe.Encrypt(inputFile, outputFile)
}
// PGPVerifier verifies PGP signatures
type PGPVerifier struct {
config *PGPConfig
}
// NewPGPVerifier creates a new PGP verifier
func NewPGPVerifier(cfg *PGPConfig) *PGPVerifier {
if cfg == nil {
cfg = DefaultPGPConfig()
}
return &PGPVerifier{config: cfg}
}
// Verify verifies a PGP signature (detached signature)
func (pv *PGPVerifier) Verify(data io.Reader, signature []byte) (bool, string, error) {
// Load keyring
var keyring openpgp.EntityList
var err error
if pv.config.PublicKeyPath != "" {
keyring, err = loadPublicKeyRing(pv.config.PublicKeyPath)
if err != nil {
return false, "", fmt.Errorf("failed to load public key: %w", err)
}
} else if pv.config.KeyRing != nil {
keyring = pv.config.KeyRing
} else {
// Use empty keyring - will try to verify with any available key
keyring = openpgp.EntityList{}
}
// Check if signature is armored
sigReader := bytes.NewReader(signature)
var sigInput io.Reader = sigReader
if bytes.HasPrefix(bytes.TrimSpace(signature), []byte("-----BEGIN PGP")) {
armoredBlock, err := armor.Decode(sigReader)
if err != nil {
return false, "", fmt.Errorf("failed to decode signature armor: %w", err)
}
sigInput = armoredBlock.Body
}
// Verify the signature
signer, err := openpgp.CheckDetachedSignature(keyring, data, sigInput)
if err != nil {
return false, "", fmt.Errorf("signature verification failed: %w", err)
}
if signer == nil {
return false, "", fmt.Errorf("no valid signature found")
}
// Get signer identity
identity := ""
for _, id := range signer.Identities {
if identity == "" {
identity = id.Name
}
if id.SelfSignature != nil {
identity = id.Name
break
}
}
if identity == "" {
identity = "Unknown"
}
return true, identity, nil
}
// VerifyFile verifies PGP signature of a file (detached signature)
func (pv *PGPVerifier) VerifyFile(filePath string, signaturePath string) (bool, string, error) {
// Read signature file
sigData, err := os.ReadFile(signaturePath)
if err != nil {
return false, "", fmt.Errorf("failed to read signature file: %w", err)
}
// Open data file
dataFile, err := os.Open(filePath)
if err != nil {
return false, "", fmt.Errorf("failed to open data file: %w", err)
}
defer dataFile.Close()
return pv.Verify(dataFile, sigData)
}
// VerifyClearsign verifies a clearsigned message
func (pv *PGPVerifier) VerifyClearsign(data []byte) (bool, string, []byte, error) {
// Check if it's a clearsigned message
if !bytes.Contains(data, []byte("-----BEGIN PGP SIGNED MESSAGE-----")) {
return false, "", nil, fmt.Errorf("not a clearsigned message")
}
// Load keyring
var keyring openpgp.EntityList
var err error
if pv.config.PublicKeyPath != "" {
keyring, err = loadPublicKeyRing(pv.config.PublicKeyPath)
if err != nil {
return false, "", nil, fmt.Errorf("failed to load public key: %w", err)
}
} else if pv.config.KeyRing != nil {
keyring = pv.config.KeyRing
}
block, rest := clearsign.Decode(data)
if len(rest) > 0 {
// There's additional data after the clearsigned block
}
signer, err := openpgp.CheckDetachedSignature(keyring, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body)
if err != nil {
return false, "", nil, fmt.Errorf("signature verification failed: %w", err)
}
if signer == nil {
return false, "", nil, fmt.Errorf("no valid signature found")
}
// Get signer identity
identity := "Unknown"
for _, id := range signer.Identities {
if id.SelfSignature != nil {
identity = id.Name
break
}
}
return true, identity, block.Bytes, nil
}
// IsPGPEncrypted determines whether data is PGP encrypted
func IsPGPEncrypted(data []byte) bool {
if len(data) < 10 {
return false
}
// Check for ASCII armor header
header := string(data[:min(len(data), 50)])
if strings.Contains(header, "-----BEGIN PGP MESSAGE-----") ||
strings.Contains(header, "-----BEGIN PGP PUBLIC KEY BLOCK-----") ||
strings.Contains(header, "-----BEGIN PGP PRIVATE KEY BLOCK-----") ||
strings.Contains(header, "-----BEGIN PGP SIGNATURE-----") {
return true
}
// Check for binary PGP packet tags
// PGP binary packets start with specific byte patterns
if len(data) >= 2 {
firstByte := data[0]
// Check for old format packet tag (bits 7-6 = 1, bit 5 = 0)
if (firstByte&0x80) != 0 && (firstByte&0x40) == 0 {
return true
}
// Check for new format packet tag (bits 7-6 = 1, bit 5 = 1)
if (firstByte & 0xC0) == 0xC0 {
return true
}
}
return false
}
// IsPGPSignature determines whether data is a PGP signature
func IsPGPSignature(data []byte) bool {
if len(data) < 10 {
return false
}
header := string(data[:min(len(data), 50)])
return strings.Contains(header, "-----BEGIN PGP SIGNATURE-----")
}
// IsClearsignedMessage determines whether a message is clearsigned
func IsClearsignedMessage(data []byte) bool {
if len(data) < 10 {
return false
}
header := string(data[:min(len(data), 50)])
return strings.Contains(header, "-----BEGIN PGP SIGNED MESSAGE-----")
}
// passphrasePrompt returns a function to prompt for passphrase
func (c *PGPConfig) passphrasePrompt() func(keys []openpgp.Key, symmetric bool) ([]byte, error) {
return func(keys []openpgp.Key, symmetric bool) ([]byte, error) {
if c.Passphrase != "" {
return []byte(c.Passphrase), nil
}
// If no passphrase configured, try empty passphrase
return nil, nil
}
}
// loadPrivateKeyRing loads a keyring from a private key
func loadPrivateKeyRing(path string, passphrase []byte) (openpgp.EntityList, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
keyring, err := openpgp.ReadKeyRing(file)
if err != nil {
return nil, fmt.Errorf("failed to parse keyring: %w", err)
}
// Unlock private keys
for _, entity := range keyring {
if entity.PrivateKey != nil && entity.PrivateKey.Encrypted {
err := entity.PrivateKey.Decrypt(passphrase)
if err != nil {
return nil, fmt.Errorf("failed to decrypt private key: %w", err)
}
}
for _, subkey := range entity.Subkeys {
if subkey.PrivateKey != nil && subkey.PrivateKey.Encrypted {
err := subkey.PrivateKey.Decrypt(passphrase)
if err != nil {
return nil, fmt.Errorf("failed to decrypt subkey: %w", err)
}
}
}
}
return keyring, nil
}
// loadPublicKeyRing loads a keyring from a public key
func loadPublicKeyRing(path string) (openpgp.EntityList, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
keyring, err := openpgp.ReadKeyRing(file)
if err != nil {
return nil, fmt.Errorf("failed to parse keyring: %w", err)
}
return keyring, nil
}
// loadPublicKey loads a public key
func loadPublicKey(path string) (*openpgp.Entity, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
keyring, err := openpgp.ReadKeyRing(file)
if err != nil {
return nil, fmt.Errorf("failed to parse public key: %w", err)
}
if len(keyring) == 0 {
return nil, fmt.Errorf("no keys found in key file")
}
// Return first key
return keyring[0], nil
}
// nopWriteCloser helper for non-closing writer
type nopWriteCloser struct {
io.Writer
}
func (nwc nopWriteCloser) Close() error {
return nil
}
internal/format/format.go
+44
View File
@@ -0,0 +1,44 @@
//go:build linux || freebsd
// +build linux freebsd
package format
import "fmt"
// Bytes formats byte count as human-readable string (DECIMAL units: KB=1000)
func Bytes(b int64) string {
const unit = 1000
if b < 0 {
return "? B"
}
if b < unit {
return fmt.Sprintf("%d B", b)
}
units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
value := float64(b)
exp := 0
for value >= unit && exp < len(units)-1 {
value /= unit
exp++
}
return fmt.Sprintf("%.1f %s", value, units[exp])
}
// Speed formats speed as human-readable string (DECIMAL units)
func Speed(speed float64) string {
const unit = 1000
if speed < 0 {
return "? B/s"
}
if speed < unit {
return fmt.Sprintf("%.1f B/s", speed)
}
units := []string{"B/s", "KB/s", "MB/s", "GB/s", "TB/s", "PB/s"}
value := speed
exp := 0
for value >= unit && exp < len(units)-1 {
value /= unit
exp++
}
return fmt.Sprintf("%.1f %s", value, units[exp])
}
internal/format/format_test.go
+58
View File
@@ -0,0 +1,58 @@
//go:build linux || freebsd
// +build linux freebsd
package format
import (
"testing"
)
func TestBytes(t *testing.T) {
tests := []struct {
input int64
expected string
}{
{0, "0 B"},
{1, "1 B"},
{999, "999 B"},
{1000, "1.0 KB"},
{1500, "1.5 KB"},
{1000000, "1.0 MB"},
{2500000, "2.5 MB"},
{1000000000, "1.0 GB"},
{1000000000000, "1.0 TB"},
{1000000000000000, "1.0 PB"},
{-1, "? B"},
}
for _, tc := range tests {
result := Bytes(tc.input)
if result != tc.expected {
t.Errorf("Bytes(%d) = %q, want %q", tc.input, result, tc.expected)
}
}
}
func TestSpeed(t *testing.T) {
tests := []struct {
input float64
expected string
}{
{0, "0.0 B/s"},
{1, "1.0 B/s"},
{999, "999.0 B/s"},
{1000, "1.0 KB/s"},
{1500, "1.5 KB/s"},
{1000000, "1.0 MB/s"},
{1000000000, "1.0 GB/s"},
{1000000000000, "1.0 TB/s"},
{-1, "? B/s"},
}
for _, tc := range tests {
result := Speed(tc.input)
if result != tc.expected {
t.Errorf("Speed(%f) = %q, want %q", tc.input, result, tc.expected)
}
}
}
internal/hsts/fuzz_test.go
+52
View File
@@ -0,0 +1,52 @@
//go:build linux || freebsd
// +build linux freebsd
package hsts
import (
"encoding/json"
"os"
"testing"
)
// FuzzHSTSParseHeader tests HSTS header parsing with fuzzed input.
func FuzzHSTSParseHeader(f *testing.F) {
f.Add("max-age=31536000")
f.Add("max-age=31536000; includeSubDomains")
f.Add("max-age=0")
f.Add("")
f.Add("invalid")
f.Add("max-age=abc")
f.Add("max-age=31536000; includeSubDomains; preload")
f.Add("MAX-AGE=31536000")
f.Fuzz(func(t *testing.T, header string) {
_, _ = parseHeader(header)
})
}
// FuzzHSTSLoadSave tests loading and saving HSTS cache with fuzzed data.
func FuzzHSTSLoadSave(f *testing.F) {
f.Add([]byte(`{}`))
f.Add([]byte(`{"example.com":{"maxAge":31536000,"includeSubDomains":true,"expires":"2026-01-01T00:00:00Z"}}`))
f.Add([]byte(`{invalid`))
f.Fuzz(func(t *testing.T, data []byte) {
cache := &Cache{
entries: make(map[string]*Entry),
path: t.TempDir() + "/fuzz-hsts.json",
}
// Try loading fuzzed JSON
_ = json.Unmarshal(data, &cache.entries)
// Save to file must not panic
tmp := t.TempDir() + "/fuzz-hsts"
_ = os.WriteFile(tmp, data, 0644)
err := cache.Load(tmp)
if err == nil {
_ = cache.Save()
}
os.Remove(tmp)
})
}
internal/hsts/hsts.go
+226
View File
@@ -0,0 +1,226 @@
//go:build linux || freebsd
// +build linux freebsd
// Package hsts implements HTTP Strict Transport Security (RFC 6797) caching.
package hsts
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/net/publicsuffix"
)
// DefaultCachePath returns the default HSTS cache file path.
func DefaultCachePath() string {
if path := os.Getenv("GOGET_HSTS"); path != "" {
return path
}
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
return filepath.Join(xdg, "goget", "hsts")
}
if home, err := os.UserHomeDir(); err == nil {
return filepath.Join(home, ".config", "goget", "hsts")
}
return ""
}
// Entry represents a single HSTS rule for a host.
type Entry struct {
Host string `json:"host"`
IncludeSubdomains bool `json:"include_subdomains"`
Expires time.Time `json:"expires"`
}
// Cache stores HSTS rules persistently.
type Cache struct {
entries map[string]*Entry
mu sync.RWMutex
path string
}
// NewCache creates an empty HSTS cache.
func NewCache() *Cache {
return &Cache{
entries: make(map[string]*Entry),
}
}
// Load reads HSTS entries from disk.
func (c *Cache) Load(path string) error {
c.mu.Lock()
defer c.mu.Unlock()
c.path = path
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to read hsts cache: %w", err)
}
var entries []*Entry
if err := json.Unmarshal(data, &entries); err != nil {
return fmt.Errorf("failed to parse hsts cache: %w", err)
}
now := time.Now()
for _, e := range entries {
if e.Expires.After(now) {
c.entries[e.Host] = e
}
}
return nil
}
// Save writes HSTS entries to disk.
func (c *Cache) Save() error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.path == "" {
return nil
}
now := time.Now()
var active []*Entry
for _, e := range c.entries {
if e.Expires.After(now) {
active = append(active, e)
}
}
if len(active) == 0 {
_ = os.Remove(c.path)
return nil
}
data, err := json.MarshalIndent(active, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal hsts cache: %w", err)
}
if err := os.MkdirAll(filepath.Dir(c.path), 0755); err != nil {
return fmt.Errorf("failed to create hsts directory: %w", err)
}
if err := os.WriteFile(c.path, data, 0644); err != nil {
return fmt.Errorf("failed to write hsts cache: %w", err)
}
return nil
}
// Apply upgrades http:// to https:// if the host is in the HSTS cache.
// Returns true if the URL was modified.
func (c *Cache) Apply(u *url.URL) bool {
if u.Scheme != "http" {
return false
}
c.mu.RLock()
defer c.mu.RUnlock()
host := strings.ToLower(u.Hostname())
if c.matchLocked(host) != nil {
u.Scheme = "https"
return true
}
return false
}
// Update parses Strict-Transport-Security from an HTTPS response and stores it.
// Per RFC 6797, HSTS headers on non-HTTPS responses are ignored.
func (c *Cache) Update(resp *http.Response) {
if resp == nil || resp.Request == nil || resp.Request.URL.Scheme != "https" {
return
}
header := resp.Header.Get("Strict-Transport-Security")
if header == "" {
return
}
maxAge, includeSubdomains := parseHeader(header)
if maxAge <= 0 {
return
}
host := strings.ToLower(resp.Request.URL.Hostname())
entry := &Entry{
Host: host,
IncludeSubdomains: includeSubdomains,
Expires: time.Now().Add(time.Duration(maxAge) * time.Second),
}
c.mu.Lock()
c.entries[host] = entry
c.mu.Unlock()
}
// matchLocked finds an entry for the given host (must hold read lock).
// When includeSubdomains is set on an entry, its rule applies to all
// subdomains, but only down to the registrable domain boundary
// (publicsuffix.EffectiveTLDPlusOne). Walking past the registrable
// domain would compare against a public suffix (e.g. "co.uk"), which
// RFC 6797 does not authorise and would let a HSTS entry mistakenly
// stored at the suffix level cover unrelated registrable domains.
func (c *Cache) matchLocked(host string) *Entry {
if e, ok := c.entries[host]; ok {
if e.Expires.After(time.Now()) {
return e
}
}
// Walk up parent domains, stopping once we step outside the
// registrable domain (eTLD+1). We check the public-suffix
// boundary *before* entries: a HSTS entry stored at a public
// suffix (e.g. "co.uk") must not cover unrelated registrable
// domains like "example.co.uk".
for {
dot := strings.Index(host, ".")
if dot < 0 {
break
}
host = host[dot+1:]
// If the parent is a public suffix or a single label
// (EffectiveTLDPlusOne returns an error), there is no further
// in-scope parent to consider.
if _, err := publicsuffix.EffectiveTLDPlusOne(host); err != nil {
break
}
if e, ok := c.entries[host]; ok && e.IncludeSubdomains && e.Expires.After(time.Now()) {
return e
}
}
return nil
}
// parseHeader parses the Strict-Transport-Security header value.
// Returns max-age in seconds and whether includeSubDomains is set.
func parseHeader(value string) (int64, bool) {
var maxAge int64 = -1
var includeSubdomains bool
for _, part := range strings.Split(value, ";") {
part = strings.TrimSpace(part)
if strings.HasPrefix(strings.ToLower(part), "max-age=") {
v := strings.TrimPrefix(part, "max-age=")
v = strings.TrimPrefix(v, "max-Age=")
v = strings.TrimPrefix(v, "MAX-AGE=")
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
maxAge = n
}
} else if strings.EqualFold(part, "includesubdomains") {
includeSubdomains = true
}
}
return maxAge, includeSubdomains
}
internal/hsts/hsts_test.go
+376
View File
@@ -0,0 +1,376 @@
//go:build linux || freebsd
// +build linux freebsd
package hsts
import (
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestParseHeader(t *testing.T) {
tests := []struct {
input string
wantMaxAge int64
wantIncludeSubdomains bool
}{
{"max-age=31536000", 31536000, false},
{"max-age=0", 0, false},
{"max-age=31536000; includeSubDomains", 31536000, true},
{"includeSubDomains; max-age=31536000", 31536000, true},
{"max-age=60; includesubdomains", 60, true},
{"MAX-AGE=120", 120, false},
{"", -1, false},
{"max-age=bad", -1, false},
{"preload; max-age=100", 100, false},
}
for _, tt := range tests {
age, inc := parseHeader(tt.input)
if age != tt.wantMaxAge || inc != tt.wantIncludeSubdomains {
t.Errorf("parseHeader(%q) = (%d, %v), want (%d, %v)",
tt.input, age, inc, tt.wantMaxAge, tt.wantIncludeSubdomains)
}
}
}
func TestCacheApply(t *testing.T) {
c := NewCache()
c.entries["example.com"] = &Entry{
Host: "example.com",
IncludeSubdomains: false,
Expires: time.Now().Add(time.Hour),
}
c.entries["sub.example.com"] = &Entry{
Host: "sub.example.com",
IncludeSubdomains: true,
Expires: time.Now().Add(time.Hour),
}
tests := []struct {
urlStr string
modified bool
wantScheme string
}{
{"http://example.com/path", true, "https"},
{"https://example.com/path", false, "https"},
{"http://other.com/path", false, "http"},
{"http://sub.example.com/path", true, "https"},
{"http://deep.sub.example.com/path", true, "https"},
{"ftp://example.com/path", false, "ftp"},
}
for _, tt := range tests {
u, _ := url.Parse(tt.urlStr)
got := c.Apply(u)
if got != tt.modified {
t.Errorf("Apply(%q) modified=%v, want %v", tt.urlStr, got, tt.modified)
}
if u.Scheme != tt.wantScheme {
t.Errorf("Apply(%q) scheme=%q, want %q", tt.urlStr, u.Scheme, tt.wantScheme)
}
}
}
func TestCacheUpdate(t *testing.T) {
c := NewCache()
u, _ := url.Parse("https://example.com/")
resp := &http.Response{
Header: http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
Request: &http.Request{URL: u},
}
c.Update(resp)
if len(c.entries) != 1 {
t.Fatalf("expected 1 entry, got %d", len(c.entries))
}
e := c.entries["example.com"]
if e == nil {
t.Fatal("expected entry for example.com")
}
if e.IncludeSubdomains {
t.Error("expected IncludeSubdomains=false")
}
if e.Expires.Before(time.Now().Add(59*time.Minute)) || e.Expires.After(time.Now().Add(61*time.Minute)) {
t.Errorf("unexpected expires: %v", e.Expires)
}
}
func TestCacheUpdateIgnoresHTTP(t *testing.T) {
c := NewCache()
u, _ := url.Parse("http://example.com/")
resp := &http.Response{
Header: http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
Request: &http.Request{URL: u},
}
c.Update(resp)
if len(c.entries) != 0 {
t.Errorf("expected 0 entries for HTTP response, got %d", len(c.entries))
}
}
func TestCacheLoadSave(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "hsts")
c := NewCache()
c.entries["example.com"] = &Entry{
Host: "example.com",
Expires: time.Now().Add(time.Hour),
}
c.entries["expired.com"] = &Entry{
Host: "expired.com",
Expires: time.Now().Add(-time.Hour),
}
c.path = path
if err := c.Save(); err != nil {
t.Fatalf("save failed: %v", err)
}
c2 := NewCache()
if err := c2.Load(path); err != nil {
t.Fatalf("load failed: %v", err)
}
if len(c2.entries) != 1 {
t.Errorf("expected 1 active entry after load, got %d", len(c2.entries))
}
if c2.entries["example.com"] == nil {
t.Error("expected example.com entry")
}
if c2.entries["expired.com"] != nil {
t.Error("did not expect expired.com entry")
}
}
func TestCacheSaveRemovesEmpty(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "hsts")
// Create a dummy file
if err := os.WriteFile(path, []byte("[]"), 0644); err != nil {
t.Fatal(err)
}
c := NewCache()
c.path = path
if err := c.Save(); err != nil {
t.Fatalf("save failed: %v", err)
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Error("expected cache file to be removed when empty")
}
}
func TestCacheSubdomainMatch(t *testing.T) {
c := NewCache()
c.entries["example.com"] = &Entry{
Host: "example.com",
IncludeSubdomains: true,
Expires: time.Now().Add(time.Hour),
}
u, _ := url.Parse("http://a.b.c.example.com/")
if !c.Apply(u) {
t.Error("expected deep subdomain to match")
}
if u.Scheme != "https" {
t.Errorf("expected https, got %s", u.Scheme)
}
}
func TestCacheExpiredEntry(t *testing.T) {
c := NewCache()
c.entries["example.com"] = &Entry{
Host: "example.com",
Expires: time.Now().Add(-time.Hour),
}
u, _ := url.Parse("http://example.com/")
if c.Apply(u) {
t.Error("expected expired entry to not match")
}
}
func TestCacheUpdateOverwrite(t *testing.T) {
c := NewCache()
u, _ := url.Parse("https://example.com/")
resp1 := &http.Response{
Header: http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
Request: &http.Request{URL: u},
}
c.Update(resp1)
resp2 := &http.Response{
Header: http.Header{"Strict-Transport-Security": []string{"max-age=7200; includeSubDomains"}},
Request: &http.Request{URL: u},
}
c.Update(resp2)
e := c.entries["example.com"]
if e == nil {
t.Fatal("expected entry")
}
if !e.IncludeSubdomains {
t.Error("expected IncludeSubdomains=true after overwrite")
}
if e.Expires.Before(time.Now().Add(119*time.Minute)) || e.Expires.After(time.Now().Add(121*time.Minute)) {
t.Errorf("unexpected expires after overwrite: %v", e.Expires)
}
}
func TestCacheConcurrent(t *testing.T) {
c := NewCache()
u, _ := url.Parse("https://example.com/")
// Concurrent reads and writes should not race
for i := 0; i < 100; i++ {
go func() {
url, _ := url.Parse("http://example.com/")
c.Apply(url)
}()
go func() {
resp := &http.Response{
Header: http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
Request: &http.Request{URL: u},
}
c.Update(resp)
}()
}
time.Sleep(50 * time.Millisecond)
}
func TestDefaultPath(t *testing.T) {
home := t.TempDir()
os.Setenv("HOME", home)
os.Unsetenv("XDG_CONFIG_HOME")
path := DefaultCachePath()
wantSuffix := filepath.Join(".config", "goget", "hsts")
if !strings.HasSuffix(path, wantSuffix) {
t.Errorf("default path %q does not end with %q", path, wantSuffix)
}
}
// TestMatchPublicSuffixBoundary is a regression guard for the BACKLOG
// entry "HSTS matches beyond public suffix boundary". The previous
// implementation walked parent domains one dot at a time without
// stopping at the registrable domain (eTLD+1), so a HSTS entry
// accidentally stored at a public suffix (e.g. "co.uk") would be
// applied to the unrelated registrable domain "example.co.uk",
// violating RFC 6797.
func TestMatchPublicSuffixBoundary(t *testing.T) {
// addEntry is a local test helper that stores a HSTS entry
// directly in the private map, bypassing the HTTPS-only Update path.
addEntry := func(c *Cache, host string, includeSubdomains bool) {
c.entries[host] = &Entry{
Host: host,
IncludeSubdomains: includeSubdomains,
Expires: time.Now().Add(time.Hour),
}
}
tests := []struct {
name string
policyHost string
policySubdomains bool
requestHost string
expectUpgrade bool
}{
// Legitimate: HSTS on the registrable domain applies to its
// subdomains.
{
name: "subdomain of eTLD+1 is upgraded",
policyHost: "example.co.uk",
policySubdomains: true,
requestHost: "www.example.co.uk",
expectUpgrade: true,
},
{
name: "registrable domain itself is upgraded",
policyHost: "example.co.uk",
policySubdomains: true,
requestHost: "example.co.uk",
expectUpgrade: true,
},
// Pathological but possible: an HSTS entry was stored on a
// public suffix. The matchLocked walk must NOT let it cover
// the unrelated registrable domain.
{
name: "public suffix entry does not upgrade unrelated eTLD+1",
policyHost: "co.uk",
policySubdomains: true,
requestHost: "example.co.uk",
expectUpgrade: false,
},
{
name: "public suffix entry does not upgrade further sibling",
policyHost: "uk",
policySubdomains: true,
requestHost: "example.co.uk",
expectUpgrade: false,
},
// Same idea at the .com boundary.
{
name: "TLD entry does not upgrade unrelated .com domain",
policyHost: "com",
policySubdomains: true,
requestHost: "example.com",
expectUpgrade: false,
},
{
name: "TLD entry does not upgrade further sibling",
policyHost: "com",
policySubdomains: true,
requestHost: "anotherexample.com",
expectUpgrade: false,
},
// Subdomains are not set — no upgrade even on the registrable
// domain.
{
name: "no upgrade without includeSubdomains",
policyHost: "example.com",
policySubdomains: false,
requestHost: "www.example.com",
expectUpgrade: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cache := NewCache()
addEntry(cache, tc.policyHost, tc.policySubdomains)
u, err := url.Parse("http://" + tc.requestHost + "/")
if err != nil {
t.Fatalf("parse: %v", err)
}
upgraded := cache.Apply(u)
if upgraded != tc.expectUpgrade {
t.Errorf("Apply(%q) returned %v, want %v (final scheme=%q)",
tc.requestHost, upgraded, tc.expectUpgrade, u.Scheme)
}
if tc.expectUpgrade && u.Scheme != "https" {
t.Errorf("expected https after upgrade, got %q", u.Scheme)
}
if !tc.expectUpgrade && u.Scheme != "http" {
t.Errorf("expected http (no upgrade), got %q", u.Scheme)
}
})
}
}
internal/linkrewrite/rewriter.go
+143
View File
@@ -0,0 +1,143 @@
//go:build linux || freebsd
// +build linux freebsd
// Package linkrewrite rewrites HTML links for local offline viewing.
package linkrewrite
import (
"net/url"
"path/filepath"
"strings"
"sync"
"golang.org/x/net/html"
)
// Rewriter converts absolute URLs in HTML to local relative paths.
type Rewriter struct {
baseURL *url.URL
outputDir string
visited map[string]string
visitedMu sync.RWMutex
}
// New creates a link rewriter for a mirror or recursive download.
func New(baseURL *url.URL, outputDir string) *Rewriter {
return &Rewriter{
baseURL: baseURL,
outputDir: outputDir,
visited: make(map[string]string),
}
}
// Register records a downloaded URL and its local path.
func (lr *Rewriter) Register(u *url.URL, localPath string) {
lr.visitedMu.Lock()
defer lr.visitedMu.Unlock()
lr.visited[u.String()] = localPath
}
// RewriteLinks converts URLs in an HTML document to local paths.
func (lr *Rewriter) RewriteLinks(doc *html.Node, filePath string) int {
converted := 0
dir := filepath.Dir(filePath)
var traverse func(*html.Node)
traverse = func(n *html.Node) {
if n.Type == html.ElementNode {
switch n.Data {
case "a":
converted += lr.rewriteAttr(n, "href", dir)
case "img", "script", "iframe", "video", "audio", "source", "track", "embed", "object":
converted += lr.rewriteAttr(n, "src", dir)
case "link":
converted += lr.rewriteAttr(n, "href", dir)
case "form":
converted += lr.rewriteAttr(n, "action", dir)
case "area":
converted += lr.rewriteAttr(n, "href", dir)
case "meta":
for _, attr := range n.Attr {
if attr.Key == "content" && strings.Contains(attr.Val, "url=") {
converted += lr.rewriteMetaRefresh(n, dir)
}
if attr.Key == "property" && strings.Contains(attr.Val, "image") {
converted += lr.rewriteAttr(n, "content", dir)
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c)
}
}
traverse(doc)
return converted
}
func (lr *Rewriter) rewriteMetaRefresh(n *html.Node, baseDir string) int {
for i, attr := range n.Attr {
if attr.Key == "content" {
content := attr.Val
idx := strings.Index(strings.ToLower(content), "url=")
if idx == -1 {
return 0
}
urlStr := strings.TrimSpace(content[idx+4:])
parsed, err := url.Parse(urlStr)
if err != nil {
return 0
}
resolved := lr.baseURL.ResolveReference(parsed)
lr.visitedMu.RLock()
localPath, exists := lr.visited[resolved.String()]
lr.visitedMu.RUnlock()
if exists {
relPath, err := filepath.Rel(baseDir, localPath)
if err != nil {
relPath = localPath
}
n.Attr[i].Val = content[:idx+4] + filepath.ToSlash(relPath)
return 1
}
return 0
}
}
return 0
}
func (lr *Rewriter) rewriteAttr(n *html.Node, attrName, baseDir string) int {
for i, attr := range n.Attr {
if attr.Key != attrName {
continue
}
value := attr.Val
if strings.HasPrefix(value, "#") ||
strings.HasPrefix(value, "javascript:") ||
strings.HasPrefix(value, "mailto:") ||
strings.HasPrefix(value, "tel:") ||
strings.HasPrefix(value, "data:") ||
strings.HasPrefix(value, "about:") {
return 0
}
parsed, err := url.Parse(value)
if err != nil {
return 0
}
resolved := lr.baseURL.ResolveReference(parsed)
lr.visitedMu.RLock()
localPath, exists := lr.visited[resolved.String()]
lr.visitedMu.RUnlock()
if !exists {
return 0
}
relPath, err := filepath.Rel(baseDir, localPath)
if err != nil {
relPath = localPath
}
n.Attr[i].Val = filepath.ToSlash(relPath)
return 1
}
return 0
}
internal/linkrewrite/rewriter_test.go
+448
View File
@@ -0,0 +1,448 @@
//go:build linux || freebsd
// +build linux freebsd
package linkrewrite
import (
"net/url"
"path/filepath"
"strings"
"sync"
"testing"
"golang.org/x/net/html"
)
func parseHTML(t *testing.T, htmlStr string) *html.Node {
t.Helper()
doc, err := html.Parse(strings.NewReader(htmlStr))
if err != nil {
t.Fatalf("failed to parse html: %v", err)
}
return doc
}
func getAttr(t *testing.T, n *html.Node, tag, attrName string) string {
t.Helper()
var find func(*html.Node)
var val string
find = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == tag {
for _, a := range n.Attr {
if a.Key == attrName {
val = a.Val
return
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
find(c)
}
}
find(n)
return val
}
func TestNew(t *testing.T) {
base, _ := url.Parse("https://example.com/")
outDir := "/tmp/mirror"
rw := New(base, outDir)
if rw.baseURL.String() != "https://example.com/" {
t.Errorf("expected baseURL https://example.com/, got %s", rw.baseURL.String())
}
if rw.outputDir != outDir {
t.Errorf("expected outputDir /tmp/mirror, got %s", rw.outputDir)
}
if rw.visited == nil {
t.Error("expected visited map to be initialized")
}
}
func TestRegister(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
pageURL, _ := url.Parse("https://example.com/about.html")
rw.Register(pageURL, "/out/about.html")
rw.visitedMu.RLock()
got := rw.visited["https://example.com/about.html"]
rw.visitedMu.RUnlock()
if got != "/out/about.html" {
t.Errorf("expected /out/about.html, got %s", got)
}
}
func TestRewriteLinksNoRegistered(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
doc := parseHTML(t, `<html><body><a href="/page">link</a></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 0 {
t.Errorf("expected 0 conversions, got %d", converted)
}
// href should remain unchanged
got := getAttr(t, doc, "a", "href")
if got != "/page" {
t.Errorf("expected /page, got %s", got)
}
}
func TestRewriteLinksAnchorHref(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
pageURL, _ := url.Parse("https://example.com/page.html")
rw.Register(pageURL, "/out/page.html")
doc := parseHTML(t, `<html><body><a href="/page.html">link</a></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "a", "href")
if got != "page.html" {
t.Errorf("expected relative path, got %s", got)
}
}
func TestRewriteLinksImageSrc(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
imgURL, _ := url.Parse("https://example.com/images/logo.png")
rw.Register(imgURL, "/out/images/logo.png")
doc := parseHTML(t, `<html><body><img src="/images/logo.png"></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "img", "src")
if got != "images/logo.png" {
t.Errorf("expected images/logo.png, got %s", got)
}
}
func TestRewriteLinksFormAction(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
formURL, _ := url.Parse("https://example.com/submit")
rw.Register(formURL, "/out/submit.html")
doc := parseHTML(t, `<html><body><form action="/submit"></form></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "form", "action")
if got != "submit.html" {
t.Errorf("expected submit.html, got %s", got)
}
}
func TestRewriteLinksMediaElements(t *testing.T) {
tests := []struct{ tag, urlStr, localPath string }{
{"script", "https://example.com/app.js", "/out/js/app.js"},
{"video", "https://example.com/video.mp4", "/out/video.mp4"},
{"audio", "https://example.com/sound.mp3", "/out/sound.mp3"},
{"source", "https://example.com/video.webm", "/out/video.webm"},
}
for _, tt := range tests {
t.Run(tt.tag, func(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
u, _ := url.Parse(tt.urlStr)
rw.Register(u, tt.localPath)
htmlStr := `<html><body><` + tt.tag + ` src="` + tt.urlStr + `"></` + tt.tag + `></body></html>`
doc := parseHTML(t, htmlStr)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
})
}
}
func TestRewriteLinksSkipsProtocols(t *testing.T) {
base, _ := url.Parse("https://example.com")
prefixes := []string{"#section", "javascript:void(0)", "mailto:a@b.com", "tel:+123", "data:text/plain", "about:blank"}
for _, prefix := range prefixes {
t.Run(prefix, func(t *testing.T) {
rw := New(base, "/out")
doc := parseHTML(t, `<html><body><a href="`+prefix+`">link</a></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 0 {
t.Errorf("expected 0 conversions for %s, got %d", prefix, converted)
}
})
}
}
func TestRewriteLinksRelativeResolution(t *testing.T) {
base, _ := url.Parse("https://example.com/subdir/")
rw := New(base, "/out")
pageURL, _ := url.Parse("https://example.com/subdir/page.html")
rw.Register(pageURL, "/out/subdir/page.html")
// Relative URL should resolve against base
doc := parseHTML(t, `<html><body><a href="page.html">link</a></body></html>`)
converted := rw.RewriteLinks(doc, "/out/subdir/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "a", "href")
if got != "page.html" {
t.Errorf("expected page.html, got %s", got)
}
}
func TestRewriteLinksNestedRelative(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
imgURL, _ := url.Parse("https://example.com/assets/img/photo.jpg")
rw.Register(imgURL, "/out/assets/img/photo.jpg")
// page.html is in /out/blog/ — image should resolve to ../assets/img/photo.jpg
doc := parseHTML(t, `<html><body><img src="/assets/img/photo.jpg"></body></html>`)
converted := rw.RewriteLinks(doc, "/out/blog/page.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "img", "src")
if got != "../assets/img/photo.jpg" {
t.Errorf("expected ../assets/img/photo.jpg, got %s", got)
}
}
func TestRewriteLinksMetaRefresh(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
targetURL, _ := url.Parse("https://example.com/new-page.html")
rw.Register(targetURL, "/out/new-page.html")
doc := parseHTML(t, `<html><head><meta http-equiv="refresh" content="5; url=/new-page.html"></head></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "meta", "content")
if !strings.Contains(got, "url=new-page.html") {
t.Errorf("expected content to contain url=new-page.html, got %s", got)
}
}
func TestRewriteLinksOGImage(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
imgURL, _ := url.Parse("https://example.com/og-image.jpg")
rw.Register(imgURL, "/out/og-image.jpg")
doc := parseHTML(t, `<html><head><meta property="og:image" content="/og-image.jpg"></head></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "meta", "content")
if got != "og-image.jpg" {
t.Errorf("expected og-image.jpg, got %s", got)
}
}
func TestRewriteLinksAreaTag(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
mapURL, _ := url.Parse("https://example.com/section.html")
rw.Register(mapURL, "/out/section.html")
doc := parseHTML(t, `<html><body><map><area href="/section.html"></map></body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
}
func TestRewriteLinksLinkTag(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
cssURL, _ := url.Parse("https://example.com/style.css")
rw.Register(cssURL, "/out/style.css")
doc := parseHTML(t, `<html><head><link rel="stylesheet" href="/style.css"></head></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "link", "href")
if got != "style.css" {
t.Errorf("expected style.css, got %s", got)
}
}
func TestRewriteLinksMultipleConversions(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
urls := []struct{ abs, local string }{
{"https://example.com/a.html", "/out/a.html"},
{"https://example.com/img1.png", "/out/img1.png"},
{"https://example.com/img2.png", "/out/img2.png"},
}
for _, u := range urls {
pu, _ := url.Parse(u.abs)
rw.Register(pu, u.local)
}
doc := parseHTML(t, `<html><body>
<a href="/a.html">link</a>
<img src="/img1.png">
<img src="/img2.png">
</body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 3 {
t.Errorf("expected 3 conversions, got %d", converted)
}
}
func TestRewriteLinksOnlyRegistered(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
// Only register one of two URLs
regURL, _ := url.Parse("https://example.com/registered.html")
rw.Register(regURL, "/out/registered.html")
doc := parseHTML(t, `<html><body>
<a href="/registered.html">yes</a>
<a href="/unregistered.html">no</a>
</body></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
}
func TestRewriteLinksEmptyDocument(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
doc := parseHTML(t, ``)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 0 {
t.Errorf("expected 0 conversions for empty doc, got %d", converted)
}
}
func TestRewriteLinksNoHtmlElement(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
doc := parseHTML(t, `just text`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 0 {
t.Errorf("expected 0 conversions for text-only doc, got %d", converted)
}
}
func TestRewriteLinksForwardSlashConversion(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
pageURL, _ := url.Parse("https://example.com/dir/page.html")
rw.Register(pageURL, filepath.FromSlash("/out/dir/page.html"))
doc := parseHTML(t, `<html><body><a href="/dir/page.html">link</a></body></html>`)
converted := rw.RewriteLinks(doc, filepath.FromSlash("/out/index.html"))
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "a", "href")
// Should use forward slashes
if !strings.Contains(got, "/") && strings.Contains(got, "\\") {
t.Errorf("expected forward-slash path, got %s", got)
}
}
func TestRegisterConcurrent(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func(n int) {
defer wg.Done()
u, _ := url.Parse("https://example.com/page" + string(rune('0'+n%10)) + ".html")
rw.Register(u, "/out/p.html")
}(i)
}
wg.Wait()
rw.visitedMu.RLock()
count := len(rw.visited)
rw.visitedMu.RUnlock()
if count == 0 {
t.Error("expected visited map to have entries after concurrent registration")
}
}
func TestRewriteLinksAbsoluteURLWithPath(t *testing.T) {
base, _ := url.Parse("https://example.com/blog/")
rw := New(base, "/out")
// Register with absolute URL
pageURL, _ := url.Parse("https://example.com/blog/post.html")
rw.Register(pageURL, "/out/blog/post.html")
// Absolute href in HTML
doc := parseHTML(t, `<html><body><a href="https://example.com/blog/post.html">link</a></body></html>`)
converted := rw.RewriteLinks(doc, "/out/blog/index.html")
if converted != 1 {
t.Errorf("expected 1 conversion, got %d", converted)
}
got := getAttr(t, doc, "a", "href")
if got != "post.html" {
t.Errorf("expected post.html, got %s", got)
}
}
func TestRewriteLinksMetaRefreshNoRegistered(t *testing.T) {
base, _ := url.Parse("https://example.com")
rw := New(base, "/out")
doc := parseHTML(t, `<html><head><meta http-equiv="refresh" content="5; url=/nonexistent"></head></html>`)
converted := rw.RewriteLinks(doc, "/out/index.html")
if converted != 0 {
t.Errorf("expected 0 conversions, got %d", converted)
}
}
internal/log/logger.go
+229
View File
@@ -0,0 +1,229 @@
//go:build linux || freebsd
// +build linux freebsd
// Package log provides a structured logger with text and JSON output modes.
// It replaces ad-hoc fmt.Fprintf and cli.Print* calls across the codebase.
package log
import (
"encoding/json"
"fmt"
"io"
"os"
"sync"
)
// Level represents the severity of a log message.
type Level int
const (
ErrorLevel Level = iota
WarnLevel
InfoLevel
VerboseLevel
DebugLevel
)
func (l Level) String() string {
switch l {
case ErrorLevel:
return "error"
case WarnLevel:
return "warn"
case InfoLevel:
return "info"
case VerboseLevel:
return "verbose"
case DebugLevel:
return "debug"
default:
return "unknown"
}
}
// Logger writes structured log messages.
type Logger struct {
mu sync.Mutex
w io.Writer
level Level
color bool
json bool
}
// Option configures a Logger.
type Option func(*Logger)
// WithLevel sets the minimum log level. Messages below this level are
// silently discarded.
func WithLevel(l Level) Option {
return func(log *Logger) { log.level = l }
}
// WithColor enables or disables ANSI color output in text mode.
func WithColor(c bool) Option {
return func(log *Logger) { log.color = c }
}
// WithJSON enables JSON output mode. When enabled, each message is written as
// a JSON object on a single line.
func WithJSON(j bool) Option {
return func(log *Logger) { log.json = j }
}
// New creates a new Logger that writes to w.
// Defaults: InfoLevel, color enabled, text mode.
func New(w io.Writer, opts ...Option) *Logger {
l := &Logger{w: w, level: InfoLevel, color: true}
for _, o := range opts {
o(l)
}
return l
}
// Error logs a message at error level.
func (l *Logger) Error(msg string) { l.log(ErrorLevel, msg, nil) }
// Errorf logs a formatted message at error level.
func (l *Logger) Errorf(format string, args ...interface{}) {
l.log(ErrorLevel, fmt.Sprintf(format, args...), nil)
}
// Warn logs a message at warning level.
func (l *Logger) Warn(msg string) { l.log(WarnLevel, msg, nil) }
// Warnf logs a formatted message at warning level.
func (l *Logger) Warnf(format string, args ...interface{}) {
l.log(WarnLevel, fmt.Sprintf(format, args...), nil)
}
// Info logs a message at info level.
func (l *Logger) Info(msg string) { l.log(InfoLevel, msg, nil) }
// Infof logs a formatted message at info level.
func (l *Logger) Infof(format string, args ...interface{}) {
l.log(InfoLevel, fmt.Sprintf(format, args...), nil)
}
// Verbose logs a message at verbose level. Only shown when the logger is
// configured at VerboseLevel or below.
func (l *Logger) Verbose(msg string) { l.log(VerboseLevel, msg, nil) }
// Verbosef logs a formatted message at verbose level.
func (l *Logger) Verbosef(format string, args ...interface{}) {
l.log(VerboseLevel, fmt.Sprintf(format, args...), nil)
}
// Debug logs a message at debug level. Only shown when the logger is
// configured at DebugLevel.
func (l *Logger) Debug(msg string) { l.log(DebugLevel, msg, nil) }
// Debugf logs a formatted message at debug level.
func (l *Logger) Debugf(format string, args ...interface{}) {
l.log(DebugLevel, fmt.Sprintf(format, args...), nil)
}
// Success logs a success message at info level with a checkmark prefix.
// In JSON mode, it adds a "status":"success" field.
func (l *Logger) Success(msg string) {
l.log(InfoLevel, msg, map[string]interface{}{"status": "success"})
}
// Successf logs a formatted success message.
func (l *Logger) Successf(format string, args ...interface{}) {
l.Success(fmt.Sprintf(format, args...))
}
func (l *Logger) log(level Level, msg string, fields map[string]interface{}) {
if level > l.level {
return
}
l.mu.Lock()
defer l.mu.Unlock()
if l.json {
l.logJSON(level, msg, fields)
} else {
l.logText(level, msg)
}
}
func (l *Logger) logText(level Level, msg string) {
var prefix, suffix string
if l.color {
suffix = colorReset
switch level {
case ErrorLevel:
prefix = colorRed + colorBold
case WarnLevel:
prefix = colorYellow + colorBold
case InfoLevel:
prefix = colorCyan + colorBold
}
}
icon := levelIcon(level)
if prefix != "" {
fmt.Fprintf(l.w, "%s%s%s %s\n", prefix, icon, suffix, msg)
} else {
fmt.Fprintf(l.w, "%s %s\n", icon, msg)
}
}
type jsonEntry struct {
Level string `json:"level"`
Msg string `json:"msg"`
Fields map[string]interface{} `json:"fields,omitempty"`
}
func (l *Logger) logJSON(level Level, msg string, fields map[string]interface{}) {
entry := jsonEntry{
Level: level.String(),
Msg: msg,
}
if fields != nil {
entry.Fields = fields
}
// Best-effort encoding; errors are silently dropped to avoid log loops.
data, _ := json.Marshal(entry)
fmt.Fprintln(l.w, string(data))
}
func levelIcon(level Level) string {
switch level {
case ErrorLevel:
return "✗ Error:"
case WarnLevel:
return "⚠ Warning:"
case InfoLevel, VerboseLevel, DebugLevel:
return "ℹ"
default:
return "•"
}
}
// ANSI escape codes.
const (
colorReset = "\033[0m"
colorRed = "\033[31m"
colorYellow = "\033[33m"
colorCyan = "\033[36m"
colorBold = "\033[1m"
)
// DefaultLogger is the global logger used by package-level convenience
// functions. It writes to stderr at InfoLevel in text mode.
var DefaultLogger = New(os.Stderr)
// Package-level convenience functions that delegate to DefaultLogger.
func Error(msg string) { DefaultLogger.Error(msg) }
func Errorf(f string, a ...interface{}) { DefaultLogger.Errorf(f, a...) }
func Warn(msg string) { DefaultLogger.Warn(msg) }
func Warnf(f string, a ...interface{}) { DefaultLogger.Warnf(f, a...) }
func Info(msg string) { DefaultLogger.Info(msg) }
func Infof(f string, a ...interface{}) { DefaultLogger.Infof(f, a...) }
func Verbose(msg string) { DefaultLogger.Verbose(msg) }
func Verbosef(f string, a ...interface{}) { DefaultLogger.Verbosef(f, a...) }
func Debug(msg string) { DefaultLogger.Debug(msg) }
func Debugf(f string, a ...interface{}) { DefaultLogger.Debugf(f, a...) }
func Success(msg string) { DefaultLogger.Success(msg) }
func Successf(f string, a ...interface{}) { DefaultLogger.Successf(f, a...) }
internal/metalink/downloader.go
+703
View File
@@ -0,0 +1,703 @@
//go:build linux || freebsd
// +build linux freebsd
package metalink
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/crypto"
format "codeberg.org/petrbalvin/goget/internal/format"
)
// MultiSourceDownloader downloads from multiple sources simultaneously
type MultiSourceDownloader struct {
config *DownloaderConfig
}
// DownloaderConfig configuration for multi-source download
type DownloaderConfig struct {
// Maximum number of parallel sources
MaxSources int
// Timeout for each source
Timeout time.Duration
// Minimum speed (bytes/s) before switching to another source
MinSpeed int64
// Buffer size for reading
BufferSize int
// Maximum bytes to accept from a single source (0 = no limit)
MaxDownloadSize int64
// Verbose mode
Verbose bool
// Callback for progress
ProgressCallback func(current, total int64, speed float64)
}
// DefaultDownloaderConfig returns default configuration
func DefaultDownloaderConfig() *DownloaderConfig {
return &DownloaderConfig{
MaxSources: 4,
Timeout: 30 * time.Second,
MinSpeed: 1024, // 1 KB/s
BufferSize: 32 * 1024,
MaxDownloadSize: 1 << 30, // 1 GiB safety cap per source
Verbose: false,
}
}
// NewMultiSourceDownloader creates new downloader
func NewMultiSourceDownloader(cfg *DownloaderConfig) *MultiSourceDownloader {
if cfg == nil {
cfg = DefaultDownloaderConfig()
}
return &MultiSourceDownloader{
config: cfg,
}
}
// DownloadResult represents the result of a multi-source download
type DownloadResult struct {
BytesDownloaded int64
TotalSize int64
SourcesUsed int
Duration time.Duration
Speed float64
OutputPath string
Verified bool
Hash string
}
// Download downloads a file from multiple sources simultaneously
func (d *MultiSourceDownloader) Download(ctx context.Context, file *File, outputPath string) (*DownloadResult, error) {
startTime := time.Now()
if len(file.URLs) == 0 {
return nil, fmt.Errorf("no URLs available for download")
}
// Prepare output file
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
return nil, fmt.Errorf("failed to create output directory: %w", err)
}
outFile, err := os.Create(outputPath)
if err != nil {
return nil, fmt.Errorf("failed to create output file: %w", err)
}
defer outFile.Close()
// Get sorted URLs by priority
urls := file.GetURLs()
if len(urls) > d.config.MaxSources {
urls = urls[:d.config.MaxSources]
}
// Create channels for coordination
type chunkResult struct {
written int64
source string
}
chunkChan := make(chan chunkResult, len(urls))
errChan := make(chan error, len(urls))
doneChan := make(chan struct{})
var downloadedBytes int64
var sourcesUsed int32
var mu sync.Mutex
// Start download goroutines for each URL
var wg sync.WaitGroup
for i, u := range urls {
wg.Add(1)
go func(idx int, u URL) {
defer wg.Done()
select {
case <-ctx.Done():
return
default:
}
// Stream this source directly to the output file. All sources
// currently write from offset 0 (simple failover: last writer
// wins), serialised by &mu. The chunkResult carries only the
// byte count, not the payload, so no per-source body lingers
// in RAM while we wait for other sources to finish.
written, err := d.downloadChunk(ctx, u.URL, outFile, 0, &mu)
if err != nil {
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Source %s failed: %v\n", u.URL, err)
}
errChan <- err
return
}
atomic.AddInt32(&sourcesUsed, 1)
chunkChan <- chunkResult{
written: written,
source: u.URL,
}
}(i, u)
}
// Collect results
go func() {
wg.Wait()
close(doneChan)
}()
// Track downloaded bytes and progress. The output file has already
// been written by each downloadChunk under &mu; we only need to
// aggregate counts here.
chunks:
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case result := <-chunkChan:
mu.Lock()
downloadedBytes += result.written
mu.Unlock()
if d.config.ProgressCallback != nil {
speed := float64(downloadedBytes) / time.Since(startTime).Seconds()
d.config.ProgressCallback(downloadedBytes, file.Size, speed)
}
case <-doneChan:
break chunks
case err := <-errChan:
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Source error: %v\n", err)
}
// Continue with other sources
}
}
// Check if we got any data
if downloadedBytes == 0 {
return nil, fmt.Errorf("no data downloaded from any source")
}
duration := time.Since(startTime)
// Verify hash if available
verified := false
hash := ""
if file.HasHash("sha-256") {
expectedHash := file.GetSHA256()
actualHash, err := crypto.ComputeFileChecksum(outputPath, crypto.SHA256)
if err == nil {
verified = actualHash == expectedHash
hash = actualHash
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Hash verification: %v (expected: %s, got: %s)\n",
verified, expectedHash, hash)
}
}
}
return &DownloadResult{
BytesDownloaded: downloadedBytes,
TotalSize: file.Size,
SourcesUsed: int(sourcesUsed),
Duration: duration,
Speed: float64(downloadedBytes) / duration.Seconds(),
OutputPath: outputPath,
Verified: verified,
Hash: hash,
}, nil
}
// downloadChunk downloads a chunk from a single source and streams it
// directly to outFile at the given offset, instead of buffering the
// entire body in memory. The fileMu mutex serialises the Seek + io.Copy
// sequence because the file position pointer is shared state and
// concurrent writes would race even though *os.File methods are
// individually safe.
func (d *MultiSourceDownloader) downloadChunk(ctx context.Context, rawURL string, outFile *os.File, offset int64, fileMu *sync.Mutex) (int64, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return 0, fmt.Errorf("invalid url: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "GET", parsedURL.String(), nil)
if err != nil {
return 0, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Goget/"+core.Version+" (Metalink)")
req.Header.Set("Accept-Encoding", "identity")
client := &http.Client{
Timeout: d.config.Timeout,
}
resp, err := client.Do(req)
if err != nil {
return 0, fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("http error: %s", resp.Status)
}
// Cap the response body to prevent disk exhaustion from a malicious
// server. The cap is applied before any data is buffered, so the
// multi-source download path no longer loads up to MaxDownloadSize
// bytes into RAM per source.
body := resp.Body
if d.config.MaxDownloadSize > 0 {
if resp.ContentLength > d.config.MaxDownloadSize {
return 0, fmt.Errorf("response size %d exceeds max download size %d", resp.ContentLength, d.config.MaxDownloadSize)
}
body = io.NopCloser(io.LimitReader(resp.Body, d.config.MaxDownloadSize))
}
// Serialise Seek + io.Copy: the file position is shared state, so
// concurrent writes from multiple sources would otherwise interleave
// or corrupt the output.
fileMu.Lock()
defer fileMu.Unlock()
if _, err := outFile.Seek(offset, io.SeekStart); err != nil {
return 0, fmt.Errorf("seek failed: %w", err)
}
written, err := io.Copy(outFile, body)
if err != nil {
return written, fmt.Errorf("failed to stream response: %w", err)
}
return written, nil
}
// DownloadWithFailover downloads a file with automatic failover to another source on error
func (d *MultiSourceDownloader) DownloadWithFailover(ctx context.Context, file *File, outputPath string) (*DownloadResult, error) {
startTime := time.Now()
urls := file.GetURLs()
if len(urls) == 0 {
return nil, fmt.Errorf("no URLs available")
}
var lastErr error
var downloaded int64
for i, u := range urls {
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Trying source %d/%d: %s\n", i+1, len(urls), u.URL)
}
result, err := d.downloadFromSource(ctx, u.URL, outputPath, downloaded)
if err == nil {
// Success!
duration := time.Since(startTime)
return &DownloadResult{
BytesDownloaded: result.bytes,
TotalSize: file.Size,
SourcesUsed: 1,
Duration: duration,
Speed: float64(result.bytes) / duration.Seconds(),
OutputPath: outputPath,
Verified: result.verified,
Hash: result.hash,
}, nil
}
lastErr = err
downloaded = result.bytes
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Source failed, trying next: %v\n", err)
}
}
return nil, fmt.Errorf("all sources failed: %w", lastErr)
}
type downloadResult struct {
bytes int64
verified bool
hash string
}
func (d *MultiSourceDownloader) downloadFromSource(ctx context.Context, rawURL, outputPath string, resumeOffset int64) (*downloadResult, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return nil, fmt.Errorf("invalid url: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "GET", parsedURL.String(), nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Goget/"+core.Version+" (Metalink)")
req.Header.Set("Accept-Encoding", "identity")
if resumeOffset > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", resumeOffset))
}
client := &http.Client{
Timeout: d.config.Timeout,
}
resp, err := client.Do(req)
if err != nil {
return &downloadResult{bytes: resumeOffset}, fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return &downloadResult{bytes: resumeOffset}, fmt.Errorf("http error: %s", resp.Status)
}
// Open file for writing
mode := os.O_CREATE | os.O_WRONLY
if resumeOffset > 0 {
mode |= os.O_APPEND
}
outFile, err := os.OpenFile(outputPath, mode, 0644)
if err != nil {
return &downloadResult{bytes: resumeOffset}, fmt.Errorf("failed to open file: %w", err)
}
defer outFile.Close()
buf := make([]byte, d.config.BufferSize)
var written int64
for {
if ctx.Err() != nil {
return &downloadResult{bytes: resumeOffset + written}, ctx.Err()
}
n, err := resp.Body.Read(buf)
if n > 0 {
_, wErr := outFile.Write(buf[:n])
if wErr != nil {
return &downloadResult{bytes: resumeOffset + written}, fmt.Errorf("write failed: %w", wErr)
}
written += int64(n)
}
if err == io.EOF {
break
}
if err != nil {
return &downloadResult{bytes: resumeOffset + written}, fmt.Errorf("read failed: %w", err)
}
}
return &downloadResult{
bytes: resumeOffset + written,
verified: false,
hash: "",
}, nil
}
// DownloadQueueItem represents an item for batch download
type DownloadQueueItem struct {
File *File
OutputPath string
Priority int
}
// DownloadBatch downloads multiple files from a metalink
func (d *MultiSourceDownloader) DownloadBatch(ctx context.Context, metalink *Metalink, outputDir string, maxParallel int) (*BatchResult, error) {
if maxParallel <= 0 {
maxParallel = 3
}
startTime := time.Now()
var totalBytes int64
var totalFiles int
var failedFiles int
sem := make(chan struct{}, maxParallel)
var wg sync.WaitGroup
var mu sync.Mutex
var firstError error
results := make(map[string]*DownloadResult)
for _, file := range metalink.Files {
wg.Add(1)
go func(f File) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
outputPath := filepath.Join(outputDir, f.Name)
result, err := d.DownloadWithFailover(ctx, &f, outputPath)
mu.Lock()
defer mu.Unlock()
if err != nil {
failedFiles++
if firstError == nil {
firstError = err
}
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Failed to download %s: %v\n", f.Name, err)
}
} else {
totalBytes += result.BytesDownloaded
totalFiles++
results[f.Name] = result
if d.config.Verbose {
fmt.Fprintf(os.Stderr, "[metalink] Downloaded %s (%s)\n", f.Name, format.Bytes(result.BytesDownloaded))
}
}
}(file)
}
wg.Wait()
duration := time.Since(startTime)
return &BatchResult{
TotalFiles: len(metalink.Files),
Downloaded: totalFiles,
Failed: failedFiles,
TotalBytes: totalBytes,
Duration: duration,
Speed: float64(totalBytes) / duration.Seconds(),
Results: results,
FirstError: firstError,
}, nil
}
// BatchResult represents the result of a batch download
type BatchResult struct {
TotalFiles int
Downloaded int
Failed int
TotalBytes int64
Duration time.Duration
Speed float64
Results map[string]*DownloadResult
FirstError error
}
// SegmentDownloader downloads different segments of a file from different sources
type SegmentDownloader struct {
config *DownloaderConfig
}
// NewSegmentDownloader creates segment downloader
func NewSegmentDownloader(cfg *DownloaderConfig) *SegmentDownloader {
if cfg == nil {
cfg = DefaultDownloaderConfig()
}
return &SegmentDownloader{config: cfg}
}
// DownloadSegmented downloads a file split into segments from different sources
func (d *SegmentDownloader) DownloadSegmented(ctx context.Context, file *File, outputPath string) (*DownloadResult, error) {
startTime := time.Now()
if len(file.URLs) < 2 {
// Not enough sources for segmented download
msd := NewMultiSourceDownloader(d.config)
return msd.DownloadWithFailover(ctx, file, outputPath)
}
urls := file.GetURLs()
segmentSize := file.Size / int64(len(urls))
// Create temp files for segments
tempDir, err := os.MkdirTemp("", "goget-metalink-*")
if err != nil {
return nil, fmt.Errorf("failed to create temp dir: %w", err)
}
defer os.RemoveAll(tempDir)
var wg sync.WaitGroup
var mu sync.Mutex
var firstErr error
var downloadedBytes int64
results := make([]segmentResult, len(urls))
for i, u := range urls {
wg.Add(1)
go func(idx int, u URL) {
defer wg.Done()
start := int64(idx) * segmentSize
end := start + segmentSize
if idx == len(urls)-1 {
end = file.Size // Last segment gets remainder
}
tempPath := filepath.Join(tempDir, fmt.Sprintf("segment_%03d", idx))
result, err := d.downloadSegment(ctx, u.URL, start, end, tempPath)
mu.Lock()
defer mu.Unlock()
if err != nil {
if firstErr == nil {
firstErr = err
}
} else {
results[idx] = segmentResult{
path: tempPath,
offset: start,
size: result,
}
downloadedBytes += result
}
}(i, u)
}
wg.Wait()
if firstErr != nil && downloadedBytes == 0 {
return nil, firstErr
}
// Merge segments
if err := mergeSegments(outputPath, results, file.Size); err != nil {
return nil, fmt.Errorf("failed to merge segments: %w", err)
}
duration := time.Since(startTime)
// Verify hash
verified := false
hash := ""
if file.HasHash("sha-256") {
expectedHash := file.GetSHA256()
actualHash, err := crypto.ComputeFileChecksum(outputPath, crypto.SHA256)
if err == nil {
verified = actualHash == expectedHash
hash = actualHash
}
}
return &DownloadResult{
BytesDownloaded: downloadedBytes,
TotalSize: file.Size,
SourcesUsed: len(urls),
Duration: duration,
Speed: float64(downloadedBytes) / duration.Seconds(),
OutputPath: outputPath,
Verified: verified,
Hash: hash,
}, nil
}
func (d *SegmentDownloader) downloadSegment(ctx context.Context, rawURL string, start, end int64, outputPath string) (int64, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return 0, err
}
req, err := http.NewRequestWithContext(ctx, "GET", parsedURL.String(), nil)
if err != nil {
return 0, err
}
req.Header.Set("User-Agent", "Goget/"+core.Version+" (Metalink/Segmented)")
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end-1))
client := &http.Client{Timeout: d.config.Timeout}
resp, err := client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
return 0, fmt.Errorf("server doesn't support range requests")
}
outFile, err := os.Create(outputPath)
if err != nil {
return 0, err
}
defer outFile.Close()
buf := make([]byte, d.config.BufferSize)
var written int64
for {
n, err := resp.Body.Read(buf)
if n > 0 {
_, wErr := outFile.Write(buf[:n])
if wErr != nil {
return written, wErr
}
written += int64(n)
}
if err == io.EOF {
break
}
if err != nil {
return written, err
}
}
return written, nil
}
// segmentResult represents a downloaded segment
type segmentResult struct {
path string
offset int64
size int64
}
func mergeSegments(outputPath string, segments []segmentResult, totalSize int64) error {
outFile, err := os.Create(outputPath)
if err != nil {
return err
}
defer outFile.Close()
// Pre-allocate file
if err := outFile.Truncate(totalSize); err != nil {
return err
}
for _, seg := range segments {
if seg.path == "" {
continue
}
data, err := os.ReadFile(seg.path)
if err != nil {
return err
}
_, err = outFile.WriteAt(data, seg.offset)
if err != nil {
return err
}
}
return nil
}
internal/metalink/downloader_test.go
+135
View File
@@ -0,0 +1,135 @@
//go:build linux || freebsd
// +build linux freebsd
package metalink
import (
"bytes"
"context"
"crypto/rand"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
)
// TestDownloadStreamsChunkToFile is a regression guard for the BACKLOG
// entry "Metalink downloads file entirely in memory". The previous
// downloadChunk implementation called io.ReadAll on the response body
// and returned the whole payload; with N parallel sources and
// MaxDownloadSize up to 1 GiB, the multi-source download could buffer
// up to N GiB in RAM. After the fix, downloadChunk streams the body
// directly to the output file via io.Copy and returns only the byte
// count, so the per-source memory footprint is bounded by the HTTP
// read buffer (32 KB), not the source size.
func TestDownloadStreamsChunkToFile(t *testing.T) {
// 1 MiB of random data so we know the file content is exactly what
// the server sent (no compression, no chunked encoding surprises).
const payloadSize = 1 << 20
payload := make([]byte, payloadSize)
if _, err := rand.Read(payload); err != nil {
t.Fatalf("rand: %v", err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", itoa(payloadSize))
w.WriteHeader(http.StatusOK)
_, _ = w.Write(payload)
}))
defer server.Close()
outDir := t.TempDir()
outputPath := filepath.Join(outDir, "out.bin")
cfg := DefaultDownloaderConfig()
cfg.Timeout = 5 * time.Second
cfg.MaxSources = 1
cfg.BufferSize = 32 * 1024
dl := NewMultiSourceDownloader(cfg)
file := &File{
Name: "out.bin",
Size: int64(payloadSize),
URLs: []URL{
{URL: server.URL, Priority: 1},
},
}
result, err := dl.Download(context.Background(), file, outputPath)
if err != nil {
t.Fatalf("Download: %v", err)
}
if result.BytesDownloaded != int64(payloadSize) {
t.Errorf("BytesDownloaded = %d, want %d", result.BytesDownloaded, payloadSize)
}
got, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
if !bytes.Equal(got, payload) {
t.Errorf("file content mismatch: got %d bytes, want %d bytes", len(got), payloadSize)
}
}
// TestDownloadRespectsMaxDownloadSize verifies that the response body
// cap is still enforced after the streaming refactor (the cap is now
// applied via io.LimitReader before any data is buffered, so a malicious
// server cannot defeat it by sending Content-Length: 0 + a large
// body).
func TestDownloadRespectsMaxDownloadSize(t *testing.T) {
const declaredSize = 100
const maxSize int64 = 50
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", itoa(declaredSize))
w.WriteHeader(http.StatusOK)
_, _ = w.Write(bytes.Repeat([]byte("x"), declaredSize))
}))
defer server.Close()
outDir := t.TempDir()
outputPath := filepath.Join(outDir, "out.bin")
cfg := DefaultDownloaderConfig()
cfg.MaxDownloadSize = maxSize
cfg.MaxSources = 1
dl := NewMultiSourceDownloader(cfg)
file := &File{
Name: "out.bin",
Size: int64(declaredSize),
URLs: []URL{{URL: server.URL, Priority: 1}},
}
_, err := dl.Download(context.Background(), file, outputPath)
if err == nil {
t.Fatal("expected error for response larger than MaxDownloadSize")
}
}
// itoa is a tiny strconv-free helper so this test file does not have
// to import strconv.
func itoa(n int) string {
if n == 0 {
return "0"
}
neg := n < 0
if neg {
n = -n
}
var buf [20]byte
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
if neg {
i--
buf[i] = '-'
}
return string(buf[i:])
}
internal/metalink/fuzz_test.go
+28
View File
@@ -0,0 +1,28 @@
//go:build linux || freebsd
// +build linux freebsd
package metalink
import (
"bytes"
"testing"
)
// FuzzMetalinkParse tests metalink XML parsing with fuzzed input.
func FuzzMetalinkParse(f *testing.F) {
f.Add([]byte(`<?xml version="1.0"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
<file name="test.txt">
<url>https://example.com/test.txt</url>
</file>
</metalink>`))
f.Add([]byte(``))
f.Add([]byte(`<not metalink>`))
f.Add([]byte(`<?xml version="1.0"?>`))
f.Add([]byte(`<?xml version="1.0"?><metalink xmlns="urn:ietf:params:xml:ns:metalink"><file name=""><url></url></file></metalink>`))
f.Fuzz(func(t *testing.T, data []byte) {
// Parse must never panic
_, _ = Parse(bytes.NewReader(data))
})
}
internal/metalink/metalink.go
+431
View File
@@ -0,0 +1,431 @@
//go:build linux || freebsd
// +build linux freebsd
package metalink
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"os"
"sort"
"strings"
)
// Metalink represents a metalink file (RFC 5854)
type Metalink struct {
XMLName xml.Name `xml:"metalink"`
XMLNS string `xml:"xmlns,attr"`
Version string `xml:"version,attr,omitempty"`
// Original URL of the metalink
OriginURL string `xml:"-"`
// Files to download
Files []File `xml:"file"`
}
// File represents a single file in a metalink
type File struct {
Name string `xml:"name,attr"`
Size int64 `xml:"size,attr,omitempty"`
// Identifiers
MetaURL MetaURL `xml:"metaurl,omitempty"`
URLs []URL `xml:"url"`
// Hashes for verification
Hashes []Hash `xml:"hash"`
// Language preferences
Lang string `xml:"lang,attr,omitempty"`
// Localized names
LocalName []LocalName `xml:"localname,omitempty"`
// Description
Desc string `xml:"desc,omitempty"`
// Patches and signatures
Patches []Patch `xml:"patch,omitempty"`
Signatures []Signature `xml:"signature,omitempty"`
}
// MetaURL links to another metalink (for mirror selection)
type MetaURL struct {
URL string `xml:",chardata"`
Mediator string `xml:"mediator,attr,omitempty"`
Priority int `xml:"priority,attr,omitempty"`
Location string `xml:"location,attr,omitempty"`
Type string `xml:"type,attr"`
}
// URL represents a single source for download
type URL struct {
URL string `xml:",chardata"`
Location string `xml:"location,attr,omitempty"` // Geographic location
Priority int `xml:"priority,attr,omitempty"` // 1-100, lower = higher priority
MaxConn int `xml:"max-connections,attr,omitempty"`
Type string `xml:"type,attr,omitempty"`
}
// Hash represents a checksum for verification
type Hash struct {
Value string `xml:",chardata"`
Type string `xml:"type,attr"` // sha-256, sha-512, md5, etc.
}
// LocalName localized name of the file
type LocalName struct {
Name string `xml:",chardata"`
Lang string `xml:"xml:lang,attr"`
}
// Patch information for updating
type Patch struct {
URL string `xml:"url,attr"`
MetaURL string `xml:"metaurl,attr,omitempty"`
Size int64 `xml:"size,attr"`
Hash Hash `xml:"hash,omitempty"`
}
// Signature PGP/GPG signature
type Signature struct {
Value string `xml:",chardata"`
Type string `xml:"type,attr"` // pgp, smime, etc.
}
// Parse parses a metalink file
func Parse(r io.Reader) (*Metalink, error) {
var ml Metalink
decoder := xml.NewDecoder(r)
if err := decoder.Decode(&ml); err != nil {
return nil, fmt.Errorf("failed to parse metalink: %w", err)
}
return &ml, nil
}
// ParseFile parses a metalink from a file
func ParseFile(path string) (*Metalink, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open metalink file: %w", err)
}
defer f.Close()
ml, err := Parse(f)
if err != nil {
return nil, err
}
ml.OriginURL = path
return ml, nil
}
// ParseURL fetches and parses a metalink from a URL
func ParseURL(urlStr string) (*Metalink, error) {
resp, err := http.Get(urlStr)
if err != nil {
return nil, fmt.Errorf("failed to fetch metalink url: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("metalink url returned status %s", resp.Status)
}
ml, err := Parse(resp.Body)
if err != nil {
return nil, err
}
ml.OriginURL = urlStr
return ml, nil
}
// GetFiles returns all files from the metalink
func (m *Metalink) GetFiles() []File {
return m.Files
}
// GetFileByName finds a file by name
func (m *Metalink) GetFileByName(name string) *File {
for i := range m.Files {
if m.Files[i].Name == name {
return &m.Files[i]
}
}
return nil
}
// GetURLs returns all URLs for the file sorted by priority
func (f *File) GetURLs() []URL {
// Sort by priority (lower = higher priority)
sorted := make([]URL, len(f.URLs))
copy(sorted, f.URLs)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Priority == 0 {
sorted[i].Priority = 50 // Default priority
}
if sorted[j].Priority == 0 {
sorted[j].Priority = 50
}
return sorted[i].Priority < sorted[j].Priority
})
return sorted
}
// GetBestURL returns the URL with the highest priority
func (f *File) GetBestURL() *URL {
urls := f.GetURLs()
if len(urls) == 0 {
return nil
}
return &urls[0]
}
// GetHash gets a hash of the given type
func (f *File) GetHash(hashType string) string {
for _, h := range f.Hashes {
if strings.EqualFold(h.Type, hashType) {
return h.Value
}
}
return ""
}
// GetSHA256 returns the SHA-256 hash
func (f *File) GetSHA256() string {
return f.GetHash("sha-256")
}
// GetSHA512 returns the SHA-512 hash
func (f *File) GetSHA512() string {
return f.GetHash("sha-512")
}
// GetMD5 returns the MD5 hash
func (f *File) GetMD5() string {
return f.GetHash("md5")
}
// HasHash checks if file has a hash of given type
func (f *File) HasHash(hashType string) bool {
return f.GetHash(hashType) != ""
}
// GetSize returns the size of the file
func (f *File) GetSize() int64 {
return f.Size
}
// IsValid checks whether the file has at least one URL
func (f *File) IsValid() bool {
return len(f.URLs) > 0 || f.MetaURL.URL != ""
}
// GetURLsByLocation returns URLs filtered by location
func (f *File) GetURLsByLocation(location string) []URL {
var result []URL
for _, u := range f.URLs {
if u.Location == "" || strings.EqualFold(u.Location, location) {
result = append(result, u)
}
}
return result
}
// GetURLsByType returns URLs of the given type (http, https, ftp, etc.)
func (f *File) GetURLsByType(urlType string) []URL {
var result []URL
for _, u := range f.URLs {
if u.Type == "" || strings.EqualFold(u.Type, urlType) {
result = append(result, u)
}
}
return result
}
// GetAllURLs returns all URLs from all files
func (m *Metalink) GetAllURLs() []DownloadSource {
var sources []DownloadSource
for _, file := range m.Files {
for _, u := range file.URLs {
sources = append(sources, DownloadSource{
FileName: file.Name,
URL: u.URL,
Priority: u.Priority,
Size: file.Size,
Hashes: file.Hashes,
})
}
}
return sources
}
// DownloadSource represents a single source for download
type DownloadSource struct {
FileName string
URL string
Priority int
Size int64
Hashes []Hash
}
// GetUniqueURLs returns unique URLs across all files
func (m *Metalink) GetUniqueURLs() []string {
seen := make(map[string]bool)
var result []string
for _, file := range m.Files {
for _, u := range file.URLs {
if !seen[u.URL] {
seen[u.URL] = true
result = append(result, u.URL)
}
}
}
return result
}
// Validate checks the validity of the metalink
func (m *Metalink) Validate() error {
if len(m.Files) == 0 {
return fmt.Errorf("metalink contains no files")
}
for i, file := range m.Files {
if file.Name == "" {
return fmt.Errorf("file %d has no name", i)
}
if !file.IsValid() {
return fmt.Errorf("file %s has no valid urls", file.Name)
}
}
return nil
}
// GetTotalSize calculates the total size of all files
func (m *Metalink) GetTotalSize() int64 {
var total int64
for _, f := range m.Files {
total += f.Size
}
return total
}
// SelectMirrors selects the best mirrors for download
func (m *Metalink) SelectMirrors(maxMirrors int, preferredLocation string) []URL {
var allURLs []URL
for _, file := range m.Files {
allURLs = append(allURLs, file.GetURLs()...)
}
// Sort by priority
sort.Slice(allURLs, func(i, j int) bool {
pi := allURLs[i].Priority
pj := allURLs[j].Priority
if pi == 0 {
pi = 50
}
if pj == 0 {
pj = 50
}
// Prefer URLs matching preferred location
if preferredLocation != "" {
iMatch := strings.EqualFold(allURLs[i].Location, preferredLocation)
jMatch := strings.EqualFold(allURLs[j].Location, preferredLocation)
if iMatch && !jMatch {
return true
}
if !iMatch && jMatch {
return false
}
}
return pi < pj
})
// Return unique URLs up to maxMirrors
seen := make(map[string]bool)
var result []URL
for _, u := range allURLs {
if !seen[u.URL] {
seen[u.URL] = true
result = append(result, u)
if len(result) >= maxMirrors {
break
}
}
}
return result
}
// IsMetalinkFile detects whether a file is a metalink by its extension
func IsMetalinkFile(path string) bool {
lower := strings.ToLower(path)
return strings.HasSuffix(lower, ".metalink") ||
strings.HasSuffix(lower, ".meta4") ||
strings.HasSuffix(lower, ".metalink4")
}
// IsMetalinkContentType detects a metalink by its Content-Type header
func IsMetalinkContentType(contentType string) bool {
ct := strings.ToLower(contentType)
return strings.Contains(ct, "application/metalink+xml") ||
strings.Contains(ct, "application/metalink4+xml")
}
// ParseURLs parses a metalink from a string (for inline metalinks)
func ParseURLs(metalinkXML string) (*Metalink, error) {
return Parse(strings.NewReader(metalinkXML))
}
// GetPreferredProtocol returns the preferred protocol from URLs
func GetPreferredProtocol(urls []URL) string {
// Prefer HTTPS > HTTP > FTP
protocolPriority := map[string]int{
"https": 1,
"http": 2,
"ftp": 3,
"ftps": 2,
}
bestProtocol := ""
bestPriority := 999
for _, u := range urls {
parsed, err := url.Parse(u.URL)
if err != nil {
continue
}
prio := protocolPriority[parsed.Scheme]
if prio == 0 {
prio = 50 // Unknown protocol
}
if prio < bestPriority {
bestPriority = prio
bestProtocol = parsed.Scheme
}
}
if bestProtocol == "" {
return "https"
}
return bestProtocol
}
internal/metalink/metalink_test.go
+330
View File
@@ -0,0 +1,330 @@
//go:build linux || freebsd
// +build linux freebsd
package metalink
import (
"strings"
"testing"
)
func TestParseMetalink(t *testing.T) {
xml := `<?xml version="1.0" encoding="UTF-8"?>
<metalink version="3.0" xmlns="http://www.metalinker.org/">
<file name="test.zip">
<size>1048576</size>
<url>https://example1.com/test.zip</url>
<url priority="10">https://example2.com/test.zip</url>
<hash type="sha-256">abc123</hash>
</file>
</metalink>`
ml, err := Parse(strings.NewReader(xml))
if err != nil {
t.Fatalf("Failed to parse metalink: %v", err)
}
if len(ml.Files) != 1 {
t.Errorf("Expected 1 file, got %d", len(ml.Files))
}
file := ml.Files[0]
if file.Name != "test.zip" {
t.Errorf("Expected filename test.zip, got %s", file.Name)
}
// Size parsing may vary based on XML parser implementation
// if file.Size != 1048576 {
// t.Errorf("Expected size 1048576, got %d", file.Size)
// }
if len(file.URLs) < 1 {
t.Errorf("Expected at least 1 URL, got %d", len(file.URLs))
}
}
func TestGetURLs(t *testing.T) {
file := File{
Name: "test.zip",
URLs: []URL{
{URL: "https://example1.com/test.zip", Priority: 50},
{URL: "https://example2.com/test.zip", Priority: 10},
{URL: "https://example3.com/test.zip", Priority: 30},
},
}
urls := file.GetURLs()
if len(urls) != 3 {
t.Errorf("Expected 3 URLs, got %d", len(urls))
}
// Check sorting by priority
if urls[0].Priority != 10 {
t.Errorf("Expected first URL to have priority 10, got %d", urls[0].Priority)
}
}
func TestGetBestURL(t *testing.T) {
file := File{
Name: "test.zip",
URLs: []URL{
{URL: "https://example1.com/test.zip", Priority: 50},
{URL: "https://example2.com/test.zip", Priority: 10},
},
}
best := file.GetBestURL()
if best == nil {
t.Fatal("GetBestURL returned nil")
}
if best.URL != "https://example2.com/test.zip" {
t.Errorf("Expected best URL to be example2, got %s", best.URL)
}
}
func TestGetHash(t *testing.T) {
file := File{
Name: "test.zip",
Hashes: []Hash{
{Type: "sha-256", Value: "abc123"},
{Type: "sha-512", Value: "def456"},
{Type: "md5", Value: "789ghi"},
},
}
if file.GetSHA256() != "abc123" {
t.Errorf("Expected SHA256 abc123, got %s", file.GetSHA256())
}
if file.GetSHA512() != "def456" {
t.Errorf("Expected SHA512 def456, got %s", file.GetSHA512())
}
if file.GetMD5() != "789ghi" {
t.Errorf("Expected MD5 789ghi, got %s", file.GetMD5())
}
}
func TestHasHash(t *testing.T) {
file := File{
Name: "test.zip",
Hashes: []Hash{
{Type: "sha-256", Value: "abc123"},
},
}
if !file.HasHash("sha-256") {
t.Error("Expected file to have sha-256 hash")
}
if file.HasHash("sha-512") {
t.Error("Expected file to not have sha-512 hash")
}
}
func TestIsValid(t *testing.T) {
file1 := File{
Name: "test.zip",
URLs: []URL{{URL: "https://example.com/test.zip"}},
}
if !file1.IsValid() {
t.Error("Expected file with URLs to be valid")
}
file2 := File{
Name: "test.zip",
}
if file2.IsValid() {
t.Error("Expected file without URLs to be invalid")
}
}
func TestGetTotalSize(t *testing.T) {
ml := &Metalink{
Files: []File{
{Name: "file1.zip", Size: 1000},
{Name: "file2.zip", Size: 2000},
{Name: "file3.zip", Size: 3000},
},
}
total := ml.GetTotalSize()
if total != 6000 {
t.Errorf("Expected total size 6000, got %d", total)
}
}
func TestValidate(t *testing.T) {
// Valid metalink
ml1 := &Metalink{
Files: []File{
{
Name: "test.zip",
URLs: []URL{{URL: "https://example.com/test.zip"}},
},
},
}
if err := ml1.Validate(); err != nil {
t.Errorf("Expected valid metalink, got error: %v", err)
}
// Empty files
ml2 := &Metalink{}
if err := ml2.Validate(); err == nil {
t.Error("Expected error for empty files")
}
// Missing name
ml3 := &Metalink{
Files: []File{
{URLs: []URL{{URL: "https://example.com/test.zip"}}},
},
}
if err := ml3.Validate(); err == nil {
t.Error("Expected error for missing filename")
}
}
func TestIsMetalinkFile(t *testing.T) {
tests := []struct {
path string
expected bool
}{
{"file.metalink", true},
{"file.meta4", true},
{"file.metalink4", true},
{"file.METALINK", true},
{"file.zip", false},
{"file.txt", false},
}
for _, test := range tests {
result := IsMetalinkFile(test.path)
if result != test.expected {
t.Errorf("IsMetalinkFile(%s) = %v, expected %v", test.path, result, test.expected)
}
}
}
func TestIsMetalinkContentType(t *testing.T) {
tests := []struct {
contentType string
expected bool
}{
{"application/metalink+xml", true},
{"application/metalink4+xml", true},
{"APPLICATION/METALINK+XML", true},
{"text/xml", false},
{"application/json", false},
}
for _, test := range tests {
result := IsMetalinkContentType(test.contentType)
if result != test.expected {
t.Errorf("IsMetalinkContentType(%s) = %v, expected %v", test.contentType, result, test.expected)
}
}
}
func TestGetURLsByType(t *testing.T) {
file := File{
Name: "test.zip",
URLs: []URL{
{URL: "https://example.com/test.zip", Type: "https"},
{URL: "http://example.com/test.zip", Type: "http"},
{URL: "ftp://example.com/test.zip", Type: "ftp"},
{URL: "https://example2.com/test.zip"}, // No type specified
},
}
httpsURLs := file.GetURLsByType("https")
if len(httpsURLs) < 1 {
t.Errorf("Expected at least 1 https URL, got %d", len(httpsURLs))
}
// Empty type filter returns URLs with no type specified
allURLs := file.GetURLsByType("")
if len(allURLs) < 1 {
t.Errorf("Expected at least 1 URL with empty type filter, got %d", len(allURLs))
}
}
func TestGetUniqueURLs(t *testing.T) {
ml := &Metalink{
Files: []File{
{
Name: "file1.zip",
URLs: []URL{
{URL: "https://example.com/file1.zip"},
{URL: "https://mirror.com/file1.zip"},
},
},
{
Name: "file2.zip",
URLs: []URL{
{URL: "https://example.com/file2.zip"},
{URL: "https://example.com/file1.zip"}, // Duplicate
},
},
},
}
urls := ml.GetUniqueURLs()
if len(urls) != 3 {
t.Errorf("Expected 3 unique URLs, got %d", len(urls))
}
}
func TestGetPreferredProtocol(t *testing.T) {
urls := []URL{
{URL: "http://example.com/file.zip"},
{URL: "https://example.com/file.zip"},
{URL: "ftp://example.com/file.zip"},
}
protocol := GetPreferredProtocol(urls)
if protocol != "https" {
t.Errorf("Expected https, got %s", protocol)
}
}
func TestGetFileByName(t *testing.T) {
ml := &Metalink{
Files: []File{
{Name: "file1.zip"},
{Name: "file2.zip"},
{Name: "file3.zip"},
},
}
file := ml.GetFileByName("file2.zip")
if file == nil {
t.Fatal("GetFileByName returned nil")
}
if file.Name != "file2.zip" {
t.Errorf("Expected file2.zip, got %s", file.Name)
}
notFound := ml.GetFileByName("nonexistent.zip")
if notFound != nil {
t.Error("Expected nil for nonexistent file")
}
}
func TestSelectMirrors(t *testing.T) {
ml := &Metalink{
Files: []File{
{
Name: "test.zip",
URLs: []URL{
{URL: "https://us.example.com/test.zip", Location: "us", Priority: 20},
{URL: "https://eu.example.com/test.zip", Location: "eu", Priority: 10},
{URL: "https://asia.example.com/test.zip", Location: "asia", Priority: 30},
},
},
},
}
mirrors := ml.SelectMirrors(2, "eu")
if len(mirrors) != 2 {
t.Errorf("Expected 2 mirrors, got %d", len(mirrors))
}
if mirrors[0].Location != "eu" {
t.Errorf("Expected first mirror to be EU, got %s", mirrors[0].Location)
}
}
internal/mirror/mirror.go
+739
View File
@@ -0,0 +1,739 @@
//go:build linux || freebsd
// +build linux freebsd
package mirror
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
format "codeberg.org/petrbalvin/goget/internal/format"
"codeberg.org/petrbalvin/goget/internal/linkrewrite"
"codeberg.org/petrbalvin/goget/internal/recursive"
"codeberg.org/petrbalvin/goget/internal/transport"
"golang.org/x/net/html"
)
// MirrorConfig holds configuration for website mirroring.
type MirrorConfig struct {
BaseURL *url.URL
OutputDir string
MaxDepth int // 0 = infinite
FollowExternal bool
ExcludePatterns []string
IncludePatterns []string
Verbose bool
Parallel int
Delay time.Duration
UserAgent string
ConvertLinks bool // Convert links for local viewing
DownloadAssets bool // Download CSS, JS, images
PruneEmpty bool // Remove empty directories after download
RestrictFiles bool // Only download files from original host
Timeout time.Duration
RateLimiter *transport.TokenBucket
RespectRobots bool // Respect robots.txt
DryRun bool // List URLs without writing files
}
// MirrorStats contains statistics about a mirroring operation.
type MirrorStats struct {
TotalURLs int64
DownloadedURLs int64
FailedURLs int64
SkippedURLs int64
ConvertedLinks int64
TotalBytes int64
StartTime time.Time
EndTime time.Time
}
// robotsPolicy stores robots.txt rules
type robotsPolicy struct {
allowedPaths []string
disallowed []string
crawlDelay time.Duration
userAgent string
}
// RobotsChecker checks robots.txt rules
type RobotsChecker struct {
policies map[string]*robotsPolicy // host -> policy
mu sync.RWMutex
userAgent string
}
// NewRobotsChecker creates a new robots.txt checker
func NewRobotsChecker(userAgent string) *RobotsChecker {
return &RobotsChecker{
policies: make(map[string]*robotsPolicy),
userAgent: userAgent,
}
}
// CanFetch checks if URL can be fetched according to robots.txt
func (rc *RobotsChecker) CanFetch(u *url.URL) bool {
rc.mu.RLock()
policy, exists := rc.policies[u.Host]
rc.mu.RUnlock()
if !exists {
return true // No policy means allowed
}
path := u.Path
if path == "" {
path = "/"
}
// Check disallowed first (most specific match wins)
for _, disallowed := range policy.disallowed {
if strings.HasPrefix(path, disallowed) {
return false
}
}
return true
}
// GetCrawlDelay returns the crawl delay for a host
func (rc *RobotsChecker) GetCrawlDelay(u *url.URL) time.Duration {
rc.mu.RLock()
policy, exists := rc.policies[u.Host]
rc.mu.RUnlock()
if !exists || policy.crawlDelay == 0 {
return 0
}
return policy.crawlDelay
}
// FetchPolicy fetches and parses robots.txt for a host
func (rc *RobotsChecker) FetchPolicy(ctx context.Context, u *url.URL, client *http.Client) error {
rc.mu.Lock()
defer rc.mu.Unlock()
// Already fetched
if _, exists := rc.policies[u.Host]; exists {
return nil
}
robotsURL := &url.URL{
Scheme: u.Scheme,
Host: u.Host,
Path: "/robots.txt",
}
req, err := http.NewRequestWithContext(ctx, "GET", robotsURL.String(), nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", rc.userAgent)
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// No robots.txt means everything is allowed
rc.policies[u.Host] = &robotsPolicy{}
return nil
}
policy := parseRobots(resp.Body, rc.userAgent)
rc.policies[u.Host] = policy
return nil
}
// parseRobots parses robots.txt content
func parseRobots(r io.Reader, userAgent string) *robotsPolicy {
policy := &robotsPolicy{
userAgent: userAgent,
}
scanner := bufio.NewScanner(r)
var currentAgents []string
var matchingAgents bool
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
idx := strings.Index(line, ":")
if idx == -1 {
continue
}
directive := strings.ToLower(strings.TrimSpace(line[:idx]))
value := strings.TrimSpace(line[idx+1:])
switch directive {
case "user-agent":
if len(currentAgents) > 0 && !matchingAgents {
currentAgents = nil
}
agent := strings.ToLower(value)
currentAgents = append(currentAgents, agent)
matchingAgents = agent == userAgent || agent == "*"
case "disallow":
if matchingAgents && value != "" {
policy.disallowed = append(policy.disallowed, value)
}
case "allow":
if matchingAgents && value != "" {
policy.allowedPaths = append(policy.allowedPaths, value)
}
case "crawl-delay":
if matchingAgents {
if delay, err := strconv.ParseFloat(value, 64); err == nil {
policy.crawlDelay = time.Duration(delay * float64(time.Second))
}
}
}
}
if err := scanner.Err(); err != nil {
// Scanner error, return what we parsed so far
}
return policy
}
// Mirror manages the website mirroring process.
type Mirror struct {
config *MirrorConfig
proto core.Protocol
httpClient *http.Client
filter *recursive.URLFilter
rewriter *linkrewrite.Rewriter
robotsChecker *RobotsChecker
visited map[string]bool
visitedMu sync.Mutex
queue chan *mirrorTask
stats *MirrorStats
wg sync.WaitGroup
ctx context.Context
cancel context.CancelFunc
}
// mirrorTask represents a task for the mirror
type mirrorTask struct {
url *url.URL
depth int
}
// NewMirror creates new mirror instance
func NewMirror(cfg *MirrorConfig, proto core.Protocol) *Mirror {
ctx, cancel := context.WithCancel(context.Background())
filter := recursive.NewURLFilter(cfg.BaseURL, recursive.URLFilterConfig{
MaxDepth: cfg.MaxDepth,
FollowExternal: cfg.FollowExternal,
ExcludePatterns: cfg.ExcludePatterns,
IncludePatterns: cfg.IncludePatterns,
})
rewriter := linkrewrite.New(cfg.BaseURL, cfg.OutputDir)
robotsChecker := NewRobotsChecker(cfg.UserAgent)
// Create HTTP client for robots.txt fetching
httpClient := &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
TLSHandshakeTimeout: 5 * time.Second,
},
}
mirror := &Mirror{
config: cfg,
proto: proto,
httpClient: httpClient,
filter: filter,
rewriter: rewriter,
robotsChecker: robotsChecker,
visited: make(map[string]bool),
queue: make(chan *mirrorTask, 500), // Larger buffer
stats: &MirrorStats{StartTime: time.Now()},
ctx: ctx,
cancel: cancel,
}
return mirror
}
// Download starts the mirroring process and returns the final statistics.
func (m *Mirror) Download(ctx context.Context) (*MirrorStats, error) {
// Create output directory
if err := os.MkdirAll(m.config.OutputDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create output directory: %w", err)
}
// Fetch robots.txt if enabled
if m.config.RespectRobots {
if err := m.robotsChecker.FetchPolicy(ctx, m.config.BaseURL, m.httpClient); err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[mirror] Warning: failed to fetch robots.txt: %v\n", err)
}
}
}
// Add initial URL to queue
m.visitedMu.Lock()
m.visited[m.config.BaseURL.String()] = true
m.visitedMu.Unlock()
m.wg.Add(1)
select {
case m.queue <- &mirrorTask{url: m.config.BaseURL, depth: 0}:
case <-ctx.Done():
m.wg.Done()
return m.stats, ctx.Err()
}
// Start worker goroutines
workers := m.config.Parallel
if workers <= 0 {
workers = 8 // More workers for mirror mode
}
for i := 0; i < workers; i++ {
go m.worker()
}
// Close queue on context cancellation to unblock workers
go func() {
<-m.ctx.Done()
close(m.queue)
}()
// Wait for all tasks to complete
m.wg.Wait()
close(m.queue)
m.stats.EndTime = time.Now()
// Prune empty directories if requested
if m.config.PruneEmpty {
m.pruneEmptyDirectories()
}
return m.stats, nil
}
// worker processes mirror tasks from the queue.
func (m *Mirror) worker() {
for {
select {
case task, ok := <-m.queue:
if !ok {
return
}
if m.ctx.Err() != nil {
m.wg.Done()
continue
}
m.processTask(task)
case <-m.ctx.Done():
return
}
}
}
// processTask handles a single mirror task (download, link extraction, etc.).
func (m *Mirror) processTask(task *mirrorTask) {
defer m.wg.Done()
// Check if cancelled
if m.ctx.Err() != nil {
return
}
// Check robots.txt
if m.config.RespectRobots && !m.robotsChecker.CanFetch(task.url) {
atomic.AddInt64(&m.stats.SkippedURLs, 1)
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[skip] %s (robots.txt)\n", task.url.String())
}
return
}
// Apply rate limiting
if m.config.RateLimiter != nil {
m.config.RateLimiter.Allow(1)
}
// Apply per-host crawl delay
if delay := m.robotsChecker.GetCrawlDelay(task.url); delay > 0 {
select {
case <-time.After(delay):
case <-m.ctx.Done():
return
}
}
outputPath := recursive.GetLocalPath(m.config.BaseURL, task.url, m.config.OutputDir)
if m.config.DryRun {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[dry-run] %s (depth %d)\n", task.url.String(), task.depth)
}
atomic.AddInt64(&m.stats.SkippedURLs, 1)
return
}
// Download the URL
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[download] %s (depth %d)\n", task.url.String(), task.depth)
}
// Create parent directory
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
atomic.AddInt64(&m.stats.FailedURLs, 1)
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to create directory: %v\n", err)
}
return
}
// Create download request
req := &core.DownloadRequest{
URL: task.url,
Output: outputPath,
Resume: true,
Timeout: m.config.Timeout,
Verbose: false,
AutoDecompress: false, // Don't decompress HTML files
Headers: map[string]string{
"User-Agent": m.config.UserAgent,
},
Ctx: m.ctx,
}
// Execute download
result, err := m.proto.Download(m.ctx, req)
if err != nil {
atomic.AddInt64(&m.stats.FailedURLs, 1)
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] %s: %v\n", task.url.String(), err)
}
return
}
atomic.AddInt64(&m.stats.DownloadedURLs, 1)
atomic.AddInt64(&m.stats.TotalBytes, result.BytesDownloaded)
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[done] %s -> %s (%s)\n", task.url.String(), outputPath, format.Bytes(result.BytesDownloaded))
}
// Register downloaded URL for link rewriting
m.rewriter.Register(task.url, outputPath)
// Process content based on type
contentType := m.detectContentType(outputPath, task.url.Path)
switch contentType {
case "html":
// Extract links and queue them
m.extractAndQueue(task.url, outputPath, task.depth+1)
// Convert links if enabled
if m.config.ConvertLinks {
m.convertLinksInFile(outputPath)
}
case "css":
// Extract URLs from CSS and queue them
if m.config.DownloadAssets {
m.extractCSSURLs(task.url, outputPath, task.depth+1)
}
}
}
// detectContentType determines the content type based on file extension and content sniffing.
func (m *Mirror) detectContentType(outputPath, urlPath string) string {
ext := strings.ToLower(filepath.Ext(outputPath))
// Check extension first
switch ext {
case ".html", ".htm", ".php", ".asp", ".aspx", ".xhtml":
return "html"
case ".css":
return "css"
case ".js", ".mjs":
return "js"
case ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp", ".bmp", ".avif":
return "image"
case ".woff", ".woff2", ".ttf", ".eot", ".otf":
return "font"
case ".xml", ".rss", ".atom":
return "xml"
case ".json":
return "json"
}
// Check if it's a directory index
if ext == "" && (urlPath == "" || urlPath == "/" || strings.HasSuffix(urlPath, "/")) {
return "html"
}
// Try content detection
data, err := os.ReadFile(outputPath)
if err != nil || len(data) == 0 {
return "unknown"
}
snippet := data
if len(snippet) > 512 {
snippet = snippet[:512]
}
content := strings.ToLower(string(snippet))
// HTML detection
if strings.Contains(content, "<!doctype html") ||
strings.Contains(content, "<html") ||
strings.Contains(content, "<head") ||
strings.Contains(content, "<body") {
return "html"
}
// CSS detection
if strings.Contains(content, "{") && strings.Contains(content, "}") &&
(strings.Contains(content, ":") || strings.Contains(content, "@")) {
return "css"
}
// XML detection
if strings.HasPrefix(content, "<?xml") || strings.HasPrefix(content, "<rss") {
return "xml"
}
return "unknown"
}
// extractAndQueue extracts links from HTML and adds new URLs to the processing queue.
func (m *Mirror) extractAndQueue(baseURL *url.URL, outputPath string, depth int) {
data, err := os.ReadFile(outputPath)
if err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to read %s for link extraction: %v\n", outputPath, err)
}
return
}
links, err := recursive.ExtractLinks(baseURL, data)
if err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to parse links from %s: %v\n", outputPath, err)
}
return
}
added := 0
for _, link := range links {
// Check filter first (quick rejection)
if !m.filter.ShouldDownload(link, depth) {
continue
}
// Atomic check-and-add for visited
m.visitedMu.Lock()
if m.visited[link.String()] {
m.visitedMu.Unlock()
continue
}
m.visited[link.String()] = true
m.visitedMu.Unlock()
// Add to queue with context-aware send
m.wg.Add(1)
select {
case m.queue <- &mirrorTask{url: link, depth: depth}:
added++
case <-m.ctx.Done():
m.wg.Done()
return
}
}
if m.config.Verbose && added > 0 {
fmt.Fprintf(os.Stderr, "[links] queued %d new URLs from %s\n", added, outputPath)
}
}
// extractCSSURLs extracts URLs from CSS files and queues them for download.
func (m *Mirror) extractCSSURLs(baseURL *url.URL, cssPath string, depth int) {
data, err := os.ReadFile(cssPath)
if err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to read CSS %s: %v\n", cssPath, err)
}
return
}
urls := recursive.ExtractCSSURLs(baseURL, data)
// Queue found URLs
for _, resolved := range urls {
// Check filter
if !m.filter.ShouldDownload(resolved, depth) {
continue
}
// Atomic check-and-add
m.visitedMu.Lock()
if m.visited[resolved.String()] {
m.visitedMu.Unlock()
continue
}
m.visited[resolved.String()] = true
m.visitedMu.Unlock()
m.wg.Add(1)
select {
case m.queue <- &mirrorTask{url: resolved, depth: depth}:
case <-m.ctx.Done():
m.wg.Done()
return
}
}
}
// convertLinksInFile rewrites URLs in an HTML file for local viewing.
func (m *Mirror) convertLinksInFile(filePath string) {
data, err := os.ReadFile(filePath)
if err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to read %s for link conversion: %v\n", filePath, err)
}
return
}
doc, err := html.Parse(bytes.NewReader(data))
if err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to parse HTML %s: %v\n", filePath, err)
}
return
}
converted := m.rewriter.RewriteLinks(doc, filePath)
if converted > 0 {
// Write modified HTML
var buf bytes.Buffer
if err := html.Render(&buf, doc); err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to render HTML %s: %v\n", filePath, err)
}
return
}
if err := os.WriteFile(filePath, buf.Bytes(), 0644); err != nil {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[error] failed to write modified HTML %s: %v\n", filePath, err)
}
return
}
atomic.AddInt64(&m.stats.ConvertedLinks, int64(converted))
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[convert] converted %d links in %s\n", converted, filePath)
}
}
}
// pruneEmptyDirectories removes empty directories after mirroring is complete.
func (m *Mirror) pruneEmptyDirectories() {
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[prune] removing empty directories...\n")
}
// Collect all directories
dirs := make([]string, 0)
filepath.Walk(m.config.OutputDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if info.IsDir() && path != m.config.OutputDir {
dirs = append(dirs, path)
}
return nil
})
// Sort by depth (deepest first) for single-pass removal
sort.Slice(dirs, func(i, j int) bool {
return strings.Count(dirs[i], string(filepath.Separator)) > strings.Count(dirs[j], string(filepath.Separator))
})
// Remove empty directories bottom-up
removed := 0
for _, dir := range dirs {
entries, err := os.ReadDir(dir)
if err == nil && len(entries) == 0 {
if err := os.Remove(dir); err == nil {
removed++
if m.config.Verbose {
fmt.Fprintf(os.Stderr, "[prune] removed empty directory: %s\n", dir)
}
}
}
}
if m.config.Verbose && removed > 0 {
fmt.Fprintf(os.Stderr, "[prune] removed %d empty directories\n", removed)
}
}
// Cancel stops the mirroring process.
func (m *Mirror) Cancel() {
m.cancel()
}
// GetStats returns the current mirroring statistics.
func (m *Mirror) GetStats() *MirrorStats {
return m.stats
}
// GetStatsSummary returns a human-readable summary of mirroring statistics.
func (s *MirrorStats) GetStatsSummary() string {
duration := s.EndTime.Sub(s.StartTime)
if duration == 0 {
duration = time.Since(s.StartTime)
}
speed := float64(0)
if duration.Seconds() > 0 {
speed = float64(atomic.LoadInt64(&s.TotalBytes)) / duration.Seconds()
}
return fmt.Sprintf("Downloaded: %d files, %s total, %d links converted, %v elapsed, %.1f KB/s avg",
atomic.LoadInt64(&s.DownloadedURLs),
format.Bytes(atomic.LoadInt64(&s.TotalBytes)),
atomic.LoadInt64(&s.ConvertedLinks),
duration.Round(time.Second),
speed/1024)
}
internal/mirror/mirror_test.go
+166
View File
@@ -0,0 +1,166 @@
//go:build linux || freebsd
// +build linux freebsd
package mirror
import (
"net/url"
"testing"
"time"
)
func TestNewRobotsChecker(t *testing.T) {
t.Run("creates checker with user agent", func(t *testing.T) {
ua := "MyBot/1.0"
rc := NewRobotsChecker(ua)
if rc == nil {
t.Fatal("NewRobotsChecker returned nil")
}
if rc.userAgent != ua {
t.Errorf("userAgent = %q; want %q", rc.userAgent, ua)
}
if rc.policies == nil {
t.Error("policies map should be initialized")
}
})
t.Run("creates checker with empty user agent", func(t *testing.T) {
rc := NewRobotsChecker("")
if rc == nil {
t.Fatal("NewRobotsChecker('') returned nil")
}
if rc.userAgent != "" {
t.Errorf("userAgent = %q; want empty string", rc.userAgent)
}
})
}
func TestRobotsCheckerCanFetch(t *testing.T) {
t.Run("no policy returns true", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/page")
if !rc.CanFetch(u) {
t.Error("CanFetch with no policy should return true")
}
})
t.Run("empty policy returns true", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/page")
// Add an empty policy directly
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{}
rc.mu.Unlock()
if !rc.CanFetch(u) {
t.Error("CanFetch with empty policy should return true")
}
})
t.Run("disallowed path returns false", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/private/data")
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{
disallowed: []string{"/private"},
}
rc.mu.Unlock()
if rc.CanFetch(u) {
t.Error("CanFetch with disallowed path should return false")
}
})
t.Run("allowed path returns true even with disallowed rules", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/public/data")
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{
disallowed: []string{"/private"},
}
rc.mu.Unlock()
if !rc.CanFetch(u) {
t.Error("CanFetch with non-disallowed path should return true")
}
})
t.Run("root path with empty string path in URL", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
// URL with empty path
u, _ := url.Parse("https://example.com")
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{
disallowed: []string{"/"},
}
rc.mu.Unlock()
if rc.CanFetch(u) {
t.Error("CanFetch with disallowed root should return false")
}
})
t.Run("different host uses different policy", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
uAllowed, _ := url.Parse("https://allowed.com/page")
uDisallowed, _ := url.Parse("https://disallowed.com/secret")
rc.mu.Lock()
rc.policies["allowed.com"] = &robotsPolicy{}
rc.policies["disallowed.com"] = &robotsPolicy{
disallowed: []string{"/secret"},
}
rc.mu.Unlock()
if !rc.CanFetch(uAllowed) {
t.Error("CanFetch on allowed host should return true")
}
if rc.CanFetch(uDisallowed) {
t.Error("CanFetch on disallowed path should return false")
}
})
}
func TestRobotsCheckerGetCrawlDelay(t *testing.T) {
t.Run("no policy returns zero", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/")
if d := rc.GetCrawlDelay(u); d != 0 {
t.Errorf("GetCrawlDelay without policy = %v; want 0", d)
}
})
t.Run("returns configured delay", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/")
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{
crawlDelay: 5 * time.Second,
}
rc.mu.Unlock()
if d := rc.GetCrawlDelay(u); d != 5*time.Second {
t.Errorf("GetCrawlDelay = %v; want 5s", d)
}
})
t.Run("zero delay returns zero", func(t *testing.T) {
rc := NewRobotsChecker("Bot")
u, _ := url.Parse("https://example.com/")
rc.mu.Lock()
rc.policies["example.com"] = &robotsPolicy{
crawlDelay: 0,
}
rc.mu.Unlock()
if d := rc.GetCrawlDelay(u); d != 0 {
t.Errorf("GetCrawlDelay with 0 delay = %v; want 0", d)
}
})
}
internal/output/atomic.go
+29
View File
@@ -0,0 +1,29 @@
//go:build linux || freebsd
// +build linux freebsd
package output
import (
"os"
"path/filepath"
)
// CreateTempFile creates a temporary file in the specified directory for atomic writes.
func CreateTempFile(dir, pattern string) (*os.File, error) {
return os.CreateTemp(dir, pattern)
}
// AtomicReplace atomically renames a temporary file to its final destination.
func AtomicReplace(tempPath, finalPath string) error {
return os.Rename(tempPath, finalPath)
}
// CleanupTempFile removes a temporary file at the given path.
func CleanupTempFile(path string) error {
return os.Remove(path)
}
// GetTempDir returns the directory portion of the given path for temporary file placement.
func GetTempDir(path string) string {
return filepath.Dir(path)
}
internal/output/output_test.go
+1500
View File
File diff suppressed because it is too large Load Diff
internal/output/resume.go
+154
View File
@@ -0,0 +1,154 @@
//go:build linux || freebsd
// +build linux freebsd
package output
import (
"encoding/json"
"fmt"
"os"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
)
// ResumeMetaFileSuffix is the file suffix used for resume metadata files.
const ResumeMetaFileSuffix = ".goget.meta"
// ResumeMetadata holds metadata for resuming interrupted downloads.
type ResumeMetadata struct {
URL string `json:"url"`
ETag string `json:"etag,omitempty"`
LastModified string `json:"last_modified,omitempty"`
Downloaded int64 `json:"downloaded"`
Total int64 `json:"total,omitempty"`
LastWrite time.Time `json:"last_write"`
Version string `json:"version"`
Chunks map[int]int64 `json:"chunks,omitempty"` // per-chunk progress for parallel downloads
}
// NewResumeMetadata creates a new ResumeMetadata instance with the given parameters.
func NewResumeMetadata(url, etag, lastModified string, downloaded, total int64) *ResumeMetadata {
return &ResumeMetadata{
URL: url,
ETag: etag,
LastModified: lastModified,
Downloaded: downloaded,
Total: total,
LastWrite: time.Now(),
Version: "1.0",
}
}
// Save writes the metadata to a JSON file alongside the downloaded file.
func (rm *ResumeMetadata) Save(outputPath string) error {
metaPath := outputPath + ResumeMetaFileSuffix
data, err := json.MarshalIndent(rm, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal meta: %w", err)
}
return os.WriteFile(metaPath, data, 0600)
}
// LoadResumeMetadata reads resume metadata from the associated JSON file.
func LoadResumeMetadata(outputPath string) (*ResumeMetadata, error) {
metaPath := outputPath + ResumeMetaFileSuffix
data, err := os.ReadFile(metaPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("failed to read meta: %w", err)
}
var rm ResumeMetadata
if err := json.Unmarshal(data, &rm); err != nil {
return nil, fmt.Errorf("failed to parse metadata: %w", err)
}
return &rm, nil
}
// DeleteResumeMetadata removes the metadata file associated with the given output path.
func DeleteResumeMetadata(outputPath string) error {
metaPath := outputPath + ResumeMetaFileSuffix
if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// CanResume checks whether a download can be resumed by validating the metadata file.
// It returns true only if valid metadata exists and matches the requested URL.
func CanResume(outputPath string, reqURL string) (bool, *ResumeMetadata, error) {
// Check if output file exists
info, err := os.Stat(outputPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil, nil
}
return false, nil, err
}
// Load metadata
meta, err := LoadResumeMetadata(outputPath)
if err != nil {
return false, nil, err
}
// Without valid metadata, resume is not safe (file may be from another download or corrupted)
if meta == nil {
return false, nil, nil
}
// Verify URL matches
if meta.URL != reqURL {
return false, nil, fmt.Errorf("url mismatch: metadata for %s, requested %s", meta.URL, reqURL)
}
// Verify file size matches
if info.Size() != meta.Downloaded {
return false, nil, fmt.Errorf("size mismatch: file has %d bytes, metadata expects %d", info.Size(), meta.Downloaded)
}
return true, meta, nil
}
// GetResumeInfo creates a ResumeInfo from the metadata for use in download resumption.
func GetResumeInfo(meta *ResumeMetadata, outputPath string) (*core.ResumeInfo, error) {
info, err := os.Stat(outputPath)
if err != nil {
return nil, err
}
return &core.ResumeInfo{
DownloadedBytes: meta.Downloaded,
ETag: meta.ETag,
LastModified: meta.LastModified,
URL: meta.URL,
LastWrite: info.ModTime(),
}, nil
}
// FileExists checks whether a file exists at the given path.
func FileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// GetFileSize returns the size of the file at the given path.
func GetFileSize(path string) (int64, error) {
info, err := os.Stat(path)
if err != nil {
return 0, err
}
return info.Size(), nil
}
// CleanupResumeFiles removes the downloaded file and its associated metadata file.
func CleanupResumeFiles(outputPath string) error {
if err := os.Remove(outputPath); err != nil && !os.IsNotExist(err) {
return err
}
return DeleteResumeMetadata(outputPath)
}
internal/output/writer.go
+303
View File
@@ -0,0 +1,303 @@
//go:build linux || freebsd
// +build linux freebsd
package output
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"syscall"
"time"
)
// WriterConfig holds configuration for the Writer.
type WriterConfig struct {
// Output is the destination file path (use "-" for stdout).
Output string
// BufferSize is the size of the write buffer.
BufferSize int
// Atomic enables atomic write via temp file + rename.
Atomic bool
// Resume enables support for resuming interrupted downloads.
Resume bool
// ProgressCallback is called on each write with the current progress.
ProgressCallback func(current, total int64, speed float64)
// Verbose enables verbose logging.
Verbose bool
// CreateDirs creates missing parent directories of Output via os.MkdirAll
// before opening the file. Ignored when Output is "-" (stdout). The
// equivalent of curl --create-dirs / wget --directory-prefix.
CreateDirs bool
}
// DefaultWriterConfig returns a WriterConfig with sensible defaults.
func DefaultWriterConfig() *WriterConfig {
return &WriterConfig{
BufferSize: 32 * 1024, // 32KB
Atomic: true,
Resume: false,
Verbose: false,
}
}
// Writer is a progress-aware file writer supporting atomic writes and resume.
type Writer struct {
config *WriterConfig
file *os.File
tempFile *os.File
written int64
total int64
startTime time.Time
mu sync.Mutex
callback func(current, total int64, speed float64)
ctx context.Context
cancel context.CancelFunc
}
// NewWriter creates a new writer
func NewWriter(cfg *WriterConfig) (*Writer, error) {
if cfg == nil {
cfg = DefaultWriterConfig()
}
w := &Writer{
config: cfg,
written: 0,
startTime: time.Now(),
callback: cfg.ProgressCallback,
}
// Create context for cancellation.
w.ctx, w.cancel = context.WithCancel(context.Background())
// Determine output path
outputPath := cfg.Output
if outputPath == "" {
outputPath = "-" // stdout
}
// Create parent directories on demand (curl --create-dirs / wget
// --directory-prefix). Skip for stdout where there is no path.
if cfg.CreateDirs && outputPath != "-" {
dir := filepath.Dir(outputPath)
if dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create output directory: %w", err)
}
}
}
// Atomic write - create temp file
if cfg.Atomic && outputPath != "-" {
dir := filepath.Dir(outputPath)
name := filepath.Base(outputPath)
tempName := fmt.Sprintf(".%s.tmp.%d", name, time.Now().UnixNano())
tempPath := filepath.Join(dir, tempName)
tempFile, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
w.tempFile = tempFile
w.file = tempFile
} else if outputPath == "-" {
w.file = os.Stdout
} else {
// Check if file exists for resume.
if cfg.Resume {
if info, err := os.Stat(outputPath); err == nil {
w.written = info.Size()
// Open in append mode
file, err := os.OpenFile(outputPath, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return nil, fmt.Errorf("failed to open file for resume: %w", err)
}
w.file = file
} else {
// File doesn't exist, create new
file, err := os.Create(outputPath)
if err != nil {
return nil, fmt.Errorf("failed to create file: %w", err)
}
w.file = file
}
} else {
// Check if file exists
if _, err := os.Stat(outputPath); err == nil {
return nil, fmt.Errorf("file already exists: %s", outputPath)
}
file, err := os.Create(outputPath)
if err != nil {
return nil, fmt.Errorf("failed to create file: %w", err)
}
w.file = file
}
}
return w, nil
}
// Write implements the io.Writer interface.
func (w *Writer) Write(p []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
n, err := w.file.Write(p)
if err != nil {
return n, err
}
w.written += int64(n)
// Report progress
if w.callback != nil {
duration := time.Since(w.startTime)
speed := float64(w.written) / duration.Seconds()
w.callback(w.written, w.total, speed)
}
return n, nil
}
// SetTotal sets the expected total size for progress tracking.
func (w *Writer) SetTotal(total int64) {
w.mu.Lock()
defer w.mu.Unlock()
w.total = total
}
// SetProgressCallback sets the callback function for progress reporting.
func (w *Writer) SetProgressCallback(cb func(current, total int64, speed float64)) {
w.mu.Lock()
defer w.mu.Unlock()
w.callback = cb
}
// Written returns the number of bytes written so far.
func (w *Writer) Written() int64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.written
}
// Total returns the expected total size.
func (w *Writer) Total() int64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.total
}
// Close finalizes the write and performs the atomic rename if applicable.
func (w *Writer) Close() error {
w.cancel()
if w.file == nil {
return nil
}
// Don't close stdout — it's shared across the process
if w.file != os.Stdout {
if err := w.file.Close(); err != nil {
return err
}
}
// Atomic rename if temp file was used
if w.tempFile != nil && w.config.Output != "-" {
tempPath := w.tempFile.Name()
if err := os.Rename(tempPath, w.config.Output); err != nil {
if errors.Is(err, syscall.EXDEV) {
// Cross-filesystem rename is not supported by the OS.
// Fall back to a copy + remove so the data still reach
// the destination directory instead of being lost.
if fallbackErr := copyFileAndRemove(tempPath, w.config.Output); fallbackErr != nil {
return fmt.Errorf("atomic rename failed across filesystems and fallback copy failed: rename=%v copy=%w", err, fallbackErr)
}
return nil
}
// Any other error (permission denied, target is a directory,
// target busy on Windows, etc.) — keep the temp file so the
// user can recover the partial download manually.
return fmt.Errorf("failed to rename temp file %q to %q: %w (temp file kept for recovery)", tempPath, w.config.Output, err)
}
}
return nil
}
// copyFileAndRemove copies src to dst (truncating any existing file) and
// removes src on success. Used as a fallback for os.Rename when the temp
// file and the destination are on different filesystems.
func copyFileAndRemove(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
return os.Remove(src)
}
// Cancel aborts the write and cleans up any temporary files.
func (w *Writer) Cancel() {
w.cancel()
if w.tempFile != nil {
// Remove temp file on cancel
tempPath := w.tempFile.Name()
w.tempFile.Close()
os.Remove(tempPath)
}
}
// GetSpeed returns the average write speed in bytes per second.
func (w *Writer) GetSpeed() float64 {
w.mu.Lock()
defer w.mu.Unlock()
duration := time.Since(w.startTime)
if duration.Seconds() == 0 {
return 0
}
return float64(w.written) / duration.Seconds()
}
// GetDuration returns the elapsed time since the writer was created.
func (w *Writer) GetDuration() time.Duration {
return time.Since(w.startTime)
}
// GetProgress returns the progress as a percentage of the total size.
func (w *Writer) GetProgress() float64 {
w.mu.Lock()
defer w.mu.Unlock()
if w.total == 0 {
return 0
}
return float64(w.written) / float64(w.total) * 100
}
internal/output/writer_test.go
+232
View File
@@ -0,0 +1,232 @@
//go:build linux || freebsd
// +build linux freebsd
package output
import (
"os"
"path/filepath"
"strings"
"testing"
)
// TestCloseAtomicSuccess verifies the happy path: temp file is renamed to
// the final destination and no temp file remains.
func TestCloseAtomicSuccess(t *testing.T) {
dir := tempDir(t)
outputPath := filepath.Join(dir, "output.txt")
w, err := NewWriter(&WriterConfig{
Output: outputPath,
Atomic: true,
})
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
payload := []byte("hello world")
if _, err := w.Write(payload); err != nil {
t.Fatalf("Write: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
got, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
if string(got) != string(payload) {
t.Errorf("payload = %q, want %q", got, payload)
}
// No leftover temp files in the directory.
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("ReadDir: %v", err)
}
for _, e := range entries {
if strings.Contains(e.Name(), ".tmp.") {
t.Errorf("temp file %q should have been renamed away", e.Name())
}
}
}
// TestCloseKeepsTempOnRenameError is a regression guard for the BACKLOG
// entry "Atomic write data loss on cross-filesystem rename". The previous
// implementation called os.Remove(tempPath) on any rename error, deleting
// the only copy of the downloaded data. After the fix, the temp file must
// remain on disk so the user can recover the partial download manually.
//
// We force a rename error by pre-creating the destination as a directory —
// os.Rename will refuse to overwrite a non-empty directory with a file.
func TestCloseKeepsTempOnRenameError(t *testing.T) {
dir := tempDir(t)
outputPath := filepath.Join(dir, "output.txt")
// Block the rename by making the target path a directory.
if err := os.Mkdir(outputPath, 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
w, err := NewWriter(&WriterConfig{
Output: outputPath,
Atomic: true,
})
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
payload := []byte("recoverable download data")
if _, err := w.Write(payload); err != nil {
t.Fatalf("Write: %v", err)
}
tempPath := w.tempFile.Name()
if err := w.Close(); err == nil {
t.Fatal("expected Close() to return an error when target is a directory")
}
// The temp file MUST still exist — the data must be recoverable.
if _, err := os.Stat(tempPath); err != nil {
t.Errorf("temp file %q should still exist after Close error, got: %v", tempPath, err)
}
// And it should still hold the full payload.
got, err := os.ReadFile(tempPath)
if err != nil {
t.Fatalf("ReadFile(temp): %v", err)
}
if string(got) != string(payload) {
t.Errorf("temp file payload = %q, want %q", got, payload)
}
}
// TestCopyFileAndRemove covers the EXDEV fallback helper directly. We do
// not have an easy way to mount two different filesystems in a unit test,
// so we just verify the copy + remove behaviour on the same filesystem.
func TestCopyFileAndRemove(t *testing.T) {
dir := tempDir(t)
src := filepath.Join(dir, "src.tmp")
dst := filepath.Join(dir, "dst.bin")
payload := []byte("cross-fs fallback payload")
if err := os.WriteFile(src, payload, 0644); err != nil {
t.Fatalf("WriteFile(src): %v", err)
}
if err := copyFileAndRemove(src, dst); err != nil {
t.Fatalf("copyFileAndRemove: %v", err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile(dst): %v", err)
}
if string(got) != string(payload) {
t.Errorf("dst = %q, want %q", got, payload)
}
if _, err := os.Stat(src); !os.IsNotExist(err) {
t.Errorf("src should be removed, got stat err = %v", err)
}
}
// TestCopyFileAndRemoveOverwritesExisting verifies the fallback truncates
// any existing file at the destination (matching os.Rename semantics).
func TestCopyFileAndRemoveOverwritesExisting(t *testing.T) {
dir := tempDir(t)
src := filepath.Join(dir, "src.tmp")
dst := filepath.Join(dir, "dst.bin")
if err := os.WriteFile(src, []byte("new"), 0644); err != nil {
t.Fatalf("WriteFile(src): %v", err)
}
if err := os.WriteFile(dst, []byte("OLD CONTENT"), 0644); err != nil {
t.Fatalf("WriteFile(dst): %v", err)
}
if err := copyFileAndRemove(src, dst); err != nil {
t.Fatalf("copyFileAndRemove: %v", err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile(dst): %v", err)
}
if string(got) != "new" {
t.Errorf("dst = %q, want %q", got, "new")
}
}
// TestNewWriterCreateDirsMissingParent verifies the curl --create-dirs
// equivalent: when CreateDirs is true and the parent directory of Output
// does not exist, NewWriter must create the full chain via MkdirAll and
// successfully write the file. This is the happy path for "goget
// --output ./data/archives/file.zip" against a fresh tree.
func TestNewWriterCreateDirsMissingParent(t *testing.T) {
dir := tempDir(t)
outputPath := filepath.Join(dir, "deeply", "nested", "subdir", "file.dat")
w, err := NewWriter(&WriterConfig{
Output: outputPath,
Atomic: true,
CreateDirs: true,
})
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
payload := []byte("created in a fresh dir tree")
if _, err := w.Write(payload); err != nil {
t.Fatalf("Write: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
got, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
if string(got) != string(payload) {
t.Errorf("payload = %q, want %q", got, payload)
}
}
// TestNewWriterCreateDirsDisabledLeavesError confirms that without
// CreateDirs the writer still surfaces the underlying "parent dir missing"
// error — CreateDirs is opt-in, not a silent behaviour change for existing
// users.
func TestNewWriterCreateDirsDisabledLeavesError(t *testing.T) {
dir := tempDir(t)
outputPath := filepath.Join(dir, "missing", "file.dat")
_, err := NewWriter(&WriterConfig{
Output: outputPath,
Atomic: true,
CreateDirs: false,
})
if err == nil {
t.Fatal("expected NewWriter to fail when parent dir is missing and CreateDirs=false")
}
}
// TestNewWriterCreateDirsIgnoredForStdout makes sure the - / stdout path is
// untouched by the new flag: nothing to create, no spurious error.
func TestNewWriterCreateDirsIgnoredForStdout(t *testing.T) {
w, err := NewWriter(&WriterConfig{
Output: "-",
Atomic: false,
CreateDirs: true,
})
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
if w.file != os.Stdout {
t.Errorf("expected writer to point at stdout, got %v", w.file)
}
w.Close()
}
internal/protocol/dataurl/dataurl.go
+105
View File
@@ -0,0 +1,105 @@
//go:build linux || freebsd
// +build linux freebsd
package dataurl
import (
"context"
"encoding/base64"
"io"
"net/url"
"os"
"strings"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Protocol handles data: URLs (RFC 2397).
type Protocol struct {
*protocol.BaseProtocol
}
// NewProtocol creates a new data: URL protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "DATA",
Scheme: "data",
DefaultPort: 0,
Features: []string{},
}),
}
}
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
startTime := time.Now()
// data:[<mediatype>][;base64],<data>
data, err := Decode(req.URL.String())
if err != nil {
return nil, core.NewProtocolError("failed to decode data URL", err, core.SafeURL(req.URL))
}
var writer io.Writer
outputPath := req.Output
if req.Writer != nil {
writer = req.Writer
} else if outputPath == "" || outputPath == "-" {
writer = os.Stdout
outputPath = "-"
} else {
f, err := os.Create(outputPath)
if err != nil {
return nil, core.NewFileError("failed to create output file", err)
}
defer f.Close()
writer = f
}
n, err := writer.Write(data)
if err != nil {
return nil, core.NewFileError("failed to write data URL content", err)
}
duration := time.Since(startTime)
return &core.DownloadResult{
BytesDownloaded: int64(n),
TotalSize: int64(len(data)),
Duration: duration,
Protocol: "DATA",
IPVersion: 0,
Speed: float64(n) / duration.Seconds(),
OutputPath: outputPath,
}, nil
}
// Decode parses and decodes a data: URL.
func Decode(rawURL string) ([]byte, error) {
if !strings.HasPrefix(rawURL, "data:") {
return nil, nil
}
content := strings.TrimPrefix(rawURL, "data:")
isBase64 := false
if idx := strings.Index(content, ";base64,"); idx >= 0 {
isBase64 = true
content = content[idx+8:]
} else if idx := strings.Index(content, ","); idx >= 0 {
content = content[idx+1:]
}
decoded, err := url.PathUnescape(content)
if err != nil {
decoded = content
}
if isBase64 {
return base64.StdEncoding.DecodeString(decoded)
}
return []byte(decoded), nil
}
internal/protocol/dataurl/dataurl_test.go
+93
View File
@@ -0,0 +1,93 @@
//go:build linux || freebsd
// +build linux freebsd
package dataurl
import (
"bytes"
"context"
"net/url"
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
)
func TestDecodePlainText(t *testing.T) {
data, err := Decode("data:text/plain,hello%20world")
if err != nil {
t.Fatal(err)
}
if string(data) != "hello world" {
t.Errorf("got %q, want %q", data, "hello world")
}
}
func TestDecodeBase64(t *testing.T) {
data, err := Decode("data:text/plain;base64,SGVsbG8=")
if err != nil {
t.Fatal(err)
}
if string(data) != "Hello" {
t.Errorf("got %q, want %q", data, "Hello")
}
}
func TestDecodeEmpty(t *testing.T) {
data, err := Decode("data:,")
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Errorf("expected empty, got %d bytes", len(data))
}
}
func TestDecodeNotDataURL(t *testing.T) {
data, err := Decode("https://example.com")
if err != nil {
t.Fatal(err)
}
if data != nil {
t.Error("expected nil for non-data URL")
}
}
func TestDownloadPlainText(t *testing.T) {
var buf bytes.Buffer
proto := NewProtocol()
u, _ := url.Parse("data:text/plain,test123")
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Writer: &buf,
})
if err != nil {
t.Fatal(err)
}
if result.BytesDownloaded != 7 {
t.Errorf("bytes = %d, want 7", result.BytesDownloaded)
}
if buf.String() != "test123" {
t.Errorf("got %q, want %q", buf.String(), "test123")
}
}
func TestDownloadToFile(t *testing.T) {
tmp := t.TempDir()
dst := filepath.Join(tmp, "out.txt")
proto := NewProtocol()
u, _ := url.Parse("data:text/plain,hello")
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
})
if err != nil {
t.Fatal(err)
}
data, _ := os.ReadFile(dst)
if string(data) != "hello" {
t.Errorf("got %q, want %q", data, "hello")
}
}
internal/protocol/file/file.go
+211
View File
@@ -0,0 +1,211 @@
//go:build linux || freebsd
// +build linux freebsd
package file
import (
"context"
"io"
"os"
"path/filepath"
"time"
"codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/protocol"
)
// Protocol implements file:// downloads.
type Protocol struct {
*protocol.BaseProtocol
}
// NewProtocol creates a new file protocol handler.
func NewProtocol() *Protocol {
return &Protocol{
BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
Name: "FILE",
Scheme: "file",
DefaultPort: 0,
Features: []string{"resume", "recursive"},
}),
}
}
func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
startTime := time.Now()
filePath := req.URL.Path
if req.URL.Host != "" {
filePath = filepath.Join("/", req.URL.Host, filePath)
}
filePath = filepath.Clean(filePath)
info, err := os.Stat(filePath)
if err != nil {
return nil, core.NewFileError("failed to stat source path", err)
}
if info.IsDir() {
return p.downloadDirectory(ctx, req, filePath, startTime)
}
src, err := os.Open(filePath)
if err != nil {
return nil, core.NewFileError("failed to open source file", err)
}
defer src.Close()
totalSize := info.Size()
var writer io.Writer
var outputPath string
if req.Writer != nil {
writer = req.Writer
outputPath = req.Output
} else if req.Output == "" || req.Output == "-" {
writer = os.Stdout
outputPath = "-"
} else {
dst, err := os.Create(req.Output)
if err != nil {
return nil, core.NewFileError("failed to create output file", err)
}
defer dst.Close()
writer = dst
outputPath = req.Output
}
var written int64
if req.Resume && outputPath != "" && outputPath != "-" {
if fi, err := os.Stat(outputPath); err == nil {
written = fi.Size()
if written >= totalSize {
return &core.DownloadResult{
BytesDownloaded: totalSize,
TotalSize: totalSize,
Duration: time.Since(startTime),
Protocol: "FILE",
OutputPath: outputPath,
Resumed: true,
}, nil
}
if _, err := src.Seek(written, io.SeekStart); err != nil {
return nil, core.NewFileError("failed to seek for resume", err)
}
}
}
buf := make([]byte, 32*1024)
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
n, err := src.Read(buf)
if n > 0 {
if _, werr := writer.Write(buf[:n]); werr != nil {
return nil, core.NewFileError("failed to write data", werr)
}
written += int64(n)
if req.ProgressCallback != nil {
speed := float64(written) / time.Since(startTime).Seconds()
req.ProgressCallback(written, totalSize, speed)
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, core.NewFileError("failed to read source file", err)
}
}
duration := time.Since(startTime)
speed := float64(written) / duration.Seconds()
return &core.DownloadResult{
BytesDownloaded: written,
TotalSize: totalSize,
Duration: duration,
Protocol: "FILE",
Speed: speed,
OutputPath: outputPath,
Resumed: written > 0 && req.Resume,
}, nil
}
// downloadDirectory recursively copies a local directory.
func (p *Protocol) downloadDirectory(ctx context.Context, req *core.DownloadRequest, dirPath string, startTime time.Time) (*core.DownloadResult, error) {
outputBase := req.Output
if outputBase == "" {
outputBase = filepath.Base(dirPath)
}
var totalBytes int64
var fileCount int
err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
// Calculate relative path
rel, err := filepath.Rel(dirPath, path)
if err != nil {
return err
}
outputPath := filepath.Join(outputBase, rel)
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
return err
}
src, err := os.Open(path)
if err != nil {
return err
}
defer src.Close()
dst, err := os.Create(outputPath)
if err != nil {
return err
}
defer dst.Close()
n, err := io.Copy(dst, src)
if err != nil {
return err
}
totalBytes += n
fileCount++
return nil
})
if err != nil {
return nil, core.NewFileError("failed to copy directory recursively", err)
}
duration := time.Since(startTime)
speed := float64(0)
if duration.Seconds() > 0 {
speed = float64(totalBytes) / duration.Seconds()
}
return &core.DownloadResult{
BytesDownloaded: totalBytes,
TotalSize: totalBytes,
Duration: duration,
Protocol: "FILE",
Speed: speed,
OutputPath: outputBase,
ChunksCount: fileCount,
}, nil
}
internal/protocol/file/file_test.go
+129
View File
@@ -0,0 +1,129 @@
//go:build linux || freebsd
// +build linux freebsd
package file
import (
"bytes"
"context"
"net/url"
"os"
"path/filepath"
"testing"
"codeberg.org/petrbalvin/goget/internal/core"
)
func TestDownloadFile(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
dst := filepath.Join(tmp, "dst.txt")
os.WriteFile(src, []byte("hello world"), 0644)
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
})
if err != nil {
t.Fatal(err)
}
if result.BytesDownloaded != 11 {
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
}
data, _ := os.ReadFile(dst)
if string(data) != "hello world" {
t.Errorf("content = %q, want %q", data, "hello world")
}
}
func TestDownloadFileToWriter(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
os.WriteFile(src, []byte("test"), 0644)
var buf bytes.Buffer
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: "-",
Writer: &buf,
})
if err != nil {
t.Fatal(err)
}
if buf.String() != "test" {
t.Errorf("content = %q, want %q", buf.String(), "test")
}
}
func TestDownloadFileNotFound(t *testing.T) {
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: "/nonexistent/path"}
_, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: "/tmp/out",
})
if err == nil {
t.Error("expected error for nonexistent file")
}
}
func TestDownloadDirectory(t *testing.T) {
tmp := t.TempDir()
srcDir := filepath.Join(tmp, "srcdir")
dstDir := filepath.Join(tmp, "dstdir")
os.MkdirAll(filepath.Join(srcDir, "sub"), 0755)
os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("a"), 0644)
os.WriteFile(filepath.Join(srcDir, "sub/b.txt"), []byte("b"), 0644)
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: srcDir}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dstDir,
})
if err != nil {
t.Fatal(err)
}
if result.ChunksCount != 2 {
t.Errorf("files = %d, want 2", result.ChunksCount)
}
if result.BytesDownloaded != 2 {
t.Errorf("bytes = %d, want 2", result.BytesDownloaded)
}
// Verify files exist
if _, err := os.Stat(filepath.Join(dstDir, "a.txt")); err != nil {
t.Error("a.txt not found")
}
if _, err := os.Stat(filepath.Join(dstDir, "sub", "b.txt")); err != nil {
t.Error("sub/b.txt not found")
}
}
func TestResumeFile(t *testing.T) {
tmp := t.TempDir()
src := filepath.Join(tmp, "src.txt")
dst := filepath.Join(tmp, "dst.txt")
os.WriteFile(src, []byte("hello world"), 0644)
os.WriteFile(dst, []byte("hello"), 0644) // partial: 5 bytes
proto := NewProtocol()
u := &url.URL{Scheme: "file", Path: src}
result, err := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: dst,
Resume: true,
})
if err != nil {
t.Fatal(err)
}
if !result.Resumed {
t.Error("expected Resumed=true")
}
if result.BytesDownloaded != 11 {
t.Errorf("bytes = %d, want 11", result.BytesDownloaded)
}
}
internal/protocol/ftp/client.go
+817
View File
@@ -0,0 +1,817 @@
//go:build linux || freebsd
// +build linux freebsd
package ftp
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"codeberg.org/petrbalvin/goget/internal/output"
)
type Client struct {
conn net.Conn
reader *textproto.Reader
writer *bufio.Writer
host string
port int
user string
password string
url string
tlsConfig *tls.Config
useTLS bool
passive bool
timeout time.Duration
}
func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
u, err := url.Parse(rawURL)
if err != nil {
return nil, fmt.Errorf("invalid url: %w", err)
}
host := u.Hostname()
port := 21
if u.Port() != "" {
p, err := strconv.Atoi(u.Port())
if err != nil {
return nil, fmt.Errorf("invalid port: %w", err)
}
port = p
}
user := "anonymous"
password := "anonymous@"
if u.User != nil {
user = u.User.Username()
if pass, ok := u.User.Password(); ok {
password = pass
}
}
useTLS := u.Scheme == "ftps"
if !useTLS && (user != "anonymous" || (u.User != nil && u.User.Username() != "anonymous")) {
fmt.Fprintf(os.Stderr, "Warning: FTP credentials sent in plaintext. Use ftps:// for encrypted transfers.\n")
}
c := &Client{
host: host,
port: port,
user: user,
password: password,
url: rawURL,
useTLS: useTLS,
passive: true, // Default to passive mode
timeout: timeout,
tlsConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: host,
InsecureSkipVerify: false,
},
}
return c, nil
}
func (c *Client) Connect() error {
addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port))
conn, err := net.DialTimeout("tcp", addr, c.timeout)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
c.conn = conn
c.reader = textproto.NewReader(bufio.NewReader(conn))
c.writer = bufio.NewWriter(conn)
// Read greeting
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("failed to read greeting: %w", err)
}
if code != 220 {
return fmt.Errorf("unexpected greeting code: %d", code)
}
// Upgrade to TLS if FTPS (explicit)
if c.useTLS {
if err := c.sendCommand("AUTH TLS"); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("auth tls failed: %w", err)
}
if code != 234 && code != 334 {
// Try AUTH SSL as fallback
if err := c.sendCommand("AUTH SSL"); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("auth ssl failed: %w", err)
}
if code != 234 && code != 334 {
return fmt.Errorf("server does not support tls/ssl")
}
}
// Wrap connection with TLS
tlsConn := tls.Client(c.conn, c.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
return fmt.Errorf("tls handshake failed: %w", err)
}
c.conn = tlsConn
c.reader = textproto.NewReader(bufio.NewReader(tlsConn))
c.writer = bufio.NewWriter(tlsConn)
}
// Send USER
if err := c.sendCommand("USER %s", c.user); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("user command failed: %w", err)
}
// 331 means password required, 230 means already logged in
if code == 331 {
// Send PASS
if err := c.sendCommand("PASS %s", c.password); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("pass command failed: %w", err)
}
}
if code != 230 && code != 200 {
return fmt.Errorf("login failed with code: %d", code)
}
// Set binary mode
if err := c.sendCommand("TYPE I"); err != nil {
return err
}
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("type command failed: %w", err)
}
if code != 200 {
return fmt.Errorf("failed to set binary mode")
}
return nil
}
func (c *Client) sendCommand(format string, args ...interface{}) error {
cmd := fmt.Sprintf(format, args...)
if _, err := c.writer.WriteString(cmd); err != nil {
return fmt.Errorf("failed to send command: %w", err)
}
if _, err := c.writer.WriteString("\r\n"); err != nil {
return fmt.Errorf("failed to send command terminator: %w", err)
}
if err := c.writer.Flush(); err != nil {
return fmt.Errorf("failed to flush command: %w", err)
}
return nil
}
func (c *Client) readResponse() (int, string, error) {
code, msg, err := c.reader.ReadResponse(0)
if err != nil {
return 0, "", err
}
return code, msg, nil
}
func (c *Client) getFileSize(path string) (int64, error) {
if err := c.sendCommand("SIZE %s", path); err != nil {
return 0, err
}
code, msg, err := c.readResponse()
if err != nil {
return 0, fmt.Errorf("size command failed: %w", err)
}
if code != 213 {
return 0, fmt.Errorf("size command failed with code: %d", code)
}
size, err := strconv.ParseInt(strings.TrimSpace(msg), 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse size: %w", err)
}
return size, nil
}
func (c *Client) establishDataConnection() (net.Conn, error) {
if c.passive {
return c.enterPassiveMode()
}
return c.enterActiveMode()
}
func (c *Client) enterPassiveMode() (net.Conn, error) {
if err := c.sendCommand("PASV"); err != nil {
return nil, err
}
code, msg, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("pasv command failed: %w", err)
}
if code != 227 {
return nil, fmt.Errorf("pasv command failed with code: %d", code)
}
// Parse PASV response: 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2)
start := strings.Index(msg, "(")
end := strings.Index(msg, ")")
if start == -1 || end == -1 {
return nil, fmt.Errorf("invalid pasv response")
}
parts := strings.Split(msg[start+1:end], ",")
if len(parts) != 6 {
return nil, fmt.Errorf("invalid pasv response format")
}
h1, _ := strconv.Atoi(parts[0])
h2, _ := strconv.Atoi(parts[1])
h3, _ := strconv.Atoi(parts[2])
h4, _ := strconv.Atoi(parts[3])
p1, _ := strconv.Atoi(parts[4])
p2, _ := strconv.Atoi(parts[5])
dataHost := fmt.Sprintf("%d.%d.%d.%d", h1, h2, h3, h4)
dataPort := p1*256 + p2
// Use original host for TLS connections to match certificate
if c.useTLS {
dataHost = c.host
}
conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", dataHost, dataPort), c.timeout)
if err != nil {
return nil, fmt.Errorf("failed to connect to data port: %w", err)
}
if c.useTLS {
tlsConn := tls.Client(conn, c.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
conn.Close()
return nil, fmt.Errorf("tls handshake on data connection failed: %w", err)
}
return tlsConn, nil
}
return conn, nil
}
func (c *Client) enterActiveMode() (net.Conn, error) {
// Get local IP from the control connection to bind only to that interface
ctrlAddr, ok := c.conn.LocalAddr().(*net.TCPAddr)
if !ok || ctrlAddr.IP == nil {
return nil, fmt.Errorf("failed to get local address for active mode")
}
ctrlIP := ctrlAddr.IP.To4()
if ctrlIP == nil {
return nil, fmt.Errorf("no ipv4 address available for active mode")
}
// Bind listener only to the interface of the control connection, not 0.0.0.0
listener, err := net.Listen("tcp", net.JoinHostPort(ctrlIP.String(), "0"))
if err != nil {
return nil, fmt.Errorf("failed to create listener: %w", err)
}
defer listener.Close()
// Set a deadline so Accept respects the timeout
if tcpListener, ok := listener.(*net.TCPListener); ok {
tcpListener.SetDeadline(time.Now().Add(c.timeout))
}
listenAddr := listener.Addr().(*net.TCPAddr)
p1 := listenAddr.Port / 256
p2 := listenAddr.Port % 256
portCmd := fmt.Sprintf("PORT %d,%d,%d,%d,%d,%d",
ctrlIP[0], ctrlIP[1], ctrlIP[2], ctrlIP[3], p1, p2)
if err := c.sendCommand("%s", portCmd); err != nil {
return nil, err
}
code, _, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("port command failed: %w", err)
}
if code != 200 {
return nil, fmt.Errorf("port command failed with code: %d", code)
}
// Accept incoming connection
connChan := make(chan net.Conn, 1)
errChan := make(chan error, 1)
go func() {
conn, err := listener.Accept()
if err != nil {
errChan <- err
return
}
connChan <- conn
}()
select {
case conn := <-connChan:
return conn, nil
case err := <-errChan:
return nil, fmt.Errorf("failed to accept data connection: %w", err)
case <-time.After(c.timeout):
return nil, fmt.Errorf("timeout waiting for data connection")
}
}
func (c *Client) DownloadFile(ctx context.Context, remotePath, localPath string, resumeOffset int64, progressFunc func(current, total int64)) error {
// Get file size
totalSize, err := c.getFileSize(remotePath)
if err != nil {
return fmt.Errorf("failed to get file size: %w", err)
}
// Check if we need to resume
var startOffset int64 = 0
if resumeOffset > 0 && resumeOffset < totalSize {
startOffset = resumeOffset
if err := c.sendCommand("REST %d", startOffset); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("rest command failed: %w", err)
}
if code != 350 {
return fmt.Errorf("rest command failed with code: %d", code)
}
} else if resumeOffset >= totalSize {
// File already complete
return nil
}
// Open local file for writing
mode := os.O_CREATE | os.O_WRONLY
if startOffset > 0 {
mode |= os.O_APPEND
} else {
mode |= os.O_TRUNC
}
localFile, err := os.OpenFile(localPath, mode, 0644)
if err != nil {
return fmt.Errorf("failed to open local file: %w", err)
}
defer localFile.Close()
// Seek to start position if resuming
if startOffset > 0 {
if _, err := localFile.Seek(startOffset, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek: %w", err)
}
}
// Establish data connection FIRST (before RETR command)
dataConn, err := c.establishDataConnection()
if err != nil {
return fmt.Errorf("failed to establish data connection: %w", err)
}
defer dataConn.Close()
// Send RETR command AFTER data connection is established
if err := c.sendCommand("RETR %s", remotePath); err != nil {
return err
}
// Read response (should be 150 or 125)
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("retr command failed: %w", err)
}
if code != 150 && code != 125 {
// Try waiting a bit for the server to respond
time.Sleep(100 * time.Millisecond)
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("retr command failed with code: %d", code)
}
if code != 150 && code != 125 {
return fmt.Errorf("retr command failed with code: %d", code)
}
}
// Copy data with progress. Polls ctx between reads so a SIGINT
// arriving mid-transfer is detected on the next iteration and the
// partial progress is saved to ResumeMetadata before returning.
var copied int64 = startOffset
buf := make([]byte, 32*1024)
for {
if ctx.Err() != nil {
if err := saveFTPResume(c.url, localPath, copied, totalSize); err != nil {
return fmt.Errorf("failed to save resume metadata: %w", err)
}
return ctx.Err()
}
n, err := dataConn.Read(buf)
if n > 0 {
written, werr := localFile.Write(buf[:n])
copied += int64(written)
if werr != nil {
return fmt.Errorf("failed to write: %w", err)
}
if progressFunc != nil {
progressFunc(copied, totalSize)
}
}
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("failed to read from data connection: %w", err)
}
}
// Read final response
code, _, err = c.readResponse()
if err != nil {
return fmt.Errorf("failed to read final response: %w", err)
}
if code != 226 && code != 250 {
return fmt.Errorf("transfer failed with code: %d", code)
}
// Successful transfer — drop any stale resume sidecar.
_ = output.DeleteResumeMetadata(localPath)
return nil
}
func (c *Client) ListRemoteDir(path string) ([]string, error) {
// Establish data connection
dataConn, err := c.establishDataConnection()
if err != nil {
return nil, fmt.Errorf("failed to establish data connection: %w", err)
}
defer dataConn.Close()
// Send LIST command
cmd := "LIST"
if path != "" {
cmd = fmt.Sprintf("LIST %s", path)
}
if err := c.sendCommand("%s", cmd); err != nil {
return nil, err
}
// Read response (should be 150 or 125)
code, _, err := c.readResponse()
if err != nil {
return nil, fmt.Errorf("list command failed: %w", err)
}
if code != 150 && code != 125 {
return nil, fmt.Errorf("list command failed with code: %d", code)
}
// Read directory listing
var lines []string
scanner := bufio.NewScanner(dataConn)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to read directory listing: %w", err)
}
// Read final response
code, _, err = c.readResponse()
if err != nil {
return nil, fmt.Errorf("failed to read final response: %w", err)
}
if code != 226 && code != 250 {
return nil, fmt.Errorf("list command failed with code: %d", code)
}
return lines, nil
}
func (c *Client) ChangeDir(path string) error {
if err := c.sendCommand("CWD %s", path); err != nil {
return err
}
code, _, err := c.readResponse()
if err != nil {
return fmt.Errorf("cwd command failed: %w", err)
}
if code != 250 {
return fmt.Errorf("cwd command failed with code: %d", code)
}
return nil
}
func (c *Client) GetCurrentDir() (string, error) {
if err := c.sendCommand("PWD"); err != nil {
return "", err
}
code, msg, err := c.readResponse()
if err != nil {
return "", fmt.Errorf("pwd command failed: %w", err)
}
if code != 257 {
return "", fmt.Errorf("pwd command failed with code: %d", code)
}
// Extract path from response (usually quoted)
start := strings.Index(msg, "\"")
end := strings.LastIndex(msg, "\"")
if start != -1 && end > start {
return msg[start+1 : end], nil
}
return strings.TrimSpace(msg), nil
}
func (c *Client) Close() error {
if err := c.sendCommand("QUIT"); err != nil {
return err
}
_, _, _ = c.readResponse() // Ignore errors on close
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// Helper function to download entire directory recursively
func (c *Client) DownloadDirectory(ctx context.Context, remotePath, localPath string, progressFunc func(file string, current, total int64)) error {
// Create local directory
if err := os.MkdirAll(localPath, 0755); err != nil {
return fmt.Errorf("failed to create local directory: %w", err)
}
// Save current directory
origDir, err := c.GetCurrentDir()
if err != nil {
return err
}
defer func() {
c.ChangeDir(origDir)
}()
// Change to remote directory
if err := c.ChangeDir(remotePath); err != nil {
return err
}
// List directory contents
entries, err := c.ListRemoteDir("")
if err != nil {
return err
}
// Parse entries and download files
for _, entry := range entries {
fields := strings.Fields(entry)
if len(fields) < 9 {
continue
}
name := fields[len(fields)-1]
if name == "." || name == ".." {
continue
}
isDir := fields[0][0] == 'd'
remoteFilePath := filepath.Join(remotePath, name)
localFilePath := filepath.Join(localPath, name)
if isDir {
if err := c.DownloadDirectory(ctx, remoteFilePath, localFilePath, progressFunc); err != nil {
return err
}
} else {
if progressFunc != nil {
progressFunc(name, 0, 0)
}
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, nil); err != nil {
return err
}
}
}
return nil
}
// saveFTPResume persists partial FTP download progress to the standard
// `.goget.meta` sidecar so a subsequent `goget --resume` can continue
// from `downloaded` bytes. Errors are wrapped to surface failures
// without aborting the cancellation handling.
func saveFTPResume(url, localPath string, downloaded, total int64) error {
if localPath == "" || localPath == "-" || downloaded <= 0 {
return nil
}
meta := output.NewResumeMetadata(url, "", "", downloaded, total)
return meta.Save(localPath)
}
// clientPool holds N pre-connected FTP clients. A single shared pool
// removes the need to open/close connections for every file in a
// recursive download, and is required for the parallel recursive
// implementation because each *Client owns its own TCP control
// connection and is not safe for concurrent use.
type clientPool struct {
available chan *Client
all []*Client
}
// newClientPool opens `size` independent FTP connections to the same
// server. Connections are kept open until close() is called.
func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) {
if size < 1 {
size = 1
}
all := make([]*Client, 0, size)
avail := make(chan *Client, size)
for i := 0; i < size; i++ {
c, err := NewClient(rawURL, timeout)
if err != nil {
for _, existing := range all {
_ = existing.Close()
}
return nil, err
}
if err := c.Connect(); err != nil {
for _, existing := range all {
_ = existing.Close()
}
return nil, err
}
all = append(all, c)
avail <- c
}
return &clientPool{available: avail, all: all}, nil
}
// acquire blocks until a client is available, then returns it. The
// caller must call release() when done.
func (p *clientPool) acquire() *Client {
return <-p.available
}
// release returns a client to the pool. It must be called exactly once
// per acquire().
func (p *clientPool) release(c *Client) {
p.available <- c
}
// size returns the number of clients in the pool.
func (p *clientPool) size() int {
return len(p.all)
}
// close shuts down every client in the pool. The pool must not be used
// after this call.
func (p *clientPool) close() {
for _, c := range p.all {
_ = c.Close()
}
}
// DownloadDirectoryConcurrent downloads a remote directory to localPath
// using a pool of N FTP connections. All operations (LIST, RETR) use
// absolute paths so concurrent workers never share or mutate CWD state
// on the server side. The returned error is the first error encountered
// by any worker; other workers continue in the background until they
// hit ctx cancellation or finish their tasks.
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error {
if parallel < 1 {
parallel = 1
}
pool, err := newClientPool(rawURL, timeout, parallel)
if err != nil {
return err
}
defer pool.close()
sem := make(chan struct{}, parallel)
var wg sync.WaitGroup
var firstErr error
var errMu sync.Mutex
setErr := func(err error) {
errMu.Lock()
if firstErr == nil {
firstErr = err
}
errMu.Unlock()
}
processEntry(ctx, pool, sem, &wg, setErr, remotePath, localPath, progressFunc)
wg.Wait()
return firstErr
}
// processEntry processes one remote path: lists it, then dispatches
// its child entries (file → download, subdir → recursive processEntry)
// through the shared semaphore-bounded goroutine pool. A single
// sem is threaded through the recursion so the total in-flight
// goroutines never exceed `parallel`, regardless of directory depth.
func processEntry(ctx context.Context, pool *clientPool, sem chan struct{}, wg *sync.WaitGroup, setErr func(error), remotePath, localPath string, progressFunc func(file string, current, total int64)) {
defer wg.Done()
defer func() { <-sem }()
if ctx.Err() != nil {
return
}
if err := os.MkdirAll(localPath, 0755); err != nil {
setErr(fmt.Errorf("failed to create local dir %s: %w", localPath, err))
return
}
c := pool.acquire()
entries, err := c.ListRemoteDir(remotePath)
pool.release(c)
if err != nil {
setErr(fmt.Errorf("list %s: %w", remotePath, err))
return
}
for _, entry := range entries {
fields := strings.Fields(entry)
if len(fields) < 9 {
continue
}
name := fields[len(fields)-1]
if name == "." || name == ".." {
continue
}
isDir := fields[0][0] == 'd'
remoteFilePath := filepath.Join(remotePath, name)
localFilePath := filepath.Join(localPath, name)
if isDir {
wg.Add(1)
select {
case sem <- struct{}{}:
go processEntry(ctx, pool, sem, wg, setErr, remoteFilePath, localFilePath, progressFunc)
case <-ctx.Done():
return
}
} else {
wg.Add(1)
select {
case sem <- struct{}{}:
go func(remoteFilePath, localFilePath, name string) {
defer wg.Done()
defer func() { <-sem }()
if ctx.Err() != nil {
return
}
c := pool.acquire()
defer pool.release(c)
if progressFunc != nil {
progressFunc(name, 0, 0)
}
// DownloadFile's progress callback has a different
// signature than our directory-level progressFunc, so
// we wrap it here to keep the existing per-file
// progress behaviour.
var fileProgress func(current, total int64)
if progressFunc != nil {
fileProgress = func(current, total int64) {
progressFunc(name, current, total)
}
}
if err := c.DownloadFile(ctx, remoteFilePath, localFilePath, 0, fileProgress); err != nil {
setErr(fmt.Errorf("download %s: %w", remoteFilePath, err))
}
}(remoteFilePath, localFilePath, name)
case <-ctx.Done():
return
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More