From 53db81e1f7be69b7c68c9164005288e1c1654b0b Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 26 Jun 2026 21:21:58 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20initial=20goget=20release=20=E2=80=94?= =?UTF-8?q?=20modern=20IPv6-first=20download=20utility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .editorconfig | 43 + .forgejo/workflows/release.yml | 172 ++ .forgejo/workflows/test.yml | 79 + .gitignore | 35 + BACKLOG.md | 187 ++ CHANGELOG.md | 177 ++ CONTRIBUTING.md | 304 +++ LICENSE | 21 + README.md | 300 +++ assets/goget-icon.svg | 43 + assets/goget-logo.svg | 55 + cmd/goget/app.go | 660 ++++++ cmd/goget/commands.go | 1022 +++++++++ cmd/goget/completion.go | 185 ++ cmd/goget/flags.go | 316 +++ cmd/goget/main.go | 356 +++ cmd/goget/main_test.go | 381 ++++ cmd/goget/man_page.go | 217 ++ cmd/goget/schedule.go | 145 ++ cmd/goget/setup.go | 262 +++ cmd/goget/verify.go | 200 ++ docs/api.md | 454 ++++ docs/architecture.md | 222 ++ docs/authentication.md | 251 +++ docs/cli-reference.md | 225 ++ docs/configuration.md | 122 ++ docs/download-pipeline.md | 312 +++ docs/migration.md | 173 ++ docs/protocols.md | 243 +++ docs/recursive-mirror.md | 256 +++ docs/security.md | 258 +++ go.mod | 14 + go.sum | 12 + internal/archive/archive_test.go | 1156 ++++++++++ internal/archive/auto.go | 43 + internal/archive/fuzz_test.go | 80 + internal/archive/interface.go | 84 + internal/archive/registry.go | 87 + internal/archive/tar.go | 134 ++ internal/archive/tarbz2.go | 25 + internal/archive/targz.go | 42 + internal/archive/zip.go | 141 ++ internal/auth/auth_test.go | 1588 ++++++++++++++ internal/auth/basic.go | 76 + internal/auth/digest.go | 206 ++ internal/auth/netrc.go | 103 + internal/auth/oauth.go | 293 +++ internal/cli/cli_test.go | 1154 ++++++++++ internal/cli/flags.go | 101 + internal/cli/help.go | 85 + internal/cli/help.txt | 161 ++ internal/cli/parser.go | 80 + internal/cli/progress.go | 357 +++ internal/compression/bzip2.go | 35 + internal/compression/compression_test.go | 1081 ++++++++++ internal/compression/flate.go | 45 + internal/compression/gzip.go | 30 + internal/compression/interface.go | 150 ++ internal/compression/lzw.go | 52 + internal/compression/zlib.go | 30 + internal/config/config.go | 356 +++ internal/config/config_test.go | 338 +++ internal/cookie/jar.go | 201 ++ internal/cookie/jar_test.go | 874 ++++++++ internal/core/config.go | 47 + internal/core/context.go | 65 + internal/core/core_test.go | 1076 ++++++++++ internal/core/errors.go | 135 ++ internal/core/types.go | 174 ++ internal/core/version.go | 10 + internal/crypto/certs.go | 91 + internal/crypto/checksum.go | 150 ++ internal/crypto/crypto_test.go | 190 ++ internal/crypto/interface.go | 107 + internal/crypto/pgp.go | 500 +++++ internal/format/format.go | 44 + internal/format/format_test.go | 58 + internal/hsts/fuzz_test.go | 52 + internal/hsts/hsts.go | 226 ++ internal/hsts/hsts_test.go | 376 ++++ internal/linkrewrite/rewriter.go | 143 ++ internal/linkrewrite/rewriter_test.go | 448 ++++ internal/log/logger.go | 229 ++ internal/metalink/downloader.go | 703 ++++++ internal/metalink/downloader_test.go | 135 ++ internal/metalink/fuzz_test.go | 28 + internal/metalink/metalink.go | 431 ++++ internal/metalink/metalink_test.go | 330 +++ internal/mirror/mirror.go | 739 +++++++ internal/mirror/mirror_test.go | 166 ++ internal/output/atomic.go | 29 + internal/output/output_test.go | 1500 +++++++++++++ internal/output/resume.go | 154 ++ internal/output/writer.go | 303 +++ internal/output/writer_test.go | 232 ++ internal/protocol/dataurl/dataurl.go | 105 + internal/protocol/dataurl/dataurl_test.go | 93 + internal/protocol/file/file.go | 211 ++ internal/protocol/file/file_test.go | 129 ++ internal/protocol/ftp/client.go | 817 +++++++ internal/protocol/ftp/ftp_test.go | 382 ++++ internal/protocol/ftp/protocol.go | 220 ++ internal/protocol/ftp/save_resume_test.go | 62 + internal/protocol/gemini/gemini.go | 203 ++ internal/protocol/gemini/gemini_test.go | 134 ++ internal/protocol/gopher/gopher.go | 384 ++++ internal/protocol/gopher/gopher_test.go | 110 + internal/protocol/http/client.go | 1137 ++++++++++ internal/protocol/http/config_types.go | 107 + internal/protocol/http/handler.go | 179 ++ internal/protocol/http/http_test.go | 1054 +++++++++ internal/protocol/http/integration_test.go | 559 +++++ internal/protocol/http/parallel_download.go | 323 +++ internal/protocol/http/upload.go | 379 ++++ internal/protocol/interface.go | 151 ++ internal/protocol/protocol_test.go | 1390 ++++++++++++ internal/protocol/register/register.go | 47 + internal/protocol/registry.go | 132 ++ internal/protocol/sftp/hostkey.go | 33 + internal/protocol/sftp/hostkey_test.go | 25 + internal/protocol/sftp/save_resume_test.go | 54 + internal/protocol/sftp/sftp.go | 1198 +++++++++++ internal/protocol/sftp/sftp_test.go | 623 ++++++ internal/protocol/webdav/protocol.go | 1320 ++++++++++++ internal/protocol/webdav/save_resume_test.go | 61 + internal/protocol/webdav/webdav_test.go | 1550 ++++++++++++++ internal/queue/queue.go | 470 ++++ internal/queue/queue_test.go | 359 ++++ internal/recursive/crawler.go | 474 ++++ internal/recursive/crawler_test.go | 157 ++ internal/recursive/filter.go | 211 ++ internal/recursive/parser.go | 180 ++ internal/recursive/recursive_test.go | 1319 ++++++++++++ internal/transport/bind_linux.go | 24 + internal/transport/bind_other.go | 15 + internal/transport/dialer.go | 285 +++ internal/transport/dialer_test.go | 60 + internal/transport/proxy.go | 405 ++++ internal/transport/retry.go | 153 ++ internal/transport/tls.go | 146 ++ internal/transport/token_bucket.go | 83 + internal/transport/transport_test.go | 2026 ++++++++++++++++++ internal/warc/writer.go | 50 + justfile | 24 + pkg/api/api.go | 341 +++ pkg/api/api_test.go | 197 ++ pkg/api/register.go | 19 + 147 files changed, 45931 insertions(+) create mode 100644 .editorconfig create mode 100644 .forgejo/workflows/release.yml create mode 100644 .forgejo/workflows/test.yml create mode 100644 .gitignore create mode 100644 BACKLOG.md create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 assets/goget-icon.svg create mode 100644 assets/goget-logo.svg create mode 100644 cmd/goget/app.go create mode 100644 cmd/goget/commands.go create mode 100644 cmd/goget/completion.go create mode 100644 cmd/goget/flags.go create mode 100644 cmd/goget/main.go create mode 100644 cmd/goget/main_test.go create mode 100644 cmd/goget/man_page.go create mode 100644 cmd/goget/schedule.go create mode 100644 cmd/goget/setup.go create mode 100644 cmd/goget/verify.go create mode 100644 docs/api.md create mode 100644 docs/architecture.md create mode 100644 docs/authentication.md create mode 100644 docs/cli-reference.md create mode 100644 docs/configuration.md create mode 100644 docs/download-pipeline.md create mode 100644 docs/migration.md create mode 100644 docs/protocols.md create mode 100644 docs/recursive-mirror.md create mode 100644 docs/security.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/archive/archive_test.go create mode 100644 internal/archive/auto.go create mode 100644 internal/archive/fuzz_test.go create mode 100644 internal/archive/interface.go create mode 100644 internal/archive/registry.go create mode 100644 internal/archive/tar.go create mode 100644 internal/archive/tarbz2.go create mode 100644 internal/archive/targz.go create mode 100644 internal/archive/zip.go create mode 100644 internal/auth/auth_test.go create mode 100644 internal/auth/basic.go create mode 100644 internal/auth/digest.go create mode 100644 internal/auth/netrc.go create mode 100644 internal/auth/oauth.go create mode 100644 internal/cli/cli_test.go create mode 100644 internal/cli/flags.go create mode 100644 internal/cli/help.go create mode 100644 internal/cli/help.txt create mode 100644 internal/cli/parser.go create mode 100644 internal/cli/progress.go create mode 100644 internal/compression/bzip2.go create mode 100644 internal/compression/compression_test.go create mode 100644 internal/compression/flate.go create mode 100644 internal/compression/gzip.go create mode 100644 internal/compression/interface.go create mode 100644 internal/compression/lzw.go create mode 100644 internal/compression/zlib.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/cookie/jar.go create mode 100644 internal/cookie/jar_test.go create mode 100644 internal/core/config.go create mode 100644 internal/core/context.go create mode 100644 internal/core/core_test.go create mode 100644 internal/core/errors.go create mode 100644 internal/core/types.go create mode 100644 internal/core/version.go create mode 100644 internal/crypto/certs.go create mode 100644 internal/crypto/checksum.go create mode 100644 internal/crypto/crypto_test.go create mode 100644 internal/crypto/interface.go create mode 100644 internal/crypto/pgp.go create mode 100644 internal/format/format.go create mode 100644 internal/format/format_test.go create mode 100644 internal/hsts/fuzz_test.go create mode 100644 internal/hsts/hsts.go create mode 100644 internal/hsts/hsts_test.go create mode 100644 internal/linkrewrite/rewriter.go create mode 100644 internal/linkrewrite/rewriter_test.go create mode 100644 internal/log/logger.go create mode 100644 internal/metalink/downloader.go create mode 100644 internal/metalink/downloader_test.go create mode 100644 internal/metalink/fuzz_test.go create mode 100644 internal/metalink/metalink.go create mode 100644 internal/metalink/metalink_test.go create mode 100644 internal/mirror/mirror.go create mode 100644 internal/mirror/mirror_test.go create mode 100644 internal/output/atomic.go create mode 100644 internal/output/output_test.go create mode 100644 internal/output/resume.go create mode 100644 internal/output/writer.go create mode 100644 internal/output/writer_test.go create mode 100644 internal/protocol/dataurl/dataurl.go create mode 100644 internal/protocol/dataurl/dataurl_test.go create mode 100644 internal/protocol/file/file.go create mode 100644 internal/protocol/file/file_test.go create mode 100644 internal/protocol/ftp/client.go create mode 100644 internal/protocol/ftp/ftp_test.go create mode 100644 internal/protocol/ftp/protocol.go create mode 100644 internal/protocol/ftp/save_resume_test.go create mode 100644 internal/protocol/gemini/gemini.go create mode 100644 internal/protocol/gemini/gemini_test.go create mode 100644 internal/protocol/gopher/gopher.go create mode 100644 internal/protocol/gopher/gopher_test.go create mode 100644 internal/protocol/http/client.go create mode 100644 internal/protocol/http/config_types.go create mode 100644 internal/protocol/http/handler.go create mode 100644 internal/protocol/http/http_test.go create mode 100644 internal/protocol/http/integration_test.go create mode 100644 internal/protocol/http/parallel_download.go create mode 100644 internal/protocol/http/upload.go create mode 100644 internal/protocol/interface.go create mode 100644 internal/protocol/protocol_test.go create mode 100644 internal/protocol/register/register.go create mode 100644 internal/protocol/registry.go create mode 100644 internal/protocol/sftp/hostkey.go create mode 100644 internal/protocol/sftp/hostkey_test.go create mode 100644 internal/protocol/sftp/save_resume_test.go create mode 100644 internal/protocol/sftp/sftp.go create mode 100644 internal/protocol/sftp/sftp_test.go create mode 100644 internal/protocol/webdav/protocol.go create mode 100644 internal/protocol/webdav/save_resume_test.go create mode 100644 internal/protocol/webdav/webdav_test.go create mode 100644 internal/queue/queue.go create mode 100644 internal/queue/queue_test.go create mode 100644 internal/recursive/crawler.go create mode 100644 internal/recursive/crawler_test.go create mode 100644 internal/recursive/filter.go create mode 100644 internal/recursive/parser.go create mode 100644 internal/recursive/recursive_test.go create mode 100644 internal/transport/bind_linux.go create mode 100644 internal/transport/bind_other.go create mode 100644 internal/transport/dialer.go create mode 100644 internal/transport/dialer_test.go create mode 100644 internal/transport/proxy.go create mode 100644 internal/transport/retry.go create mode 100644 internal/transport/tls.go create mode 100644 internal/transport/token_bucket.go create mode 100644 internal/transport/transport_test.go create mode 100644 internal/warc/writer.go create mode 100644 justfile create mode 100644 pkg/api/api.go create mode 100644 pkg/api/api_test.go create mode 100644 pkg/api/register.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8310d60 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,43 @@ +# EditorConfig: https://editorconfig.org + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.{rb,erb,gemspec}] +indent_style = space +indent_size = 2 + +[{Gemfile,Rakefile}] +indent_style = space +indent_size = 2 + +[*.{js,html,css,vue}] +indent_style = space +indent_size = 2 + +[*.{yml,yaml,json,toml}] +indent_style = space +indent_size = 2 + +[*.{rs,cj}] +indent_style = space +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 4 + +[justfile] +indent_style = tab + +[*.md] +indent_style = space +indent_size = 2 +trim_trailing_whitespace = false diff --git a/.forgejo/workflows/release.yml b/.forgejo/workflows/release.yml new file mode 100644 index 0000000..046ff2f --- /dev/null +++ b/.forgejo/workflows/release.yml @@ -0,0 +1,172 @@ +name: Release + +on: + push: + tags: ["v*"] + +jobs: + build: + runs-on: codeberg-small + strategy: + fail-fast: false + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + - goos: linux + goarch: riscv64 + - goos: linux + goarch: loong64 + - goos: freebsd + goarch: amd64 + - goos: freebsd + goarch: arm64 + - goos: freebsd + goarch: riscv64 + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Download dependencies + run: go mod download + + - name: Build + id: build + env: + REF: ${{ forgejo.ref }} + run: | + set -euo pipefail + + VERSION="${REF##*/}" + + if [[ ! "$VERSION" =~ ^v[0-9]+(\.[0-9]+){0,2}([-+].*)?$ ]]; then + echo "ERROR: expected a semver tag like v1.2.3, got: '$VERSION' (ref: '$REF')" + exit 1 + fi + + VERSION_NO_V="${VERSION#v}" + echo "version_no_v=${VERSION_NO_V}" >> "$FORGEJO_OUTPUT" + + LDFLAGS="-s -w -X codeberg.org/petrbalvin/goget/internal/core.Version=${VERSION}" + mkdir -p bin + GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} \ + go build -ldflags "$LDFLAGS" \ + -o "bin/goget-${VERSION_NO_V}-${{ matrix.goos }}-${{ matrix.goarch }}" \ + ./cmd/goget + + - name: Upload artifact + uses: https://data.forgejo.org/actions/upload-artifact@v3 + with: + name: goget-${{ matrix.goos }}-${{ matrix.goarch }} + path: bin/goget-${{ steps.build.outputs.version_no_v }}-${{ matrix.goos }}-${{ matrix.goarch }} + if-no-files-found: error + + release: + runs-on: codeberg-tiny + needs: build + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - name: Download all artifacts + uses: https://data.forgejo.org/actions/download-artifact@v3 + with: + path: dist + + - name: Extract CHANGELOG section + env: + VERSION: ${{ forgejo.ref_name }} + run: | + set -euo pipefail + VERSION_NO_V="${VERSION#v}" + echo "Looking for CHANGELOG section: ${VERSION_NO_V}" + + sed -n "/^## \[${VERSION_NO_V}\] /,/^## \[/p" CHANGELOG.md \ + | sed '$d' \ + | tail -n +2 \ + > release-body.md + + if [ ! -s release-body.md ]; then + echo "ERROR: no CHANGELOG section found for ${VERSION_NO_V}" + echo "Expected a heading like: ## [${VERSION_NO_V}] — YYYY-MM-DD" + exit 1 + fi + + echo "Release body preview:" + head -n 20 release-body.md + + - name: Create release + env: + FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + run: | + set -euo pipefail + # JSON-escape the release body with sed: backslashes first, then + # quotes, tabs, and carriage returns, and finally fold newlines into + # \n. Wrap the result in quotes to form a JSON string. + BODY=$(sed -e 's/\\/\\\\/g' \ + -e 's/"/\\"/g' \ + -e 's/\t/\\t/g' \ + -e 's/\r//g' \ + release-body.md \ + | sed ':a;N;$!ba;s/\n/\\n/g') + BODY="\"${BODY}\"" + + response=$(curl -sS -w '\n%{http_code}' \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST \ + "${FORGEJO_SERVER_URL}/api/v1/repos/${FORGEJO_REPOSITORY}/releases" \ + -d "{ + \"tag_name\": \"${FORGEJO_REF_NAME}\", + \"name\": \"${FORGEJO_REF_NAME}\", + \"body\": ${BODY}, + \"draft\": false, + \"prerelease\": false + }") + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | sed '$d') + + echo "HTTP ${http_code}" + + if [ "$http_code" != "201" ]; then + echo "Failed to create release: ${body}" + exit 1 + fi + + RELEASE_ID=$(echo "$body" | grep -oE '"id"[[:space:]]*:[[:space:]]*[0-9]+' | head -1 | grep -oE '[0-9]+') + echo "Created release ID=${RELEASE_ID}" + printf '%s' "${RELEASE_ID}" > release-id.txt + + - name: Upload assets + env: + FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + run: | + set -euo pipefail + RELEASE_ID=$(cat release-id.txt) + echo "Release ID: ${RELEASE_ID}" + + for binary in dist/goget-*/goget-*; do + [ -f "$binary" ] || continue + fname=$(basename "$binary") + echo "Uploading ${fname}..." + http_code=$(curl -sS -o /dev/null -w '%{http_code}' \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/octet-stream" \ + -X POST \ + --data-binary "@${binary}" \ + "${FORGEJO_SERVER_URL}/api/v1/repos/${FORGEJO_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${fname}") + echo " HTTP ${http_code}" + if [ "$http_code" != "201" ]; then + echo "Failed to upload ${fname}" + exit 1 + fi + done + + echo + echo "Release ${FORGEJO_REF_NAME} is live with the following assets:" + ls -lh dist/goget-*/ diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml new file mode 100644 index 0000000..61b2687 --- /dev/null +++ b/.forgejo/workflows/test.yml @@ -0,0 +1,79 @@ +name: Test + +on: + push: + branches: [development] + pull_request: + branches: [development] + +jobs: + vet: + runs-on: codeberg-small + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Download dependencies + run: go mod download + + - name: gofmt + # `gofmt -l` prints the names of unformatted files. An empty + # output is the only acceptable result. + run: gofmt -l cmd internal pkg + + - name: go vet + run: go vet ./... + + test: + runs-on: codeberg-small + needs: vet + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Download dependencies + run: go mod download + + - name: go test -race + run: go test -race -count=1 ./... + + - name: Coverage gate + run: | + go test -coverprofile=coverage.out ./... + coverage=$(go tool cover -func=coverage.out | awk '/^total:/ { gsub("%",""); print $3 }') + echo "Total coverage: ${coverage}%" + awk -v c="$coverage" 'BEGIN { exit !(c+0 < 40) }' \ + && { echo "Coverage ${coverage}% is below 40% threshold"; exit 1; } \ + || echo "Coverage ${coverage}% OK (threshold: 40%)" + + fuzz: + runs-on: codeberg-small + needs: vet + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Download dependencies + run: go mod download + + - name: go test -fuzz + run: | + for pkg in $(go list ./...); do + fuzz_funcs=$(go test -list='Fuzz.*' "$pkg" 2>/dev/null | grep '^Fuzz' || true) + if [ -n "$fuzz_funcs" ]; then + echo "==> Fuzzing $pkg" + for fn in $fuzz_funcs; do + echo " $fn …" + go test -parallel=2 -timeout=5m -fuzz="^${fn}$" -fuzztime=8s "$pkg" + done + fi + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4136138 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# goget .gitignore — build artifacts, IDE files, OS files, and logs + +# Build output +bin/ + +# Test artifacts +*.test +*.out +coverage.html + +# Go workspace (go.work and go.work.sum) +go.work +go.work.sum + +# Vendored dependencies (we use Go modules) +vendor/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Environment +.env +.env.local +.env.*.local + +# Logs +*.log diff --git a/BACKLOG.md b/BACKLOG.md new file mode 100644 index 0000000..0a8426f --- /dev/null +++ b/BACKLOG.md @@ -0,0 +1,187 @@ +# Backlog — goget + +This file tracks planned features and improvements across releases. +Items are grouped by target version and priority. Completed items +move to `CHANGELOG.md` when the version ships. + +--- + +## v1.1.0 — Protocol completeness + +Finish the protocol layer so every supported protocol handles +recursive, parallel, resume, and shutdown correctly. + +### 🔴 Supreme Priority + +- [ ] **Graceful shutdown for all protocols** — persist `.goget.meta` on + SIGINT for FTP single-file and SFTP parallel recursive paths. +- [ ] **WebDAV connection pooling** — shared `http.Transport` across WebDAV + requests instead of a fresh transport per request (currently defeats HTTP + keep-alive, TLS session resumption, and HTTP/2 multiplexing). + +### 🟡 High Priority + +- [ ] **WebDAV recursive parallel downloads** — extend `--recursive-parallel` + to WebDAV using the shared transport. +- [ ] **SFTP recursive parallel downloads** — parallel SFTP recursive with + session pooling. +- [ ] **FTP recursive parallel** — connection pool for independent control + connections in FTP recursive mode. +- [ ] **`--dry-run` for WebDAV recursive** — list matched entries without + downloading. + +### 🟢 Medium Priority + +- [ ] **`--adjust-extension` for non-HTML protocols** — apply `.html` + extension to text/html files from WebDAV, Gopher. +- [ ] **`--backup-converted` for WebDAV mirrors** — keep `.orig` backups + before link conversion. +- [ ] **Parallel recursive across all protocols** — unified worker pool + independent of protocol handler. + +--- + +## v1.2.0 — curl/wget parity & UX + +Close the gap with curl and wget, and improve scripting/automation UX. + +### 🟡 High Priority + +- [ ] **`--create-dirs`** — auto-create parent directories for `--output` + paths (curl `--create-dirs` / wget `--directory-prefix` equivalent). +- [ ] **Short flags** — `-O`, `-o`, `-s`, `-v` aliases for curl/wget + muscle memory. `--output-document` alias for `--output`. +- [ ] **Brotli decompression** — RFC 7932, written from scratch. + Currently only gzip/deflate/bzip2/LZW. + +### 🟢 Medium Priority + +- [ ] **`--progress=bar:force:noscroll`** — curl-compatible progress options. +- [ ] **`--metalink --follow-metalink=mem`** — keep Metalink in memory instead + of writing to disk. +- [ ] **Bandwidth throttling per-domain** — global rate limit only at present. +- [ ] **MIME type detection for local files** — determine Content-Type from + file extension when uploading. +- [ ] **`--cut-dirs` for recursive mode** — strip directory components from + output paths. +- [ ] **Notification hooks** — `--on-error` with richer environment variables + (exit code, error type, checksum). +- [ ] **Retry with backoff visualization** — show retry attempts in progress + bar (attempt 2/3, waiting 4s...). +- [ ] **Idempotent `--resume`** — auto-detect whether resume is possible + without manual flag. +- [ ] **`--show-metadata`** — display `.goget.meta` contents for debugging. + +### 🔵 Lower Priority + +- [ ] **URL glob expansion (`--glob`)** — expand `[1-3]` and `{a,b}` patterns + in URLs. + +--- + +## v1.3.0 — Security hardening + +TLS, authentication, and verification improvements. + +### 🔴 Supreme Priority + +- [ ] **PGP replacement** — `golang.org/x/crypto/openpgp` is frozen (archived + by Go team). Evaluate: inline a minimal subset, or wait for a community + `golang.org/x/` successor. + +### 🟡 High Priority + +- [ ] **`--pinned-cert` for all TLS protocols** — extend certificate pinning + to FTPS, SFTP, Gemini, and WebDAVS (currently HTTP/HTTPS only). +- [ ] **SOCKS5 proxy for non-HTTP protocols** — FTP and SFTP through SOCKS5 + proxy (currently HTTP/HTTPS only). +- [ ] **DNS-over-HTTPS (DoH)** — optional DoH resolver. Must use only + `golang.org/x/` libraries. + +### 🟢 Medium Priority + +- [ ] **Structured error output with `--json`** — machine-readable error + objects with type, message, URL, and hint fields. + +--- + +## v2.0.0 — New protocols + +Cloud providers, additional network protocols, and long-lived services. + +### 🔵 Lower Priority + +- [ ] **S3** — generic S3-compatible protocol with signature v4 auth + (covers AWS S3, MinIO, and any S3-compatible storage). +- [ ] **Huawei OBS** — Huawei Cloud Object Storage. +- [ ] **Alibaba OSS** — Alibaba Cloud Object Storage Service. +- [ ] **Tencent COS** — Tencent Cloud Object Storage. +- [ ] **Rsync** — incremental file transfer. Written from scratch + (rolling checksum, delta transfer, rsync wire protocol). +- [ ] **TFTP** — trivial file transfer, useful for embedded devices and PXE. +- [ ] **NFS** — NFS v3/v4 client. Written from scratch + (ONC RPC/XDR, portmapper, mount protocol, NFS protocol). +- [ ] **goget-as-a-service** — daemon mode with REST API for remote download + management. + +--- + +## v2.1.0 — Extensibility & UI + +### 🔵 Lower Priority + +- [ ] **Plugin system** — protocol handlers as external plugins. +- [ ] **TUI (Terminal User Interface)** — interactive download manager with + live queue, progress, and log views. Keyboard navigation, color-coded + status, inline log viewer. +- [ ] **Web GUI** — browser-based download manager (Vue 3 frontend, REST API, + live progress via SSE). +- [ ] **Download speed test / benchmark mode** — measure maximum throughput + against a target server without saving the file. + +--- + +## 💡 Backlog (unscheduled) + +Items that need more design, depend on external factors, or are +lower priority than the versioned roadmap. + +### Performance + +- [ ] **Memory-mapped I/O for large files** — `mmap` for checksum + verification of multi-gigabyte downloads. +- [ ] **Zero-copy for `file://` and `data:` protocols** — `sendfile(2)` / + `splice(2)` where available on Linux. +- [ ] **HTTP/3 (QUIC)** — when Go stdlib gains QUIC support. + +### Advanced UX + +- [ ] **Scheduling — pause/resume queued downloads** — ability to pause the + queue, reorder items, and resume later. + +### Code Quality & Testing + +- [ ] **Integration test matrix** — CI against real FTP, SFTP, and WebDAV + servers. +- [ ] **Coverage threshold increase** — raise from 40% to 60%. +- [ ] **Benchmark suite** — `go test -bench` in CI. +- [ ] **Fuzzing corpus in CI** — store and replay past crashes automatically. +- [ ] **`internal/log` migration** — migrate remaining ~140 `fmt.Fprintf` and + `cli.Print*` callsites to `log.Infof`. +- [ ] **`unsafe` audit** — verify no `unsafe` usage outside build-tag-gated + platform-specific code. + +### Project Infrastructure + +- [ ] **RPM spec file** — Fedora / RHEL / openEuler packaging. +- [ ] **FreeBSD port** — official FreeBSD ports tree entry. +- [ ] **systemd unit** — for daemon / scheduled mode. + +### Documentation + +- [ ] **Man page packaging** — already generated by `--generate-man-page`, + needs installation via packaging. +- [ ] **Architecture decision records** — document why specific choices were + made (no short flags, IPv6-first, long flags only, no global state). +- [ ] **Protocol-specific guides** — one doc per protocol with examples. +- [ ] **Troubleshooting guide** — common errors and solutions. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2fec20b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,177 @@ +# Changelog + +All notable changes to **goget** are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [development] + +## [1.0.0] — 2026-06-26 + +First public release: a modern IPv6-first command-line download utility built on +the Go standard library. goget provides parallel chunked downloads, resume +support, checksum verification, recursive website mirroring, and support for 9 +network protocols — all in a single static binary with zero runtime dependencies. + +### Added + +**Core engine** + +- Unified download and upload architecture via `core.Protocol` interface. +- Protocol registry with scheme normalization (https→http, ftps→ftp). +- Structured error types (`NETWORK`, `PROTOCOL`, `FILE`, `TIMEOUT`, `AUTH`, + `CHECKSUM`, `UNSUPPORTED`, `CANCELLED`). +- Request context with protocol chain tracking, attempt counting, and + cancellation. +- Smart timeout calculation based on file size, minimum speed, and safety factor. +- IPv6-first transport dialer with automatic IPv4 fallback. + +**Protocols** + +- **HTTP/HTTPS** — HTTP/1.1 and HTTP/2, TLS 1.2/1.3, content negotiation, + auto-decompression. +- **FTP/FTPS** — active and passive modes, explicit TLS (AUTH TLS), resume via + REST command. +- **SFTP** — custom SFTP v3 implementation over SSH, `known_hosts` verification, + resume, private key and password authentication. +- **WebDAV** — file download, recursive directory download, upload via HTTP PUT, + MKCOL for directory creation. +- **file://** — local file copy with recursive directory support. +- **data:** — RFC 2397 data URIs (base64 and plain text). +- **Gopher** — RFC 1436 with item type parsing. +- **Gemini** — TLS 1.2+ connections, redirect following, status code handling + (1x–6x). + +**Network & transport** + +- Concurrent chunked downloads for files over 100 MB with `Range` requests. +- Per-chunk resume metadata for interrupted parallel downloads. +- HTTP CONNECT and SOCKS5 (`socks5://`, `socks5h://`) proxy support with + username/password authentication. +- Custom DNS servers via `--dns-servers`. +- Network interface binding via `SO_BINDTODEVICE` (Linux). +- SSRF protection — blocks connections to private/internal IPs by default, + override with `--allow-private`. +- IPv4-mapped IPv6 normalization to prevent SSRF bypass. +- Token bucket rate limiting. +- Exponential backoff retry logic with configurable max retries. +- `--retry-all-errors` to retry on HTTP 4xx/5xx responses. +- `--pinned-cert` SHA-256 certificate pinning. + +**Authentication** + +- HTTP Basic authentication (RFC 7617). +- HTTP Digest authentication (RFC 7616). +- OAuth 2.0 — client credentials, authorization code, password, and refresh + token grant types with 30-second expiry grace period. +- OAuth error sanitization (RFC 6749 §5.2). +- `.netrc` auto-reading with machine and default entry resolution. +- Netscape-format cookie jar with import/export. +- Per-request cookies via `--cookie` CLI flag. +- SFTP SSH key auto-detection. +- `--password-file` for secure password loading. +- Config sanitization — strips credentials when saving to disk. +- `core.SafeURL()` — strips credentials from URLs in error messages and logs. + +**Recursive downloading & mirroring** + +- Recursive HTML link following with configurable depth limit. +- Full website mirroring with infinite depth and automatic link conversion. +- HTML parsing via `golang.org/x/net/html`. +- Link filtering by domain, accept pattern (glob and MIME), reject pattern, and + exclusion list. +- Page requisites — automatic CSS, JavaScript, and image downloading. +- `--no-parent`, `--span-hosts`, `--domains`, `--random-wait`. +- `--adjust-extension`, `--timestamping`, `--cut-dirs`. +- Link rewriting for offline viewing — absolute, protocol-relative, and + root-relative URLs. +- `--backup-converted` keeps `.orig` backups before link conversion. +- `robots.txt` compliance with `--no-robots` override. + +**Verification & security** + +- Checksum verification — SHA-256, SHA-512, SHA3-256, SHA3-512, BLAKE2b, MD5. +- SHA256SUMS file parsing (plain and binary-mode). +- PGP detached signature verification (`.asc`, `.sig`, `.gpg`). +- PGP decryption with private key and passphrase. +- HSTS cache (RFC 6797) — automatic HTTP→HTTPS upgrade. +- TLS certificate pinning, mTLS support, custom CA bundle. +- `--ssl-key-log` TLS session key logging (Wireshark-compatible). +- Atomic file writes with mode `0600` for config, resume metadata, and temp + files. + +**Download management** + +- Persistent download queue with priority (1–10) and JSON storage. +- Multi-source Metalink downloads (`.meta4`, `.metalink`) with automatic + failover. +- Batch URL loading from files. +- Cron-based scheduling (`MIN HOUR DOM MON DOW` and `YYYY-MM-DD HH:MM`). +- Post-download command hooks with environment variables. +- WARC (Web ARChive) output for archival compliance. +- Upload support — HTTP PUT and POST with multipart form data. + +**Output & progress** + +- Unicode progress bar with filled/empty blocks and real-time speed. +- Speed sparkline — mini ASCII chart of recent speed history. +- Estimated time remaining (ETA) calculation. +- Dot progress bar style, JSON progress output for scripting. +- `--write-out` metadata output, `--trace-time` HTTP timing breakdown. +- `--show-headers`, `--keep-timestamps`, `--content-disposition`. +- Color output with `always`, `never`, and `auto` modes (honors `NO_COLOR`). + +**Archive & compression** + +- Transparent decompression — gzip, deflate, zlib, bzip2, LZW. +- Archive extraction — TAR, TAR.GZ, TAR.BZ2, ZIP with auto-detection. + +**Internationalization** + +- IDN (International Domain Names) support with Punycode conversion via + `golang.org/x/net/idna`. + +**CLI** + +- Long flags only (`--url`, `--output`, `--verbose`). +- Shell completion generation — bash, zsh, and fish. +- Man page generation in troff format. +- Category-organized help text. +- `--info`, `--benchmark`, `--spider`, `--dry-run`, `--glob`. + +**Configuration** + +- TOML configuration file at `~/.config/goget/config.toml`. +- `GOGET_CONFIG` environment variable for alternative config path. +- CLI flag overrides merged on top of config and defaults. +- Config commands — init, get, set, list, unset. + +**Library API (`pkg/api/`)** + +- `Client` type with functional options (`WithTimeout`, `WithUserAgent`, + `WithVerbose`). +- `Client.Download()` and `Client.Upload()` with structured request/result types. +- `QueueClient` for programmatic queue management. +- `ParseMetalink()`, `FetchMetalink()`, `LoadConfig()`, `DefaultConfig()`, + `ParseSpeed()`, `Version()`. +- One-shot convenience functions: `Download()`, `Upload()`. + +**Build & platforms** + +- Linux — amd64, arm64, riscv64, loong64. +- FreeBSD — amd64, arm64, riscv64. +- Static binary with zero runtime dependencies (stdlib only + + `golang.org/x/` packages). +- Race detector enabled in tests, fuzz tests for security-critical parsers. +- Coverage gate at 40%. + +**Project & tooling** + +- Documentation set: architecture, API, authentication, CLI reference, + configuration, download pipeline, protocols, recursive & mirror, security. +- `just` task set (`install`, `run`, `build`, `test`, `uninstall`). +- Forgejo Actions CI (go vet, test + race, fuzz, release). +- `CONTRIBUTING.md` with Conventional Commits and release process. + +[1.0.0]: https://codeberg.org/petrbalvin/goget/releases/tag/v1.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fefbc16 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,304 @@ +# Contributing + +Thank you for considering contributing to **goget** — a modern IPv6-first +download utility built on the Go standard library. + +## Development Setup + +### Requirements + +- Go 1.26+ +- Linux (amd64, arm64, riscv64, loong64) or FreeBSD (amd64, arm64, riscv64) + +### Quick Start + +```bash +git clone https://codeberg.org/petrbalvin/goget.git +cd goget +just install # go mod download +just test # go test -race -count=1 ./... +just build # go vet + gofmt check + build → bin/goget +``` + +### Running Tests + +```bash +just test +``` + +The test suite (`*_test.go` files alongside the code) covers: + +- Protocol handlers (HTTP, FTP, SFTP, WebDAV, file, data, Gopher, Gemini). +- Transport layer (IPv6-first dialer, proxy, retry, rate limiting). +- Checksum verification, PGP signature handling, HSTS cache. +- Recursive downloading, link rewriting, Metalink parsing. +- Configuration loading, merging, and sanitization. +- CLI flag parsing, output formatting, progress bar. + +### Linting + +```bash +just build # go vet + gofmt check (also part of CI) +``` + +CI runs both `go vet ./...` and a `gofmt -l` diff check. These are enforced — +make sure they pass before submitting a pull request. + +## Code Style + +- Follow [Effective Go](https://go.dev/doc/effective_go) conventions. +- Keep lines under 100 characters where practical. +- Use `camelCase` for variables and functions, `PascalCase` for exported + symbols. +- Prefix every file with a one-line comment describing its purpose. +- Use build tags `//go:build linux || freebsd` on all source files. +- Use long flags in CLI (`--header` not `-H`). +- Prefer the Go standard library over external dependencies. External + dependencies are only allowed from `golang.org/x/` unless justified. +- Error messages in English, lowercase, no trailing period, always with context + (`open config: permission denied`). +- `gofmt` all code before committing. + +### Testing + +- Write tests alongside the code in `_test.go` files. +- Use table-driven tests for multiple test cases. +- Include both happy path and error path tests. +- Name tests following Go convention: `TestFunctionName_Scenario`. +- Use `testing.F` for fuzz tests in security-critical parsing code. + +## Commit Conventions + +We follow [Conventional Commits](https://www.conventionalcommits.org/). + +| Type | Use | +|------------|----------------------------------------------| +| `feat` | New feature | +| `fix` | Bug fix | +| `refactor` | Restructuring without behaviour change | +| `docs` | Documentation | +| `test` | Adding or updating tests | +| `chore` | Maintenance, CI, tooling | +| `perf` | Performance improvement | + +``` +feat: add Gemini protocol support +fix: handle timeout during parallel chunk downloads +refactor: unify transport dialer configuration +docs: add README quick start examples +test: add edge cases for cron schedule parser +chore: update golang.org/x/crypto +``` + +- English, lowercase subject, imperative mood (`add`, not `added`). +- No trailing period, max 72 characters. +- No version numbers in commit messages — versions belong to tags. + +## Pull Request Flow + +1. Create a feature branch from `development`. +2. Make your changes with Conventional-Commits messages. +3. Record your changes under the `## [development]` section at the top of + `CHANGELOG.md`, using the Keep a Changelog categories (`Added`, `Changed`, + `Fixed`, etc.). +4. Ensure `just build` and `just test` pass. +5. Add or update tests for your change. +6. Update documentation if the public API, configuration, or behaviour changes: + - `README.md` — high-level overview. + - `docs/configuration.md` — config keys. + - `docs/api.md` — Go library API. + - `docs/cli-reference.md` — CLI flags. + - `docs/architecture.md` — structural changes. + - `docs/security.md` — anything touching auth, TLS, or verification. +7. Open a pull request against `development`. Releases are cut by merging + `development` into `main` and pushing a `vX.Y.Z` tag — see + [Cutting a release](#cutting-a-release). + +## Project Structure + +``` +goget/ +├── cmd/ +│ └── goget/ # CLI entry point +│ ├── main.go # Core run loop, flag setup, orchestrator +│ ├── commands.go # Command handlers (download, upload, queue, config, etc.) +│ ├── flags.go # CLI flag definitions (struct + defineFlags) +│ ├── completion.go # Shell completion generators (bash, zsh, fish) +│ ├── man_page.go # Man page generation +│ ├── schedule.go # Cron and datetime scheduling +│ └── verify.go # Checksum and PGP verification +├── internal/ +│ ├── archive/ # Archive extraction (tar, targz, tarbz2, zip) +│ ├── auth/ # Authentication (basic, digest, OAuth 2.0, netrc) +│ ├── cli/ # CLI utilities (help text, progress bar, color output) +│ ├── compression/ # Decompression (gzip, bzip2, zlib, flate, lzw) +│ ├── config/ # TOML config loading, saving, and merging +│ ├── cookie/ # Netscape-format cookie jar +│ ├── core/ # Core types (DownloadRequest, Result, Protocol interface, errors) +│ ├── crypto/ # Checksum verification, PGP, certificate handling +│ ├── format/ # Human-readable formatting (bytes, speed) +│ ├── hsts/ # HSTS cache (RFC 6797) +│ ├── log/ # Structured logger (text/JSON output, level filtering) +│ ├── linkrewrite/ # HTML link rewriting for offline mirroring +│ ├── metalink/ # Metalink (multi-source downloads) +│ ├── mirror/ # Website mirroring logic +│ ├── output/ # Output writing (atomic writes, resume metadata) +│ ├── protocol/ # Protocol abstraction layer +│ │ ├── http/ # HTTP/HTTPS client, parallel download, upload +│ │ ├── ftp/ # FTP/FTPS client +│ │ ├── sftp/ # SFTP client +│ │ ├── file/ # Local file:// protocol +│ │ ├── dataurl/ # data: URI protocol +│ │ ├── gopher/ # Gopher protocol +│ │ ├── gemini/ # Gemini protocol +│ │ └── webdav/ # WebDAV protocol +│ ├── queue/ # Download queue management +│ ├── recursive/ # Recursive download (HTML parser, URL filter, crawler) +│ ├── warc/ # WARC archiving writer +│ └── transport/ # HTTP transport (dialer, proxy, TLS, retry, rate limiting) +├── pkg/ +│ └── api/ # Public library API (Client, Download, Upload, Queue) +└── docs/ # Architecture and design documentation +``` + +## Architecture Overview + +```mermaid +graph TD + CLI[cmd/goget/ - CLI Entry] + Flags[Flags parsing] + Config[internal/config/] + Core[internal/core/ - Types & Protocol Interface] + Registry[internal/protocol/ - Registry] + HTTP[internal/protocol/http/] + FTP[internal/protocol/ftp/] + SFTP[internal/protocol/sftp/] + FILE[internal/protocol/file/] + DataURL[internal/protocol/dataurl/] + Gopher[internal/protocol/gopher/] + Gemini[internal/protocol/gemini/] + WebDAV[internal/protocol/webdav/] + Transport[internal/transport/ - Dialer, Proxy, TLS] + Output[internal/output/ - Atomic write, Resume] + Crypto[internal/crypto/ - Checksum, PGP] + Auth[internal/auth/] + Cookie[internal/cookie/] + HSTS[internal/hsts/] + Recursive[internal/recursive/] + Mirror[internal/mirror/] + Metalink[internal/metalink/] + QueueInt[internal/queue/] + Archive[internal/archive/] + API[pkg/api/ - Public Library] + + CLI --> Flags + CLI --> Config + CLI --> Core + CLI --> Auth + CLI --> Crypto + CLI --> Recursive + CLI --> Mirror + CLI --> Metalink + CLI --> QueueInt + CLI --> Archive + Core --> Registry + Registry --> HTTP + Registry --> FTP + Registry --> SFTP + Registry --> FILE + Registry --> DataURL + Registry --> Gopher + Registry --> Gemini + Registry --> WebDAV + HTTP --> Transport + HTTP --> Cookie + HTTP --> HSTS + HTTP --> Output + API --> Core + API --> Registry +``` + +### Key Design Decisions + +- **Protocol abstraction** — Every network protocol implements `core.Protocol` + and registers with a global registry. The CLI and API resolve protocols by URL + scheme at runtime. +- **IPv6-first** — The transport dialer prefers IPv6 addresses and falls back to + IPv4 only when no IPv6 address is available. +- **Parallel downloads** — Files over 100 MB are automatically split into + chunks. Each chunk downloads via a separate connection with byte-range + requests. +- **Atomic output** — Downloads write to a temporary file first, then atomically + rename on completion. Resume metadata is stored alongside the partial file. +- **Config merging** — CLI flags override config file values, which override + defaults. The `config.Merge()` method applies CLI overrides after loading. +- **No global state** — The CLI is the only singleton. Internal packages are + designed for testability with injectable dependencies. + +## Testing Strategy + +| Layer | Approach | Tools | +|----------------|------------------------------------------------------------|---------------------| +| Unit tests | Table-driven tests for parsers, validators, handlers | `testing` stdlib | +| Integration | HTTP/HTTPS against local test servers, FTP against mock | `net/http/httptest` | +| Fuzz tests | Random input for checksum parsing, Metalink, HSTS cache | `testing.F` | +| Race detection | All tests run with `-race` in CI | `go test -race` | + +## CI + +goget uses **Forgejo Actions**; both workflows live under `.forgejo/workflows/` +and run on the `codeberg-small` / `codeberg-medium` runners. + +### Test — `.forgejo/workflows/test.yml` + +Triggered on push and pull requests targeting `development`. + +| Job | What it does | +|-------|-----------------------------------------------| +| `vet` | `go vet ./...` (must report zero warnings) | +| `test`| `go test -race -count=1 ./...` + coverage gate | +| `fuzz`| `go test -fuzz` on every `FuzzXxx` function | + +Run the same locally before opening a PR: + +```bash +go vet ./... +just test +``` + +### Release — `.forgejo/workflows/release.yml` + +Triggered by pushing a `v*` tag. It verifies the tag matches +`internal/core.Version`, runs the quality gate (go vet + tests), builds static +binaries for all 7 platform targets, extracts the matching `CHANGELOG.md` +section, creates a Forgejo release via the REST API, and attaches the binaries. + +Requires a `FORGEJO_TOKEN` secret with permission to create releases. + +### Cutting a release + +1. Bump the `Version` var in `internal/core/version.go`. +2. Rename `## [development]` to `## [X.Y.Z] — YYYY-MM-DD` at the top of + `CHANGELOG.md`, and add a fresh empty `## [development]` section above it. +3. Ensure CI is green on `development`. +4. Merge `development` into `main` (the only time `main` changes outside a tag). +5. Tag and push: + ```bash + git tag -a vX.Y.Z -m "goget vX.Y.Z" + git push origin main + git push origin vX.Y.Z + ``` +6. The release workflow builds the binaries and publishes the release from the + CHANGELOG section. + +## Report a Bug + +Open an issue at with: + +- Goget version (`goget --version`). +- Operating system and architecture. +- Command used and full output. +- Expected vs actual behaviour. + +For **security issues**, please email **opensource@petrbalvin.org** rather than +opening a public issue. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f83dd2a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b191c74 --- /dev/null +++ b/README.md @@ -0,0 +1,300 @@ +# Goget — Modern IPv6-First Command-Line Download Utility + +**goget** is a modern command-line download utility built on the **Go standard +library**. It provides IPv6-first connections, parallel chunked downloads, +resume support, checksum verification, recursive website mirroring, and support +for 9 network protocols — all in a single static binary with zero runtime +dependencies. + +It is designed to be: + +- **IPv6-first** with automatic IPv4 fallback — always connects via the fastest + available path. +- **Safe by default** — TLS 1.2 minimum, certificate verification, SSRF + protection, and HSTS cache (RFC 6797). +- **Self-contained** — one static binary, zero runtime dependencies beyond the + Go standard library and `golang.org/x/` packages. +- **Scriptable** — JSON progress output, `--write-out` metadata, post-download + hooks, and a public Go library API. + +--- + +## Stack + +| Concern | Choice | +|-----------------|------------------------------------------| +| Language | Go (≥ 1.26) | +| Protocols | HTTP/1.1, HTTP/2, FTP, SFTP, WebDAV, Gemini, Gopher | +| HTTP transport | Custom dialer with IPv6-first resolution | +| TLS | Go `crypto/tls` (1.2 minimum) | +| Markup | `golang.org/x/net/html` for recursive downloads | +| Crypto | stdlib + `golang.org/x/crypto` for PGP | + +--- + +## Features + +- **9 protocols** — HTTP/1.1, HTTP/2, FTP (active/passive, explicit TLS), SFTP + with `known_hosts` verification, WebDAV (`webdav://`, `webdavs://`) with + recursive directory download and upload, Gemini, Gopher, `data:`, `file://`. +- **Parallel chunked downloads** — automatically splits files over 100 MB into + concurrent chunks using byte-range requests. +- **Resume interrupted transfers** — metadata-driven resume with `.goget.meta` + tracking across HTTP, FTP, SFTP, and WebDAV. +- **Recursive downloading & mirroring** — follow links, filter by pattern or + MIME type, limit depth, stay within domain. Full site mirrors with link + conversion for offline viewing and `robots.txt` compliance. +- **Checksum verification** — SHA-256, SHA-512, SHA3-256, SHA3-512, BLAKE2b, + MD5; auto-parse SHA256SUMS files. +- **PGP support** — detached signature verification and OpenPGP decryption. +- **Metalink** — multi-source downloads with automatic failover (`.meta4`, + `.metalink`). +- **Download queue** — persistent queue with priority (1–10). +- **Scheduling** — absolute datetime or cron expressions. +- **OAuth 2.0** — client credentials, authorization code, and refresh token + flows. +- **Proxy support** — HTTP CONNECT, SOCKS5 (`socks5://`, `socks5h://`). +- **HSTS cache** — RFC 6797 compliance, automatic HTTP→HTTPS upgrade. +- **WARC output** — Web ARChive format for legal and archival compliance. +- **Library API** — `pkg/api/` package for programmatic downloads, uploads, and + queue management. +- **Archive extraction** — TAR, TAR.GZ, TAR.BZ2, ZIP with auto-detection. +- **Shell completion** — bash, zsh, and fish. +- **IDN support** — international domain names with automatic Punycode + conversion. + +--- + +## Quick Start + +```bash +git clone https://codeberg.org/petrbalvin/goget && cd goget && just install && just run --version +``` + +> `just install` downloads Go module dependencies. `just build` builds a static +> binary to `bin/goget`. + +```bash +# Download a file +goget --url https://example.com/file.zip + +# Parallel chunked download (auto-enabled for files >100 MB) +goget --url https://example.com/large.iso --parallel 4 + +# Resume interrupted download +goget --url https://example.com/file.zip --resume + +# Multiple URLs +goget --url https://a.com/1.zip --url https://b.com/2.zip + +# Batch download from file +goget --batch-file urls.txt + +# Custom output directory +goget --url https://example.com/file.zip --output ./downloads/file.zip +``` + +--- + +## Usage + +### Common Flags + +```bash +# Custom HTTP headers +goget --header "Authorization: Bearer token" --url https://api.example.com + +# Cookies +goget --cookie "session=abc123" --url https://example.com + +# POST data +goget --data "key=value" --url https://httpbin.org/post + +# Skip TLS verification (⚠ testing only) +goget --insecure --url https://self-signed.example.com + +# Client certificate (mTLS) +goget --cert client.pem --key client.key --url https://mtls.example.com + +# HTTP trace timing +goget --trace-time --url https://example.com +# → DNS: 12ms | TCP: 34ms | TLS: 56ms | TTFB: 78ms | Total: 180ms + +# Timeouts +goget --max-time 30s --url https://slow.example.com +goget --connect-timeout 5s --url https://example.com + +# Rate limiting +goget --rate-limit 1MB/s --url https://example.com/file.zip + +# Proxy +goget --proxy http://proxy.example.com:8080 --url https://example.com/file.zip +goget --proxy socks5h://127.0.0.1:1080 --url https://example.com/file.zip +``` + +### Recursive & Mirroring + +```bash +# Recursive download with depth limit +goget --url https://example.com/docs/ --recursive --max-depth 3 + +# Pattern filtering +goget --recursive --accept "*.pdf,*.html" --url https://example.com + +# Dry-run — list without downloading +goget --url https://example.com/docs/ --recursive --dry-run + +# Mirror entire website with link conversion +goget --url https://example.com --mirror --convert-links --output ./mirror + +# WARC archiving +goget --warc-file archive.warc --url https://example.com/page.html +``` + +### Verification + +```bash +# Checksum +goget --checksum abc123... --url https://example.com/file.zip +goget --checksum-file SHA256SUMS --url https://example.com/file.zip + +# PGP +goget --pgp-verify --pgp-sig file.sig --url https://example.com/file + +# Metalink multi-source +goget --metalink --url https://example.com/release.meta4 +``` + +### Advanced + +```bash +# Schedule a download +goget --schedule "2026-06-01 02:00" --url https://example.com/daily.zip +goget --schedule "0 2 * * *" --url https://example.com/daily.zip + +# Post-download hook (env: GOGET_OUTPUT, GOGET_SIZE, GOGET_URL) +goget --on-complete "process.sh" --url https://example.com/data.csv + +# JSON progress for scripting +goget --json --no-progress --url https://example.com/file.zip + +# Shell completion +goget --completion bash > /etc/bash_completion.d/goget +goget --completion zsh > /usr/share/zsh/site-functions/_goget +goget --completion fish > ~/.config/fish/completions/goget.fish + +# Configuration management +goget --init-config +goget --config-get timeout +goget --config-set timeout 5m +goget --config-list +``` + +--- + +## Configuration + +Default location: `~/.config/goget/config.toml`. See +[`docs/configuration.md`](docs/configuration.md) for the full schema. + +```toml +timeout = "30m0s" +parallel = 0 +user_agent = "Goget/1.0.0" +max_speed = 0 +auto_resume = true +auto_decompress = true +insecure_skip_verify = false +checksum_algo = "sha256" +color = "auto" +progress_style = "unicode" +max_retries = 0 +proxy = "" + +[recursive] +enabled = false +max_depth = 3 +follow_external = false +delay = "100ms" +``` + +--- + +## Library Usage + +```go +package main + +import ( + "fmt" + "log" + "time" + + "codeberg.org/petrbalvin/goget/pkg/api" +) + +func main() { + client := api.NewClient( + api.WithTimeout(5 * time.Minute), + api.WithUserAgent("MyApp/1.0"), + ) + defer client.Close() + + result, err := client.Download(&api.DownloadRequest{ + URL: "https://example.com/file.zip", + Output: "output.zip", + Resume: true, + }) + if err != nil { + log.Fatalf("download failed: %v", err) + } + fmt.Printf("Downloaded %d bytes in %v (%.1f MB/s, IPv%d)\n", + result.BytesDownloaded, result.Duration, + result.Speed/1024/1024, result.IPVersion) +} +``` + +Full API reference: [`docs/api.md`](docs/api.md). + +--- + +## Development + +### Requirements + +- Go 1.26+ +- Linux (amd64, arm64, riscv64, loong64) or FreeBSD (amd64, arm64, riscv64) + +```bash +just # list recipes +just install # go mod download +just run # run from source (development) +just build # go vet + gofmt check + build → bin/goget +just test # go test -race -count=1 ./... +just uninstall # remove build artifacts +``` + +--- + +## Documentation + +- [`docs/architecture.md`](docs/architecture.md) — components and request flow +- [`docs/configuration.md`](docs/configuration.md) — config schema (TOML) +- [`docs/api.md`](docs/api.md) — public Go library API +- [`docs/cli-reference.md`](docs/cli-reference.md) — full CLI flag reference +- [`docs/protocols.md`](docs/protocols.md) — protocol handler details +- [`docs/recursive-mirror.md`](docs/recursive-mirror.md) — recursive downloading + and mirroring +- [`docs/download-pipeline.md`](docs/download-pipeline.md) — download flow +- [`docs/authentication.md`](docs/authentication.md) — auth mechanisms +- [`docs/security.md`](docs/security.md) — threat model and controls + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, code style, commit +conventions, the pull request flow, and how releases are cut. + +## License + +MIT — see [LICENSE](LICENSE). +Copyright © 2026 [Petr Balvín](https://petrbalvin.org) diff --git a/assets/goget-icon.svg b/assets/goget-icon.svg new file mode 100644 index 0000000..bf4234f --- /dev/null +++ b/assets/goget-icon.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/goget-logo.svg b/assets/goget-logo.svg new file mode 100644 index 0000000..fd721f6 --- /dev/null +++ b/assets/goget-logo.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + goget + diff --git a/cmd/goget/app.go b/cmd/goget/app.go new file mode 100644 index 0000000..5b30a2a --- /dev/null +++ b/cmd/goget/app.go @@ -0,0 +1,660 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "context" + "flag" + "fmt" + "net/url" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "codeberg.org/petrbalvin/goget/internal/auth" + "codeberg.org/petrbalvin/goget/internal/cli" + "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/cookie" + "codeberg.org/petrbalvin/goget/internal/core" + format "codeberg.org/petrbalvin/goget/internal/format" + "codeberg.org/petrbalvin/goget/internal/hsts" + "codeberg.org/petrbalvin/goget/internal/log" + "codeberg.org/petrbalvin/goget/internal/output" + "codeberg.org/petrbalvin/goget/internal/protocol" + protocolreg "codeberg.org/petrbalvin/goget/internal/protocol/register" + "codeberg.org/petrbalvin/goget/internal/transport" +) + +// app holds the runtime state of a goget command execution. +type app struct { + cfg *config.Config + flags *Flags + fs *flag.FlagSet + configPath string + allURLs []string + sigCtx context.Context + sigCancel context.CancelFunc +} + +// run is the main entry point. It is kept small — the work is delegated to +// init, dispatch, setup, and execute. +func (a *app) run() int { + // init() assigns a.sigCancel; a no-op here makes the early + // `return` paths (e.g. "missing required flag" on a fresh + // invocation) safe to clean up. Once init() runs, the real + // cancel function replaces this no-op. + defer func() { + if a.sigCancel != nil { + a.sigCancel() + } + }() + + if code := a.init(); code != 0 { + return code + } + if code := a.dispatch(); code >= 0 { + return code + } + return exitCodeForRun(a.sigCtx, a.execute()) +} + +// exitCodeForRun maps the inner exit code to the final process exit +// code. If the signal context was cancelled (SIGINT/SIGTERM) but +// execute() returned 0 — because the download propagated the cancel +// cleanly — surface 130 so shell scripts can detect the interruption. +// On a normal completion (no signal) the inner code is returned +// unchanged; non-zero inner codes are preserved regardless of signal +// state so genuine errors are not masked. +func exitCodeForRun(sigCtx context.Context, innerCode int) int { + if innerCode != 0 { + return innerCode + } + if sigCtx != nil && sigCtx.Err() != nil { + return 130 + } + return 0 +} + +// init loads config, sets up the signal context, and parses CLI flags. +func (a *app) init() int { + a.configPath = extractConfigPath(os.Args[1:]) + + cfg, err := config.Load(a.configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + a.cfg = cfg + + sigCtx, stop := signalContext() + a.sigCtx = sigCtx + a.sigCancel = stop + + a.fs = flag.NewFlagSet("goget", flag.ExitOnError) + a.fs.Usage = func() { cli.PrintUsage(os.Stderr) } + a.flags = defineFlags(a.fs, a.cfg) + + if err := a.fs.Parse(os.Args[1:]); err != nil { + cli.PrintError(os.Stderr, err) + return 1 + } + + a.allURLs = extractURLs(os.Args[1:]) + return 0 +} + +// dispatch handles all standalone subcommands (help, version, config-*, info, +// benchmark, etc.). Returns the exit code, or -1 if no subcommand matched +// (meaning the caller should proceed to the download pipeline). +func (a *app) dispatch() int { + f := a.flags + cfg := a.cfg + + if *f.Help { + cli.PrintUsage(os.Stdout) + return 0 + } + if *f.Version { + fmt.Printf("goget version %s\n", core.Version) + return 0 + } + if *f.ConfigList { + return handleConfigList(a.configPath) + } + if *f.ConfigGet != "" { + return handleConfigGet(a.configPath, *f.ConfigGet) + } + if *f.ConfigSet != "" { + return handleConfigSet(a.configPath, *f.ConfigSet, os.Args) + } + if *f.ConfigUnset != "" { + return handleConfigUnset(a.configPath, *f.ConfigUnset) + } + if *f.Info != "" { + return handleInfo(*f.Info, cfg) + } + if *f.Benchmark != "" { + return handleBenchmark(*f.Benchmark, cfg) + } + if *f.Spider { + return handleSpider(*f.URL, cfg) + } + if *f.GenManPage { + return handleGenManPage() + } + if *f.Completion != "" { + return handleCompletion(*f.Completion) + } + if *f.InitConfig { + return handleInitConfig(a.configPath) + } + if *f.BatchFile != "" { + return handleBatchFile(*f.BatchFile, *f.Output, cfg, a.sigCtx) + } + if *f.QueueList { + return handleQueueList(*f.QueueFile) + } + if *f.QueueProcess { + return handleQueueProcess(*f.QueueFile, cfg.Verbose) + } + if *f.QueueClear { + return handleQueueClear(*f.QueueFile, cfg.Verbose) + } + if *f.MetalinkFile != "" { + return handleMetalinkFile(*f.MetalinkFile, *f.Output, cfg.Verbose, *f.Queue, *f.QueuePriority, *f.QueueFile) + } + if *f.Metalink { + return handleMetalinkURL(*f.URL, *f.Output, cfg.Verbose, *f.Queue, *f.QueuePriority, *f.QueueFile) + } + if *f.Upload { + return handleUpload(*f.URL, *f.UploadMethod, *f.UploadFile, *f.UploadFormData, *f.UploadFileField, cfg.Verbose, cfg) + } + if len(a.allURLs) > 1 { + return handleMultipleURLs(a.allURLs, *f.Output, cfg, a.sigCtx) + } + return -1 // proceed to download pipeline +} + +// execute runs the download pipeline after dispatch determined that a +// download should be performed. It handles URL validation, config merging, +// authentication, transport setup, and the actual download via the protocol +// handler. +func (a *app) execute() int { + f := a.flags + cfg := a.cfg + + if *f.URL == "" { + cli.PrintError(os.Stderr, fmt.Errorf("--url is a required parameter")) + a.fs.Usage() + return 1 + } + + parsedURL, err := url.Parse(*f.URL) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid url: %w", err)) + return 1 + } + // Convert internationalized domain name to Punycode + parsedURL = idnaURL(parsedURL) + + excludePatterns := []string{} + if *f.ExcludePattern != "" { + for _, p := range strings.Split(*f.ExcludePattern, ",") { + p = strings.TrimSpace(p) + if p != "" { + excludePatterns = append(excludePatterns, p) + } + } + } + + oauthScopes := []string{} + if *f.OAuthScopes != "" { + for _, s := range strings.Split(*f.OAuthScopes, ",") { + s = strings.TrimSpace(s) + if s != "" { + oauthScopes = append(oauthScopes, s) + } + } + } + + if cfg.IsExcluded(parsedURL.String()) { + cli.PrintError(os.Stderr, fmt.Errorf("url matches exclude pattern: %s", parsedURL.String())) + return 1 + } + + maxSpeed := config.ParseSpeed(*f.MaxSpeed) + cfg.Merge(&config.CLIFlags{ + Timeout: *f.Timeout, + Parallel: *f.Parallel, + Verbose: *f.Verbose, + Debug: *f.Debug, + MaxSpeed: maxSpeed, + Proxy: *f.Proxy, + NoIPv4: *f.NoIPv4, + NoDecompress: *f.NoDecompress, + Username: *f.Username, + Password: *f.Password, + CookieJar: *f.CookieJar, + Recursive: *f.Recursive, + MaxDepth: *f.MaxDepth, + FollowExternal: *f.FollowExternal, + ExcludePatterns: excludePatterns, + Extract: *f.Extract, + ExtractDir: *f.ExtractDir, + StripComponents: *f.StripComponents, + }) + + // Authentication settings + if *f.AuthType != "" { + cfg.Auth.AuthType = *f.AuthType + } + + // OAuth 2.0 settings + if *f.OAuthClientID != "" || *f.OAuthClientSecret != "" || *f.OAuthTokenURL != "" || + *f.OAuthAccessToken != "" || *f.OAuthRefreshToken != "" { + if *f.OAuthClientID != "" { + cfg.Auth.OAuth.ClientID = *f.OAuthClientID + } + if *f.OAuthClientSecret != "" { + cfg.Auth.OAuth.ClientSecret = *f.OAuthClientSecret + } + if *f.OAuthTokenURL != "" { + cfg.Auth.OAuth.TokenURL = *f.OAuthTokenURL + } + if *f.OAuthAuthURL != "" { + cfg.Auth.OAuth.AuthURL = *f.OAuthAuthURL + } + if *f.OAuthRedirectURI != "" { + cfg.Auth.OAuth.RedirectURI = *f.OAuthRedirectURI + } + if len(oauthScopes) > 0 { + cfg.Auth.OAuth.Scopes = oauthScopes + } + if *f.OAuthGrantType != "" { + cfg.Auth.OAuth.GrantType = *f.OAuthGrantType + } + if *f.OAuthAccessToken != "" { + cfg.Auth.OAuth.AccessToken = *f.OAuthAccessToken + } + if *f.OAuthRefreshToken != "" { + cfg.Auth.OAuth.RefreshToken = *f.OAuthRefreshToken + } + } + + // Checksum algorithm settings + if *f.ChecksumAlgo != "" { + cfg.ChecksumAlgo = *f.ChecksumAlgo + } + + // Password file handling + if *f.PasswordFile != "" { + data, err := os.ReadFile(*f.PasswordFile) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to read password file: %w", err)) + return 1 + } + cfg.Auth.Password = strings.TrimSpace(string(data)) + } else if passwordEnv := os.Getenv("GOGET_PASSWORD"); passwordEnv != "" { + cfg.Auth.Password = passwordEnv + } + + // Quiet mode: suppress verbose output + if *f.Quiet { + cfg.Verbose = false + } + + // Configure structured logger based on CLI flags. + logLevel := log.InfoLevel + if cfg.Verbose { + logLevel = log.VerboseLevel + } + if cfg.Debug { + logLevel = log.DebugLevel + } + if *f.Quiet { + logLevel = log.WarnLevel + } + log.DefaultLogger = log.New(os.Stderr, + log.WithLevel(logLevel), + log.WithColor(cfg.ShouldUseColors()), + log.WithJSON(*f.JSON), + ) + + // Restart: delete existing file before download + if *f.Restart && *f.Output != "" { + os.Remove(*f.Output) + output.CleanupResumeFiles(*f.Output) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigChan + log.Warn("Interrupted by user.") + cancel() + // Also cancel the app-level signal context so run() can + // observe the interruption and return 130. + a.sigCancel() + }() + + // DNS servers + if *f.DNSServers != "" { + dnsServers := transport.ParseDNSServers(*f.DNSServers) + if len(dnsServers) > 0 { + cfg.DNSServers = dnsServers + } + } + + if *f.Schedule != "" { + if code := waitForSchedule(*f.Schedule, cfg, sigChan); code != 0 { + a.sigCancel() + return code + } + } + + // Checksum file handling + if *f.ChecksumFile != "" && *f.Checksum == "" { + expectedChecksum, err := parseChecksumFile(*f.ChecksumFile, *f.Output) + if err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed to read checksum from file: %v", err)) + } else if expectedChecksum != "" { + *f.Checksum = expectedChecksum + if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("Using checksum from file: %s", expectedChecksum)) + } + } + } + + httpClient, err := protocolreg.RegisterAll(protocol.GlobalRegistry) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to register protocols: %w", err)) + return 1 + } + + // HSTS cache + hstsCache := hsts.NewCache() + hstsPath := *f.HSTSFile + if hstsPath == "" { + hstsPath = hsts.DefaultCachePath() + } + if hstsPath != "" { + if err := hstsCache.Load(hstsPath); err != nil && cfg.Verbose { + cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to load hsts cache: %v", err)) + } + httpClient.SetHSTSCache(hstsCache) + } + + var cookieJar *cookie.Jar + cookieJarPath := cfg.CookieJar + if cookieJarPath != "" { + cookieJar, err = cookie.NewJar() + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create cookie jar: %w", err)) + return 1 + } + if err := cookieJar.LoadFromFile(cookieJarPath); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load cookies: %w", err)) + return 1 + } + if cfg.Verbose { + cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Loaded cookies from %s", cookieJarPath)) + } + } + + if cfg.Verbose { + safeURL := core.SafeURL(parsedURL) + log.Infof("Goget %s", core.Version) + log.Infof("URL: %s", safeURL) + if cfg.Auth.Username != "" { + log.Infof("Auth: %s:%s", cfg.Auth.Username, auth.MaskPassword(cfg.Auth.Password)) + } + if cookieJar != nil { + log.Infof("Cookies: %d loaded", cookieJar.GetCookieCount(parsedURL)) + } + if *f.Mirror { + log.Info("Mirror: enabled (infinite depth, convert links)") + } + if *f.Recursive { + log.Infof("Recursive: enabled (max-depth=%d)", cfg.Recursive.MaxDepth) + } + if *f.Extract { + extractInfo := "enabled" + if cfg.Extract.OutputDir != "" { + extractInfo = fmt.Sprintf("enabled → %s", cfg.Extract.OutputDir) + } + log.Infof("Extract: %s", extractInfo) + } + if *f.Timeout > 0 { + log.Infof("Timeout: %v (explicit)", *f.Timeout) + } else { + log.Infof("Timeout: %v (auto)", cfg.Timeout) + } + if cfg.Parallel > 0 { + log.Infof("Parallel: %d connections", cfg.Parallel) + } else { + log.Info("Parallel: auto") + } + if cfg.MaxSpeed > 0 { + log.Infof("Max speed: %s/s", format.Bytes(cfg.MaxSpeed)) + } + + // DNS server information + if len(cfg.DNSServers) > 0 { + log.Infof("DNS: %v", cfg.DNSServers) + } + } + + if !*f.Recursive && *f.Resume && *f.Output != "" { + canResume, meta, err := output.CanResume(*f.Output, parsedURL.String()) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("resume check failed: %w", err)) + if !*f.Overwrite { + return 1 + } + output.CleanupResumeFiles(*f.Output) + if cfg.Verbose { + cli.PrintProgressInfo(os.Stderr, "Starting fresh download (overwrite mode)") + } + } else if canResume && meta != nil { + if cfg.Verbose { + cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Resuming download: %s already downloaded", format.Bytes(meta.Downloaded))) + } + } + } + + // --no-clobber: skip if output file exists + if !*f.Recursive && *f.NoClobber && *f.Output != "" && *f.Output != "-" { + if _, err := os.Stat(*f.Output); err == nil { + if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("File exists, skipping: %s", *f.Output)) + } + return 0 + } + } + + // Default output filename: derive from the URL path when no + // --output flag is given. Matches curl/wget behaviour and prevents + // binary data being dumped to stdout. Recursive downloads, stdout + // mode (--output -), and content-disposition mode are exempt. + if *f.Output == "" && !*f.Recursive && !*f.ContentDisp { + name := filepath.Base(parsedURL.Path) + if name == "" || name == "/" || name == "." { + name = "index.html" + } + *f.Output = name + } + + writer, progressBar := setupOutputWriter(f, cfg) + if writer == nil && !*f.Recursive { + // Error already printed by setupOutputWriter + return 1 + } + if writer != nil { + defer writer.Close() + } + + httpCfg := setupHTTPConfig(f, cfg, cookieJar, parsedURL) + + if *f.Recursive { + httpCfg.Recursive.Enabled = true + httpCfg.Recursive.MaxDepth = cfg.Recursive.MaxDepth + httpCfg.Recursive.FollowExternal = cfg.Recursive.FollowExternal + httpCfg.Recursive.ExcludePatterns = cfg.Recursive.ExcludePatterns + httpCfg.Output.OutputDir = *f.Output + if *f.Accept != "" { + httpCfg.Recursive.Accept = *f.Accept + } + if *f.NoParent { + httpCfg.Recursive.NoParent = true + } + if *f.SpanHosts { + httpCfg.Recursive.SpanHosts = true + httpCfg.Recursive.FollowExternal = true + } + if *f.PageRequisites { + httpCfg.Recursive.PageRequisites = true + } + if *f.Wait != "" { + if d, err := time.ParseDuration(*f.Wait); err == nil { + httpCfg.Recursive.Wait = d + } + } + if *f.RandomWait { + httpCfg.Recursive.RandomWait = true + } + if *f.ConvertLinks { + httpCfg.Recursive.ConvertLinks = true + } + } + if *f.DryRun { + httpCfg.Recursive.DryRun = true + } + if *f.Mirror { + httpCfg.Recursive.Mirror = true + httpCfg.Recursive.ConvertLinks = !*f.NoConvertLinks + httpCfg.Recursive.MirrorAssets = !*f.NoMirrorAssets + httpCfg.Recursive.RespectRobots = !*f.NoRobots + httpCfg.Output.OutputDir = *f.Output + httpCfg.Recursive.Enabled = false // Mirror takes precedence + + // Parse rate limit + if *f.RateLimit != "" { + rate, perr := parseRateLimit(*f.RateLimit) + if perr != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("ignoring --max-speed: %v; download will be unthrottled", perr)) + } + httpCfg.RateLimit = rate + } + } + if *f.Extract { + httpCfg.Output.AutoExtract = true + httpCfg.Output.ExtractDir = cfg.Extract.OutputDir + } + if *f.Proxy != "" { + proxyCfg, perr := transport.ParseProxyConfig(*f.Proxy) + if perr == nil { + httpCfg.Transport.Proxy = proxyCfg + } + } + + proto, err := protocol.Get(parsedURL) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("protocol error: %w", err)) + return 1 + } + + var resumeInfo *core.ResumeInfo + if !*f.Recursive && *f.Resume && *f.Output != "" { + meta, _ := output.LoadResumeMetadata(*f.Output) + if meta != nil { + resumeInfo, _ = output.GetResumeInfo(meta, *f.Output) + } + } + + req := &core.DownloadRequest{ + URL: parsedURL, + Output: *f.Output, + Writer: writer, + Resume: *f.Resume, + Timeout: cfg.GetEffectiveTimeout(-1, *f.Timeout), + Verbose: cfg.Verbose, + DebugTransport: cfg.Debug, + Checksum: *f.Checksum, + Headers: make(map[string]string), + Proxy: cfg.Proxy, + AutoDecompress: cfg.AutoDecompress, + ProgressCallback: nil, + ResumeInfo: resumeInfo, + Parallel: httpCfg.Parallel, + Recursive: *f.Recursive, + RecursiveParallel: httpCfg.Recursive.Parallel, + DryRun: httpCfg.Recursive.DryRun, + MaxDepth: cfg.Recursive.MaxDepth, + AcceptPatterns: parseStringSlice(*f.Accept), + RejectPatterns: httpCfg.Recursive.Reject, + KeepTimestamps: *f.KeepTimestamps, + Ctx: ctx, + SSHKnownHosts: *f.KnownHosts, + SSHInsecure: *f.SSHInsecure, + } + + // Configure TLS for WebDAV if supported. + if webdavProto, ok := proto.(*webdavProtocol); ok { + tlsCfg := httpCfg.Transport.TLS + insecure := httpCfg.Transport.TLS != nil && httpCfg.Transport.TLS.InsecureSkipVerify + webdavProto.ConfigureTLS(tlsCfg, insecure) + + if cfg.Proxy != "" { + webdavProto.ConfigureProxy(cfg.Proxy) + } + } + + if cfg.Verbose { + if *f.Recursive { + cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Starting recursive download: %s", core.SafeURL(parsedURL))) + } else { + cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Starting download: %s", core.SafeURL(parsedURL))) + } + } + + var result *core.DownloadResult + var downloadErr error + + // Wire the progress bar to the download via a callback so the + // user sees live speed and ETA during the transfer, not just a + // static "Starting download" line. The parallel downloader + // aggregates per-chunk progress; the sequential path ignores + // the callback when the bar was not created. + if progressBar != nil && !*f.Recursive { + req.ProgressCallback = func(current, total int64, speed float64) { + progressBar.SetTotal(total) + progressBar.Update(current) + } + } + + startTime := time.Now() + + result, downloadErr = proto.Download(ctx, req) + + if progressBar != nil { + if downloadErr != nil { + progressBar.Cancel() + } else { + progressBar.Finish() + } + } + + if downloadErr != nil { + cli.PrintError(os.Stderr, downloadErr) + return 1 + } + + return runPostDownload(f, cfg, result, cookieJar, cookieJarPath, hstsCache, writer, parsedURL, startTime) +} diff --git a/cmd/goget/commands.go b/cmd/goget/commands.go new file mode 100644 index 0000000..7fe0c2f --- /dev/null +++ b/cmd/goget/commands.go @@ -0,0 +1,1022 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + + stdhttp "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "codeberg.org/petrbalvin/goget/internal/cli" + "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/core" + format "codeberg.org/petrbalvin/goget/internal/format" + "codeberg.org/petrbalvin/goget/internal/metalink" + "codeberg.org/petrbalvin/goget/internal/protocol" + httpConfig "codeberg.org/petrbalvin/goget/internal/protocol/http" + "codeberg.org/petrbalvin/goget/internal/queue" + "codeberg.org/petrbalvin/goget/internal/transport" + "codeberg.org/petrbalvin/interpres" +) + +func extractConfigPath(args []string) string { + for i := 0; i < len(args); i++ { + arg := args[i] + if arg == "--config" && i+1 < len(args) { + return args[i+1] + } + if strings.HasPrefix(arg, "--config=") { + return strings.TrimPrefix(arg, "--config=") + } + } + return "" +} + +// parseRateLimit parses rate limit string like "100KB/s" or "1MB/s" to bytes per second. +// It returns 0 with no error for the empty string and literal "0" (both mean +// "unlimited" in the rest of the codebase). Any other unparsable input +// returns a non-nil error so the caller can warn the user instead of +// silently disabling rate limiting — a typo like "1oMB/s" (letter 'o' +// instead of digit '0') must not be mistaken for "no limit". +func parseRateLimit(s string) (int64, error) { + s = strings.TrimSpace(s) + if s == "" || s == "0" { + return 0, nil + } + + // Remove /s suffix if present + s = strings.TrimSuffix(s, "/s") + s = strings.ToUpper(s) + + // Parse number and unit + var multiplier int64 = 1 + numStr := s + + if strings.HasSuffix(s, "PB") { + multiplier = 1000 * 1000 * 1000 * 1000 * 1000 + numStr = strings.TrimSuffix(s, "PB") + } else if strings.HasSuffix(s, "TB") { + multiplier = 1000 * 1000 * 1000 * 1000 + numStr = strings.TrimSuffix(s, "TB") + } else if strings.HasSuffix(s, "GB") { + multiplier = 1000 * 1000 * 1000 + numStr = strings.TrimSuffix(s, "GB") + } else if strings.HasSuffix(s, "MB") { + multiplier = 1000 * 1000 + numStr = strings.TrimSuffix(s, "MB") + } else if strings.HasSuffix(s, "KB") { + multiplier = 1000 + numStr = strings.TrimSuffix(s, "KB") + } else if strings.HasSuffix(s, "B") { + numStr = strings.TrimSuffix(s, "B") + } + + num, err := strconv.ParseFloat(strings.TrimSpace(numStr), 64) + if err != nil { + return 0, fmt.Errorf("invalid rate limit %q: %w", s, err) + } + + return int64(num * float64(multiplier)), nil +} + +// extractURLs extracts all --url values from args +func extractURLs(args []string) []string { + var urls []string + for i := 0; i < len(args); i++ { + if args[i] == "--url" && i+1 < len(args) { + urls = append(urls, args[i+1]) + i++ + } else if strings.HasPrefix(args[i], "--url=") { + urls = append(urls, strings.TrimPrefix(args[i], "--url=")) + } + } + return urls +} + +// downloadSingleURL parses and downloads a single URL, returning the result or an error. +// It creates the HTTP client, validates the URL, handles output, and performs the download. +func downloadSingleURL(rawURL string, output string, cfg *config.Config, verbose bool, ctx context.Context) (*core.DownloadResult, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + + // Look up the protocol + proto, err := protocol.Get(parsedURL) + if err != nil { + return nil, err + } + + // Build the download request (use defaults when cfg is nil) + httpCfg := httpConfig.DefaultConfig() + timeout := 30 * time.Minute + userAgent := core.Name + "/" + core.Version + ipv4Fallback := true + autoResume := true + if cfg != nil { + timeout = cfg.Timeout + userAgent = cfg.UserAgent + ipv4Fallback = cfg.IPv4Fallback + autoResume = cfg.AutoResume + } + httpCfg.Transport.Timeout = timeout + httpCfg.UserAgent = userAgent + httpCfg.Output.Verbose = verbose + httpCfg.Transport.Dialer.IPv4Fallback = ipv4Fallback + + req := &core.DownloadRequest{ + URL: parsedURL, + Output: output, + Resume: autoResume, + Timeout: timeout, + Verbose: verbose, + Headers: map[string]string{ + "User-Agent": userAgent, + }, + Ctx: ctx, + } + + return proto.Download(ctx, req) +} + +// handleMultipleURLs processes multiple URLs sequentially +func handleMultipleURLs(urls []string, outputDir string, cfg *config.Config, ctx context.Context) int { + var successCount, failCount int + var totalBytes int64 + startTime := time.Now() + + for i, urlStr := range urls { + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "\n[%d/%d] %s\n", i+1, len(urls), core.SafeURL(mustParseURL(urlStr))) + } + + // Determine output path from URL + outputPath := outputDir + if outputPath == "" || outputPath == "." { + outputPath = urlToFilename(urlStr, i+1) + } + + result, err := downloadSingleURL(urlStr, outputPath, cfg, cfg.Verbose, ctx) + if err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed: %s - %v", + core.SafeURL(mustParseURL(urlStr)), err)) + failCount++ + continue + } + + totalBytes += result.BytesDownloaded + successCount++ + + if cfg.Verbose { + cli.PrintSuccess(os.Stderr, fmt.Sprintf("Downloaded: %s (%s)", + filepath.Base(outputPath), format.Bytes(result.BytesDownloaded))) + } + } + + duration := time.Since(startTime) + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "\nAll downloads complete: %d succeeded, %d failed, %s total in %v\n", + successCount, failCount, format.Bytes(totalBytes), duration.Round(time.Second)) + } + + if failCount > 0 { + return 1 + } + return 0 +} + +// urlToFilename extracts a safe filename from a URL string, falling back to a +// generated name. It is used for auto-generating output filenames. +func urlToFilename(rawURL string, index int) string { + parsed, err := url.Parse(rawURL) + if err != nil { + return fmt.Sprintf("download_%d", index) + } + name := filepath.Base(parsed.Path) + if name == "" || name == "/" || name == "." { + return fmt.Sprintf("download_%d", index) + } + return name +} + +// mustParseURL parses a URL and returns nil on failure – safe for use with +// core.SafeURL which handles nil gracefully. +func mustParseURL(raw string) *url.URL { + u, _ := url.Parse(raw) + return u +} + +func handleQueueList(queueFile string) int { + q, err := queue.NewQueue(&queue.QueueConfig{QueueFile: queueFile}) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load queue: %w", err)) + return 1 + } + + stats := q.GetStats() + fmt.Printf("Queue Statistics:\n") + fmt.Printf(" Total: %d | Pending: %d | Downloading: %d | Completed: %d | Failed: %d\n\n", + stats.Total, stats.Pending, stats.Downloading, stats.Completed, stats.Failed) + + items := q.GetAll() + if len(items) == 0 { + fmt.Println("Queue is empty.") + return 0 + } + + fmt.Printf("%-12s %-8s %-10s %-10s %-10s %s\n", "ID", "Priority", "Status", "Downloaded", "Size", "URL") + fmt.Println(strings.Repeat("-", 80)) + + for _, item := range items { + statusIcon := "○" + switch item.Status { + case "completed": + statusIcon = "✓" + case "failed": + statusIcon = "✗" + case "downloading": + statusIcon = "⟳" + } + fmt.Printf("%-12s %-8d %-10s %-10s %-10s %s\n", + statusIcon+" "+item.ID, + item.Priority, + item.Status, + format.Bytes(item.Downloaded), + format.Bytes(item.Size), + item.URL) + } + + return 0 +} + +// handleQueueProcess processes the download queue +func handleQueueProcess(queueFile string, verbose bool) int { + q, err := queue.NewQueue(&queue.QueueConfig{QueueFile: queueFile}) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load queue: %w", err)) + return 1 + } + + if q.Len() == 0 { + if verbose { + cli.PrintInfo(os.Stderr, "Queue is empty") + } + return 0 + } + + stats := q.GetStats() + if verbose { + fmt.Printf("Processing queue with %d items (%d pending, %d failed)\n", + stats.Total, stats.Pending, stats.Failed) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigChan + cancel() + }() + + var totalDownloaded int64 + var successCount, failCount int + + for { + item := q.GetNext() + if item == nil { + break + } + + if verbose { + fmt.Printf("\n[%d/%d] Downloading: %s\n", + successCount+failCount+1, stats.Pending, item.URL) + } + + q.UpdateStatus(item.ID, "downloading") + + result, err := downloadSingleURL(item.URL, item.Output, nil, verbose, ctx) + if err != nil { + q.UpdateStatus(item.ID, "failed") + q.UpdateProgress(item.ID, 0, 0) + cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed: %s - %v", item.URL, err)) + failCount++ + continue + } + + q.UpdateStatus(item.ID, "completed") + q.UpdateProgress(item.ID, result.BytesDownloaded, result.TotalSize) + totalDownloaded += result.BytesDownloaded + successCount++ + + if verbose { + cli.PrintSuccess(os.Stderr, fmt.Sprintf("Downloaded: %s (%s)", + item.URL, format.Bytes(result.BytesDownloaded))) + } + } + + if verbose { + fmt.Printf("\nQueue processing complete: %d succeeded, %d failed, %s total\n", + successCount, failCount, format.Bytes(totalDownloaded)) + } + + return 0 +} + +// handleQueueClear clears completed items from the queue +func handleQueueClear(queueFile string, verbose bool) int { + q, err := queue.NewQueue(&queue.QueueConfig{QueueFile: queueFile}) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load queue: %w", err)) + return 1 + } + + count := q.ClearCompleted() + if verbose { + fmt.Printf("Cleared %d completed items from queue.\n", count) + } else { + fmt.Printf("Cleared %d items.\n", count) + } + + return 0 +} + +// handleMetalinkFile processes a local metalink file +func handleMetalinkFile(metalinkPath, outputDir string, verbose, addToQueue bool, priority int, queueFile string) int { + ml, err := metalink.ParseFile(metalinkPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to parse metalink: %w", err)) + return 1 + } + + if err := ml.Validate(); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid metalink: %w", err)) + return 1 + } + + if verbose { + fmt.Printf("Metalink: %d file(s), total size: %s\n", + len(ml.Files), format.Bytes(ml.GetTotalSize())) + } + + if outputDir == "" { + outputDir = "." + } + + if addToQueue { + // Add all files to queue + q, err := queue.NewQueue(&queue.QueueConfig{QueueFile: queueFile}) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load queue: %w", err)) + return 1 + } + + var items []queue.QueueItem + for _, file := range ml.Files { + items = append(items, queue.QueueItem{ + URL: file.GetBestURL().URL, + Output: filepath.Join(outputDir, file.Name), + Priority: priority, + Size: file.Size, + }) + } + + q.AddMultiple(items) + if verbose { + fmt.Printf("Added %d items to queue.\n", len(items)) + } + } else { + // Download directly using multi-source downloader + if verbose { + fmt.Println("Downloading from metalink (multi-source)...") + } + + dl := metalink.NewMultiSourceDownloader(metalink.DefaultDownloaderConfig()) + + for _, file := range ml.Files { + outputPath := filepath.Join(outputDir, file.Name) + dlCtx, dlCancel := context.WithCancel(context.Background()) + defer dlCancel() + + dlSigChan := make(chan os.Signal, 1) + signal.Notify(dlSigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-dlSigChan + dlCancel() + }() + + result, err := dl.DownloadWithFailover(dlCtx, &file, outputPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to download %s: %w", file.Name, err)) + continue + } + if verbose { + fmt.Printf("✓ Downloaded %s: %s in %v (%s/s)\n", + file.Name, format.Bytes(result.BytesDownloaded), + result.Duration.Round(time.Second), format.Speed(result.Speed)) + } + } + } + + return 0 +} + +// handleMetalinkURL downloads from a metalink URL +func handleMetalinkURL(metalinkURL, outputDir string, verbose, addToQueue bool, priority int, queueFile string) int { + if verbose { + fmt.Printf("Fetching metalink from: %s\n", metalinkURL) + } + + // Download the raw metalink XML + resp, err := stdhttp.Get(metalinkURL) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to download metalink: %w", err)) + return 1 + } + defer resp.Body.Close() + + if resp.StatusCode != stdhttp.StatusOK { + cli.PrintError(os.Stderr, fmt.Errorf("metalink url returned status %s", resp.Status)) + return 1 + } + + // Save to temp file for processing + tempFile, err := os.CreateTemp("", "goget-metalink-*.xml") + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create temp file: %w", err)) + return 1 + } + tempPath := tempFile.Name() + + if _, err := io.Copy(tempFile, resp.Body); err != nil { + tempFile.Close() + os.Remove(tempPath) + cli.PrintError(os.Stderr, fmt.Errorf("failed to save metalink: %w", err)) + return 1 + } + tempFile.Close() + defer os.Remove(tempPath) + + // Process the downloaded metalink file + return handleMetalinkFile(tempPath, outputDir, verbose, addToQueue, priority, queueFile) +} + +// handleUpload handles file upload via PUT or POST +func handleUpload(targetURL, method, filePath, formDataStr, fileField string, verbose bool, cfg *config.Config) int { + if filePath == "" { + cli.PrintError(os.Stderr, fmt.Errorf("--upload requires --upload-file")) + return 1 + } + + // Check if file exists + if _, err := os.Stat(filePath); os.IsNotExist(err) { + cli.PrintError(os.Stderr, fmt.Errorf("file not found: %s", filePath)) + return 1 + } + + if verbose { + fmt.Fprintf(os.Stderr, "[upload] Uploading %s to %s (%s)\n", filePath, targetURL, method) + } + + // Create HTTP client + httpCfg := httpConfig.DefaultConfig() + httpCfg.Output.Verbose = verbose + httpCfg.Transport.Timeout = cfg.GetEffectiveTimeout(-1, 0) + httpCfg.Auth.Username = cfg.Auth.Username + httpCfg.Auth.Password = cfg.Auth.Password + httpCfg.Auth.AuthType = cfg.Auth.AuthType + httpCfg.Auth.OAuth = &cfg.Auth.OAuth + + if cfg.MaxSpeed > 0 { + httpCfg.RateLimiter = transport.NewTokenBucket(cfg.MaxSpeed, cfg.MaxSpeed/10) + } + + client, err := httpConfig.NewClient(httpCfg) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create http client: %w", err)) + return 1 + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigChan + cancel() + }() + + // Parse form data + formData := make(map[string]string) + if formDataStr != "" { + for _, pair := range strings.Split(formDataStr, ",") { + pair = strings.TrimSpace(pair) + if parts := strings.SplitN(pair, "=", 2); len(parts) == 2 { + formData[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + } + } + + var result *httpConfig.UploadResult + + if len(formData) > 0 || fileField != "" { + // Multipart POST upload + files := []httpConfig.UploadFile{ + { + FieldName: fileField, + FilePath: filePath, + }, + } + result, err = client.UploadMultipart(ctx, targetURL, files, formData, verbose) + } else { + // Simple PUT or POST upload + result, err = client.UploadSimple(ctx, method, targetURL, filePath, verbose) + } + + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("upload failed: %w", err)) + return 1 + } + + if verbose { + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "[upload] Upload complete\n") + fmt.Fprintf(os.Stderr, " Status: %d\n", result.StatusCode) + fmt.Fprintf(os.Stderr, " Uploaded: %s in %v (%s/s)\n", + format.Bytes(result.BytesUploaded), + result.Duration.Round(time.Second), + format.Speed(result.Speed)) + if result.ContentLength > 0 { + fmt.Fprintf(os.Stderr, " Response: %s\n", format.Bytes(result.ContentLength)) + } + if result.ResponseBody != "" { + fmt.Fprintf(os.Stderr, " Response body: %s\n", result.ResponseBody) + } + } else { + fmt.Printf("Upload complete: %d %s\n", result.StatusCode, format.Bytes(result.BytesUploaded)) + } + + return 0 +} + +// handleBatchFile downloads multiple URLs from a file +func handleBatchFile(batchPath, outputDir string, cfg *config.Config, ctx context.Context) int { + data, err := os.ReadFile(batchPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to read batch file: %w", err)) + return 1 + } + + urls := strings.Split(strings.TrimSpace(string(data)), "\n") + var validURLs []string + for _, line := range urls { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { + continue + } + validURLs = append(validURLs, line) + } + + if len(validURLs) == 0 { + cli.PrintError(os.Stderr, fmt.Errorf("no valid URLs found in batch file")) + return 1 + } + + if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("Batch downloading %d URL(s) from %s", + len(validURLs), batchPath)) + } + + var successCount, failCount int + var totalBytes int64 + startTime := time.Now() + + for i, urlStr := range validURLs { + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "\n[%d/%d] %s\n", i+1, len(validURLs), + core.SafeURL(mustParseURL(urlStr))) + } + + // Auto-generate output filename from URL + outputPath := outputDir + if outputPath == "" || outputPath == "." { + outputPath = urlToFilename(urlStr, i+1) + } else if fi, err := os.Stat(outputPath); err == nil && fi.IsDir() { + outputPath = filepath.Join(outputPath, urlToFilename(urlStr, i+1)) + } + + result, err := downloadSingleURL(urlStr, outputPath, cfg, cfg.Verbose, ctx) + if err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed: %s - %v", + core.SafeURL(mustParseURL(urlStr)), err)) + failCount++ + continue + } + + totalBytes += result.BytesDownloaded + successCount++ + + if cfg.Verbose { + cli.PrintSuccess(os.Stderr, fmt.Sprintf("Downloaded: %s (%s)", + filepath.Base(outputPath), format.Bytes(result.BytesDownloaded))) + } + } + + duration := time.Since(startTime) + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "\nBatch complete: %d succeeded, %d failed, %s total in %v\n", + successCount, failCount, format.Bytes(totalBytes), duration.Round(time.Second)) + } + + if failCount > 0 { + return 1 + } + return 0 +} + +// handleCompletion generates shell completion script +func handleCompletion(shell string) int { + var script string + switch strings.ToLower(shell) { + case "bash": + script = generateBashCompletion() + case "zsh": + script = generateZshCompletion() + case "fish": + script = generateFishCompletion() + default: + cli.PrintError(os.Stderr, fmt.Errorf("unsupported shell: %s (supported: bash, zsh, fish)", shell)) + return 1 + } + fmt.Print(script) + return 0 +} + +// handleInfo shows file information from HEAD request +func handleInfo(urlStr string, cfg *config.Config) int { + parsedURL, err := url.Parse(urlStr) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid url: %w", err)) + return 1 + } + + client := &stdhttp.Client{ + Timeout: 30 * time.Second, + } + + req, err := stdhttp.NewRequest("HEAD", urlStr, nil) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create request: %w", err)) + return 1 + } + req.Header.Set("User-Agent", cfg.UserAgent) + + resp, err := client.Do(req) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("head request failed: %w", err)) + return 1 + } + resp.Body.Close() + + fmt.Printf("URL: %s\n", parsedURL.String()) + fmt.Printf("Status: %s\n", resp.Status) + fmt.Printf("Content-Type: %s\n", resp.Header.Get("Content-Type")) + fmt.Printf("Content-Length: %s (%d bytes)\n", format.Bytes(resp.ContentLength), resp.ContentLength) + fmt.Printf("Last-Modified: %s\n", resp.Header.Get("Last-Modified")) + fmt.Printf("ETag: %s\n", resp.Header.Get("ETag")) + fmt.Printf("Accept-Ranges: %s\n", resp.Header.Get("Accept-Ranges")) + fmt.Printf("Server: %s\n", resp.Header.Get("Server")) + + // Check if supports compression + if resp.Header.Get("Content-Encoding") != "" { + fmt.Printf("Content-Encoding: %s\n", resp.Header.Get("Content-Encoding")) + } + + return 0 +} + +// handleBenchmark benchmarks download speed from a URL +func handleBenchmark(urlStr string, cfg *config.Config) int { + fmt.Printf("Benchmarking: %s\n\n", urlStr) + + _, err := url.Parse(urlStr) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid url: %w", err)) + return 1 + } + + client := &stdhttp.Client{ + Timeout: 30 * time.Second, + } + + // First, send a HEAD request to get file info + headReq, err := stdhttp.NewRequest("HEAD", urlStr, nil) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create head request: %w", err)) + return 1 + } + headReq.Header.Set("User-Agent", cfg.UserAgent) + + headResp, err := client.Do(headReq) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("head request failed: %w", err)) + return 1 + } + headResp.Body.Close() + + totalSize := headResp.ContentLength + fmt.Printf("File size: %s\n", format.Bytes(totalSize)) + fmt.Printf("Server: %s\n", headResp.Header.Get("Server")) + fmt.Printf("Content-Type: %s\n", headResp.Header.Get("Content-Type")) + + // Download a portion to benchmark (first 10MB or whole file if smaller) + downloadSize := int64(10 * 1024 * 1024) // 10MB + if totalSize > 0 && totalSize < downloadSize { + downloadSize = totalSize + } + + req, err := stdhttp.NewRequest("GET", urlStr, nil) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create get request: %w", err)) + return 1 + } + req.Header.Set("User-Agent", cfg.UserAgent) + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", downloadSize-1)) + + start := time.Now() + resp, err := client.Do(req) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("get request failed: %w", err)) + return 1 + } + defer resp.Body.Close() + + // Read the benchmark data + buf := make([]byte, 32*1024) + var totalRead int64 + for { + n, err := resp.Body.Read(buf) + totalRead += int64(n) + if err != nil { + break + } + // Only read up to our download limit + if totalRead >= downloadSize { + break + } + } + + duration := time.Since(start) + speed := float64(totalRead) / duration.Seconds() + + fmt.Printf("\n--- Benchmark Results ---\n") + fmt.Printf("Downloaded: %s in %v\n", format.Bytes(totalRead), duration.Round(time.Millisecond)) + fmt.Printf("Speed: %s/s\n", format.Speed(speed)) + + // Show in different units + bitsPerSec := speed * 8 + fmt.Printf("Bandwidth: %.2f Mbps\n", bitsPerSec/1000000) + + // Estimate total download time + if totalSize > 0 && speed > 0 { + estTime := time.Duration(float64(totalSize)/speed) * time.Second + fmt.Printf("ETA (full): %v\n", estTime.Round(time.Second)) + } + + return 0 +} + +// handleWriteOut prints metadata after download in curl-compatible format. +func handleWriteOut(format string, result *core.DownloadResult, size int64, duration time.Duration, httpCode int) { + replacer := strings.NewReplacer( + "%{http_code}", fmt.Sprintf("%d", httpCode), + "%{size_download}", fmt.Sprintf("%d", result.BytesDownloaded), + "%{size_total}", fmt.Sprintf("%d", result.TotalSize), + "%{speed_download}", fmt.Sprintf("%d", int64(result.Speed)), + "%{time_total}", fmt.Sprintf("%.3f", duration.Seconds()), + "%{url}", result.OutputPath, + "%{protocol}", result.Protocol, + "%{num_connects}", fmt.Sprintf("%d", result.ChunksCount), + ) + fmt.Println(replacer.Replace(format)) +} + +// handleSpider checks URL existence via HEAD request without downloading. +func handleSpider(urlStr string, cfg *config.Config) int { + parsedURL, err := url.Parse(urlStr) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid url: %w", err)) + return 1 + } + + client := &stdhttp.Client{Timeout: 30 * time.Second} + req, err := stdhttp.NewRequest("HEAD", urlStr, nil) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to create request: %w", err)) + return 1 + } + req.Header.Set("User-Agent", cfg.UserAgent) + + resp, err := client.Do(req) + if err != nil { + fmt.Printf("%s: error (%v)\n", parsedURL.String(), err) + return 1 + } + resp.Body.Close() + + fmt.Printf("%s: HTTP %d %s\n", parsedURL.String(), resp.StatusCode, resp.Status) + if resp.StatusCode >= 400 { + return 1 + } + return 0 +} + +// handleGenManPage generates a man page for goget + +// handleInitConfig generates a default config file +func handleInitConfig(configPath string) int { + cfg, err := config.Load(configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + if err := cfg.Save(); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to save config: %w", err)) + return 1 + } + fmt.Printf("Config saved to %s\n", cfg.Path()) + return 0 +} + +// handleConfigList prints all config values +func handleConfigList(configPath string) int { + if configPath == "" { + fmt.Println("No config file loaded") + return 0 + } + + cfg, err := config.Load(configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + + data, err := interpres.Marshal(*cfg) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to marshal config: %w", err)) + return 1 + } + + fmt.Print(string(data)) + return 0 +} + +// handleConfigGet prints a single config value +func handleConfigGet(configPath, key string) int { + if configPath == "" { + fmt.Println("No config file loaded") + return 0 + } + + cfg, err := config.Load(configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + + // Use reflection to get the value + cfgMap := make(map[string]interface{}) + data, _ := json.Marshal(cfg) + json.Unmarshal(data, &cfgMap) + + value, exists := cfgMap[key] + if !exists { + cli.PrintError(os.Stderr, fmt.Errorf("unknown config key: %s", key)) + return 1 + } + + fmt.Printf("%s = %v\n", key, value) + return 0 +} + +// handleConfigSet sets a config value and saves +func handleConfigSet(configPath, key string, args []string) int { + if configPath == "" { + cli.PrintError(os.Stderr, fmt.Errorf("no config file loaded, use --init-config first")) + return 1 + } + + // Find the value from args (--config-set key value) + var value string + for i := 0; i < len(args); i++ { + if args[i] == "--config-set" && i+2 < len(args) { + value = args[i+2] + break + } + } + + if value == "" { + cli.PrintError(os.Stderr, fmt.Errorf("--config-set requires a value: --config-set key value")) + return 1 + } + + cfg, err := config.Load(configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + + // Parse value to the right type + cfgMap := make(map[string]interface{}) + data, _ := json.Marshal(cfg) + json.Unmarshal(data, &cfgMap) + + // Try to parse as number first + if numVal, err := strconv.ParseInt(value, 10, 64); err == nil { + cfgMap[key] = numVal + } else if floatVal, err := strconv.ParseFloat(value, 64); err == nil { + cfgMap[key] = floatVal + } else if boolVal, err := strconv.ParseBool(value); err == nil { + cfgMap[key] = boolVal + } else { + cfgMap[key] = value + } + + // Marshal the modified map back to a Config struct, then save as TOML. + jsonData, _ := json.Marshal(cfgMap) + var updatedCfg config.Config + if err := json.Unmarshal(jsonData, &updatedCfg); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to parse updated config: %w", err)) + return 1 + } + + tomlData, err := interpres.Marshal(updatedCfg) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to marshal config: %w", err)) + return 1 + } + + if err := os.WriteFile(configPath, tomlData, 0600); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to save config: %w", err)) + return 1 + } + + fmt.Printf("Set %s = %v in %s\n", key, cfgMap[key], configPath) + return 0 +} + +// handleConfigUnset removes a config key +func handleConfigUnset(configPath, key string) int { + if configPath == "" { + cli.PrintError(os.Stderr, fmt.Errorf("no config file loaded")) + return 1 + } + + cfg, err := config.Load(configPath) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to load config: %w", err)) + return 1 + } + + cfgMap := make(map[string]interface{}) + data, _ := json.Marshal(cfg) + json.Unmarshal(data, &cfgMap) + + delete(cfgMap, key) + + // Marshal the modified map back to a Config struct, then save as TOML. + jsonData, _ := json.Marshal(cfgMap) + var updatedCfg config.Config + if err := json.Unmarshal(jsonData, &updatedCfg); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to parse updated config: %w", err)) + return 1 + } + + tomlData, err := interpres.Marshal(updatedCfg) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to marshal config: %w", err)) + return 1 + } + + if err := os.WriteFile(configPath, tomlData, 0600); err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to save config: %w", err)) + return 1 + } + + fmt.Printf("Unset %s in %s\n", key, configPath) + return 0 +} diff --git a/cmd/goget/completion.go b/cmd/goget/completion.go new file mode 100644 index 0000000..619820b --- /dev/null +++ b/cmd/goget/completion.go @@ -0,0 +1,185 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +func generateBashCompletion() string { + return `# goget bash completion +_goget() { + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + opts="--url --output --resume --overwrite --verbose --debug + --timeout --proxy --no-ipv4 --checksum --checksum-algo + --no-decompress --parallel --max-speed --username --password + --password-file --auth-type --cookie-jar --cookie --recursive --recursive-parallel --max-depth + --follow-external --mirror --no-convert-links --no-mirror-assets + --no-robots --rate-limit --exclude-pattern --extract --extract-dir + --strip-components --upload --upload-method --upload-file + --form-data --file-field --pgp-decrypt --pgp-verify --pgp-sig + --pgp-key --pgp-passphrase --oauth-client-id --oauth-client-secret + --oauth-token-url --oauth-auth-url --oauth-redirect-uri + --oauth-scopes --oauth-grant-type --oauth-access-token + --oauth-refresh-token --metalink --metalink-file --queue + --queue-list --queue-process --queue-clear --queue-file + --queue-priority --help --version --batch-file --quiet + --init-config --content-disposition --restart --json --info + --show-headers --keep-timestamps --dns-servers --max-retries + --no-redirect --retry-all-errors --ssl-key-log --completion --config" + + case "${prev}" in + --url|--output|--proxy|--checksum|--password|--password-file|--cookie-jar) + return 0 + ;; + --timeout|--max-speed|--rate-limit) + return 0 + ;; + --batch-file|--metalink-file|--pgp-sig|--pgp-key|--pgp-passphrase) + return 0 + ;; + --dns-servers|--max-retries|--parallel|--recursive-parallel|--max-depth|--queue-priority) + return 0 + ;; + --completion) + COMPREPLY=($(compgen -W "bash zsh fish" -- ${cur})) + return 0 + ;; + --auth-type) + COMPREPLY=($(compgen -W "basic digest auto" -- ${cur})) + return 0 + ;; + --checksum-algo) + COMPREPLY=($(compgen -W "sha256 sha512 blake2b sha3-256 sha3-512 md5" -- ${cur})) + return 0 + ;; + --upload-method) + COMPREPLY=($(compgen -W "PUT POST" -- ${cur})) + return 0 + ;; + --exclude-pattern|--form-data|--oauth-scopes) + return 0 + ;; + esac + + if [[ ${cur} == -* ]]; then + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + fi +} +complete -F _goget goget +` +} + +func generateZshCompletion() string { + return `#compdef goget + +_goget() { + local -a opts + opts=( + '--url[URL to download]:url:_urls' + '--output[Output file]:file:_files' + '--verbose[Verbose output]' + '--debug[Debug mode]' + '--resume[Resume interrupted download]' + '--quiet[Suppress all output except errors]' + '--json[JSON progress output]' + '--info[Show file information without downloading]' + '--show-headers[Print HTTP response headers]' + '--keep-timestamps[Set file modification time from server]' + '--restart[Delete existing file and restart download]' + '--timeout[Timeout (0 = auto)]:timeout' + '--proxy[Proxy server URL]:proxy:_urls' + '--no-ipv4[Disable IPv4 fallback]' + '--checksum[Expected checksum]:checksum' + '--checksum-algo[Checksum algorithm]:algorithm:(sha256 sha512 blake2b sha3-256 sha3-512 md5)' + '--no-decompress[Disable automatic decompression]' + '--parallel[Number of parallel connections]:number' + '--max-speed[Max speed]:speed' + '--username[Username]:username' + '--password[Password]:password' + '--password-file[Read password from file]:file:_files' + '--auth-type[Auth type]:(basic digest auto)' + '--batch-file[Download URLs from file]:file:_files' + '--cookie[Cookie name=value]' + '--retry-all-errors[Retry on any HTTP error]' + '--ssl-key-log[TLS key log file]:file:_files' + '--max-retries[Maximum retries]:number' + '--dns-servers[Custom DNS servers]:dns' + '--no-redirect[Disable following redirects]' + '--completion[Generate completion]:shell:(bash zsh fish)' + '--content-disposition[Use server-provided filename]' + '--init-config[Generate default config file]' + '--help[Show help]' + '--version[Show version]' + ) + _describe 'goget' opts +} + +compdef _goget goget +` +} + +func generateFishCompletion() string { + return `# goget fish completion +function __fish_goget_no_subcommand + for i in (commandline -opc) + if contains -- $i (__fish_goget_commands) + return 1 + end + end + return 0 +end + +function __fish_goget_commands + echo url output resume overwrite verbose debug + echo timeout proxy no-ipv4 checksum checksum-algo + echo no-decompress parallel max-speed username password + echo password-file auth-type cookie-jar cookie recursive recursive-parallel max-depth + echo retry-all-errors ssl-key-log + echo follow-external mirror no-convert-links no-mirror-assets + echo no-robots rate-limit exclude-pattern extract extract-dir + echo strip-components upload upload-method upload-file + echo form-data file-field pgp-decrypt pgp-verify pgp-sig + echo pgp-key pgp-passphrase help version batch-file quiet + echo init-config content-disposition restart json info + echo show-headers keep-timestamps dns-servers max-retries + echo no-redirect completion +end + +# Flags +complete -c goget -l url -d "URL to download" -r +complete -c goget -l output -d "Output file" -r +complete -c goget -l verbose -d "Verbose output" +complete -c goget -l debug -d "Debug mode" +complete -c goget -l resume -d "Resume interrupted download" +complete -c goget -l quiet -d "Suppress all output except errors" +complete -c goget -l json -d "JSON progress output" +complete -c goget -l info -d "Show file information without downloading" +complete -c goget -l show-headers -d "Print HTTP response headers" +complete -c goget -l keep-timestamps -d "Set file modification time from server" +complete -c goget -l restart -d "Delete existing file and restart download" +complete -c goget -l init-config -d "Generate default config file" +complete -c goget -l content-disposition -d "Use server-provided filename" +complete -c goget -l no-redirect -d "Disable following redirects" +complete -c goget -l batch-file -d "Download URLs from file" -r +complete -c goget -l max-retries -d "Maximum retries" -r +complete -c goget -l dns-servers -d "Custom DNS servers" -r +complete -c goget -l completion -d "Generate completion" -r -xa "bash zsh fish" +complete -c goget -l timeout -d "Timeout (0 = auto)" -r +complete -c goget -l proxy -d "Proxy server URL" -r +complete -c goget -l checksum -d "Expected checksum" -r +complete -c goget -l checksum-algo -d "Algorithm" -r -xa "sha256 sha512 blake2b sha3-256 sha3-512 md5" +complete -c goget -l parallel -d "Parallel connections" -r +complete -c goget -l max-speed -d "Max speed" -r +complete -c goget -l username -d "Username" -r +complete -c goget -l password -d "Password" -r +complete -c goget -l password-file -d "Read password from file" -r +complete -c goget -l auth-type -d "Auth type" -r -xa "basic digest auto" +complete -c goget -l cookie -d "Cookie name=value" -r +complete -c goget -l retry-all-errors -d "Retry on any HTTP error" +complete -c goget -l ssl-key-log -d "TLS key log file" -r +complete -c goget -l help -d "Show help" +complete -c goget -l version -d "Show version" +` +} diff --git a/cmd/goget/flags.go b/cmd/goget/flags.go new file mode 100644 index 0000000..7d25902 --- /dev/null +++ b/cmd/goget/flags.go @@ -0,0 +1,316 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "flag" + "strings" + "time" + + "codeberg.org/petrbalvin/goget/internal/config" +) + +// headerFlags accumulates multiple --header values. +type headerFlags []string + +func (h *headerFlags) String() string { return strings.Join(*h, ", ") } +func (h *headerFlags) Set(v string) error { + *h = append(*h, v) + return nil +} + +// formFlags accumulates multiple --form values. +type formFlags []string + +func (f *formFlags) String() string { return strings.Join(*f, ", ") } +func (f *formFlags) Set(v string) error { + *f = append(*f, v) + return nil +} + +// Flags holds all CLI flag values. +type Flags struct { + URL *string + Output *string + Resume *bool + Overwrite *bool + Restart *bool + Verbose *bool + Quiet *bool + NoProgress *bool + JSON *bool + Debug *bool + Timeout *time.Duration + Proxy *string + NoIPv4 *bool + NoRedirect *bool + MaxRetries *int + DNSServers *string + Checksum *string + ChecksumAlgo *string + ChecksumFile *string + NoDecompress *bool + Parallel *int + MaxSpeed *string + Username *string + Password *string + PasswordFile *string + AuthType *string + CookieJar *string + ContentDisp *bool + ShowHeaders *bool + KeepTimestamps *bool + Recursive *bool + RecursiveParallel *int + MaxDepth *int + FollowExternal *bool + Mirror *bool + NoConvertLinks *bool + NoMirrorAssets *bool + NoRobots *bool + RateLimit *string + ExcludePattern *string + Extract *bool + ExtractDir *string + StripComponents *int + Upload *bool + UploadMethod *string + UploadFile *string + UploadFormData *string + UploadFileField *string + PGPDecrypt *bool + PGPVerify *bool + PGPSignature *string + PGPKey *string + PGPPassphrase *string + PGPPassphraseFile *string + OAuthClientID *string + OAuthClientSecret *string + OAuthTokenURL *string + OAuthAuthURL *string + OAuthRedirectURI *string + OAuthScopes *string + OAuthGrantType *string + OAuthAccessToken *string + OAuthRefreshToken *string + Metalink *bool + MetalinkFile *string + Queue *bool + QueueList *bool + QueueProcess *bool + QueueClear *bool + QueueFile *string + QueuePriority *int + BatchFile *string + InitConfig *bool + ConfigGet *string + ConfigSet *string + ConfigList *bool + ConfigUnset *string + Completion *string + Info *string + Benchmark *string + Schedule *string + BindInterface *string + KnownHosts *string + SSHInsecure *bool + DryRun *bool + OnComplete *string + GenManPage *bool + Help *bool + Version *bool + // curl/wget compatible flags + Headers headerFlags + Cookies headerFlags // --cookie, repeatable + Insecure *bool + CertFile *string + KeyFile *string + SSLKeyLogFile *string + Data *string + MaxTime *time.Duration + MaxFileSize *string + Accept *string + NoParent *bool + Spider *bool + WriteOut *string + ProgressStyle *string + WarcFile *string + TraceTime *bool + Form formFlags + PageRequisites *bool + SpanHosts *bool + AdjustExt *bool + NoPrivate *bool // deprecated, use AllowPrivate inversely + Wait *string + RandomWait *bool + ConvertLinks *bool + Fail *bool + Compressed *bool + Referer *string + RetryDelay *time.Duration + PinnedCert *string + AllowPrivate *bool + Range *string + ConnectTimeout *time.Duration + Domains *string + Reject *string + CutDirs *int + PostFile *string + CACertificate *string + NoClobber *bool + RetryHTTPError *string + RetryAllErrors *bool + Glob *bool + InputFile *string + Timestamping *bool + BackupConverted *bool + Continue *bool + ProtoDirs *bool + HSTSFile *string + CreateDirs *bool +} + +// defineFlags registers all CLI flags on the given FlagSet and returns them. +func defineFlags(fs *flag.FlagSet, cfg *config.Config) *Flags { + f := &Flags{ + URL: fs.String("url", "", "URL to download (can be specified multiple times)"), + Output: fs.String("output", "", "Output file path"), + Resume: fs.Bool("resume", cfg.AutoResume, "Resume interrupted download"), + Overwrite: fs.Bool("overwrite", false, "Overwrite existing file"), + Restart: fs.Bool("restart", false, "Delete existing file and restart download from scratch"), + Verbose: fs.Bool("verbose", cfg.Verbose, "Verbose output"), + Quiet: fs.Bool("quiet", false, "Suppress all output except errors"), + NoProgress: fs.Bool("no-progress", false, "Hide progress bar (keep other output)"), + JSON: fs.Bool("json", false, "Output machine-readable progress as JSON lines"), + Debug: fs.Bool("debug", cfg.Debug, "Debug mode"), + Timeout: fs.Duration("timeout", 0, "Timeout (0 = auto)"), + Proxy: fs.String("proxy", cfg.Proxy, "Proxy server URL (http://, https://, socks5://, socks5h://)"), + NoIPv4: fs.Bool("no-ipv4", !cfg.IPv4Fallback, "Disable IPv4 fallback"), + NoRedirect: fs.Bool("no-redirect", false, "Disable following HTTP redirects"), + MaxRetries: fs.Int("max-retries", 0, "Maximum number of retries on failure (0 = default)"), + DNSServers: fs.String("dns-servers", "", "Custom DNS servers (comma-separated, e.g. 1.1.1.1,8.8.8.8)"), + Checksum: fs.String("checksum", "", "Expected checksum value"), + ChecksumAlgo: fs.String("checksum-algo", cfg.ChecksumAlgo, "Checksum algorithm (sha256, sha512, blake2b, sha3-256, sha3-512, md5)"), + ChecksumFile: fs.String("checksum-file", "", "Read checksum from file (SHA256SUMS format)"), + NoDecompress: fs.Bool("no-decompress", !cfg.AutoDecompress, "Disable automatic decompression"), + Parallel: fs.Int("parallel", cfg.Parallel, "Number of parallel connections (0 = auto)"), + MaxSpeed: fs.String("max-speed", "", "Max speed (e.g., 10MB/s, 0 = unlimited)"), + Username: fs.String("username", cfg.Auth.Username, "Username for HTTP Basic/Digest Auth"), + Password: fs.String("password", "", "Password for HTTP Basic/Digest Auth"), + PasswordFile: fs.String("password-file", "", "Read password from file"), + AuthType: fs.String("auth-type", cfg.Auth.AuthType, "Auth type: basic, digest, auto"), + CookieJar: fs.String("cookie-jar", cfg.CookieJar, "File for loading/saving cookies"), + ContentDisp: fs.Bool("content-disposition", false, "Use server-provided filename from Content-Disposition header"), + ShowHeaders: fs.Bool("show-headers", false, "Print HTTP response headers during download"), + KeepTimestamps: fs.Bool("keep-timestamps", false, "Set file modification time from Last-Modified header"), + Recursive: fs.Bool("recursive", cfg.Recursive.Enabled, "Recursive downloading"), + RecursiveParallel: fs.Int("recursive-parallel", cfg.Recursive.Parallel, "Number of concurrent file downloads in recursive mode (0 = sequential)"), + MaxDepth: fs.Int("max-depth", cfg.Recursive.MaxDepth, "Maximum recursion depth"), + FollowExternal: fs.Bool("follow-external", cfg.Recursive.FollowExternal, "Follow external domains"), + Mirror: fs.Bool("mirror", false, "Mirror entire website (infinite depth, convert links)"), + NoConvertLinks: fs.Bool("no-convert-links", false, "Don't convert links for local viewing in mirror mode"), + NoMirrorAssets: fs.Bool("no-mirror-assets", false, "Don't download assets (CSS, JS, images) in mirror mode"), + NoRobots: fs.Bool("no-robots", false, "Don't respect robots.txt in mirror mode"), + RateLimit: fs.String("rate-limit", "", "Rate limit (e.g., 100KB/s, 1MB/s, 0 = unlimited)"), + ExcludePattern: fs.String("exclude-pattern", strings.Join(cfg.Recursive.ExcludePatterns, ","), "Exclude URL patterns (comma-separated)"), + Extract: fs.Bool("extract", cfg.Extract.Enabled, "Auto-extract archive after download"), + ExtractDir: fs.String("extract-dir", cfg.Extract.OutputDir, "Directory for extracted files"), + StripComponents: fs.Int("strip-components", cfg.Extract.StripComponents, "Strip N path components during extraction"), + Upload: fs.Bool("upload", false, "Upload file instead of downloading"), + UploadMethod: fs.String("upload-method", "PUT", "Upload method: PUT or POST"), + UploadFile: fs.String("upload-file", "", "File to upload"), + UploadFormData: fs.String("form-data", "", "Form data (key=value, comma-separated)"), + UploadFileField: fs.String("file-field", "file", "Form field name for file upload"), + PGPDecrypt: fs.Bool("pgp-decrypt", false, "Decrypt PGP encrypted file after download"), + PGPVerify: fs.Bool("pgp-verify", false, "Verify PGP signature after download"), + PGPSignature: fs.String("pgp-sig", "", "Path to PGP signature file (.asc, .sig)"), + PGPKey: fs.String("pgp-key", "", "Path to PGP key file (public or private)"), + PGPPassphrase: fs.String("pgp-passphrase", "", "Passphrase for PGP private key"), + PGPPassphraseFile: fs.String("pgp-passphrase-file", "", "Read PGP passphrase from file"), + OAuthClientID: fs.String("oauth-client-id", cfg.Auth.OAuth.ClientID, "OAuth 2.0 Client ID"), + OAuthClientSecret: fs.String("oauth-client-secret", cfg.Auth.OAuth.ClientSecret, "OAuth 2.0 Client Secret"), + OAuthTokenURL: fs.String("oauth-token-url", cfg.Auth.OAuth.TokenURL, "OAuth 2.0 Token URL"), + OAuthAuthURL: fs.String("oauth-auth-url", cfg.Auth.OAuth.AuthURL, "OAuth 2.0 Authorization URL"), + OAuthRedirectURI: fs.String("oauth-redirect-uri", cfg.Auth.OAuth.RedirectURI, "OAuth 2.0 Redirect URI"), + OAuthScopes: fs.String("oauth-scopes", strings.Join(cfg.Auth.OAuth.Scopes, ","), "OAuth 2.0 Scopes (comma-separated)"), + OAuthGrantType: fs.String("oauth-grant-type", cfg.Auth.OAuth.GrantType, "OAuth 2.0 Grant Type (client_credentials, authorization_code, password, refresh_token)"), + OAuthAccessToken: fs.String("oauth-access-token", cfg.Auth.OAuth.AccessToken, "OAuth 2.0 Access Token (direct)"), + OAuthRefreshToken: fs.String("oauth-refresh-token", cfg.Auth.OAuth.RefreshToken, "OAuth 2.0 Refresh Token"), + Metalink: fs.Bool("metalink", false, "Treat URL as Metalink (multi-source download)"), + MetalinkFile: fs.String("metalink-file", "", "Path to local Metalink file"), + Queue: fs.Bool("queue", false, "Add to download queue instead of downloading immediately"), + QueueList: fs.Bool("queue-list", false, "List items in download queue"), + QueueProcess: fs.Bool("queue-process", false, "Process download queue"), + QueueClear: fs.Bool("queue-clear", false, "Clear completed items from queue"), + QueueFile: fs.String("queue-file", "", "Custom queue file path"), + QueuePriority: fs.Int("queue-priority", 5, "Priority for queue item (1-10)"), + BatchFile: fs.String("batch-file", "", "Download multiple URLs from a file (one per line)"), + InitConfig: fs.Bool("init-config", false, "Generate default config file"), + ConfigGet: fs.String("config-get", "", "Get a config value (e.g. --config-get timeout)"), + ConfigSet: fs.String("config-set", "", "Set a config value (e.g. --config-set timeout 5m)"), + ConfigList: fs.Bool("config-list", false, "List all config values"), + ConfigUnset: fs.String("config-unset", "", "Unset a config value (e.g. --config-unset timeout)"), + Completion: fs.String("completion", "", "Generate shell completion script (bash, zsh, fish)"), + Info: fs.String("info", "", "Show file information without downloading"), + Benchmark: fs.String("benchmark", "", "Benchmark download speed from URL (downloads first 10MB)"), + Schedule: fs.String("schedule", "", "Schedule download at specified time (e.g. '2026-06-01 02:00' or cron '0 2 * * *')"), + BindInterface: fs.String("bind-interface", "", "Bind to specific network interface"), + KnownHosts: fs.String("known-hosts", "", "Path to SSH known_hosts file for SFTP (default: ~/.ssh/known_hosts)"), + SSHInsecure: fs.Bool("ssh-insecure", false, "Skip SSH host key verification for SFTP"), + DryRun: fs.Bool("dry-run", false, "List URLs without saving files in recursive/mirror mode"), + OnComplete: fs.String("on-complete", "", "Run command after successful download"), + GenManPage: fs.Bool("generate-man-page", false, "Generate man page and exit"), + Help: fs.Bool("help", false, "Show help"), + Version: fs.Bool("version", false, "Show version"), + // curl/wget compatible flags + Insecure: fs.Bool("insecure", false, "Skip TLS certificate verification"), + CertFile: fs.String("cert", "", "Client certificate file (PEM, for mTLS)"), + KeyFile: fs.String("key", "", "Client private key file (PEM)"), + SSLKeyLogFile: fs.String("ssl-key-log", "", "File to log TLS session keys to (SSLKEYLOGFILE for Wireshark)"), + Data: fs.String("data", "", "Send POST data (e.g., --data 'key=value')"), + MaxTime: fs.Duration("max-time", 0, "Maximum total transfer time (0 = unlimited)"), + MaxFileSize: fs.String("max-filesize", "", "Maximum file size to download (e.g., 100MB)"), + Accept: fs.String("accept", "", "Accept patterns for recursive download (e.g., '*.pdf,*.html')"), + NoParent: fs.Bool("no-parent", false, "Do not ascend to parent directory in recursive download"), + Spider: fs.Bool("spider", false, "Check URL existence only (HEAD request, no download)"), + WriteOut: fs.String("write-out", "", "Output metadata after download (curl format: %{http_code}, %{size_download}, %{time_total})"), + ProgressStyle: fs.String("progress", "", "Progress display style: bar (default) or dot"), + WarcFile: fs.String("warc-file", "", "Write WARC output file for web archiving"), + TraceTime: fs.Bool("trace-time", false, "Print HTTP timing breakdown after download"), + PageRequisites: fs.Bool("page-requisites", false, "Download all assets (CSS, JS, images) needed to display the page"), + SpanHosts: fs.Bool("span-hosts", false, "Follow links to external domains in recursive download"), + AdjustExt: fs.Bool("adjust-extension", false, "Append .html extension to downloaded HTML files"), + Wait: fs.String("wait", "", "Wait between requests in recursive/mirror mode (e.g., 1s, 500ms)"), + RandomWait: fs.Bool("random-wait", false, "Randomize wait time (0.5x to 1.5x) in recursive/mirror mode"), + ConvertLinks: fs.Bool("convert-links", false, "Convert links for local viewing in recursive mode"), + Fail: fs.Bool("fail", false, "Exit with error on HTTP 4xx/5xx status codes"), + Compressed: fs.Bool("compressed", false, "Send Accept-Encoding header for auto-decompression"), + Referer: fs.String("referer", "", "Send Referer header"), + RetryDelay: fs.Duration("retry-delay", 0, "Delay between retry attempts"), + PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning"), + AllowPrivate: fs.Bool("allow-private", false, "Allow connections to private/internal IPs (disables SSRF protection)"), + Range: fs.String("range", "", "Download only a range of bytes (e.g., 0-1000)"), + ConnectTimeout: fs.Duration("connect-timeout", 0, "Timeout for connection establishment"), + Domains: fs.String("domains", "", "Only follow these domains in recursive mode (comma-separated)"), + Reject: fs.String("reject", "", "Reject URL patterns in recursive mode (glob, comma-separated)"), + CutDirs: fs.Int("cut-dirs", 0, "Strip N directory levels when saving files"), + PostFile: fs.String("post-file", "", "Send file contents as POST body"), + CACertificate: fs.String("ca-certificate", "", "Path to custom CA certificate"), + NoClobber: fs.Bool("no-clobber", false, "Skip download if output file already exists"), + RetryHTTPError: fs.String("retry-on-http-error", "", "Retry on these HTTP status codes (comma-separated, e.g. 503,429)"), + RetryAllErrors: fs.Bool("retry-all-errors", false, "Retry on any HTTP error (4xx/5xx) instead of just selected codes"), + Glob: fs.Bool("glob", false, "Expand [1-3], {a,b} patterns in URL"), + Timestamping: fs.Bool("timestamping", false, "Download only if remote file is newer"), + BackupConverted: fs.Bool("backup-converted", false, "Backup original file before link conversion"), + Continue: fs.Bool("continue", false, "Auto-resume download if output file exists"), + ProtoDirs: fs.Bool("protocol-directories", false, "Create .http/, .ftp/ subdirectories in recursive mode"), + InputFile: fs.String("input-file", "", "Read URLs from file (use - for stdin)"), + HSTSFile: fs.String("hsts-file", "", "HSTS cache file (default: ~/.config/goget/hsts)"), + CreateDirs: fs.Bool("create-dirs", true, "Create missing parent directories of --output (curl --create-dirs)"), + } + fs.Var(&f.Headers, "header", "Custom HTTP header (repeatable, e.g. --header 'Key: Value')") + fs.Var(&f.Cookies, "cookie", "Set cookie (repeatable, e.g. --cookie 'name=value')") + fs.Var(&f.Form, "form", "Multipart form data (repeatable, e.g. --form 'field=value' or --form 'field=@file')") + return f +} diff --git a/cmd/goget/main.go b/cmd/goget/main.go new file mode 100644 index 0000000..39e6fee --- /dev/null +++ b/cmd/goget/main.go @@ -0,0 +1,356 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "codeberg.org/petrbalvin/goget/internal/archive" + "codeberg.org/petrbalvin/goget/internal/cli" + "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/cookie" + "codeberg.org/petrbalvin/goget/internal/core" + format "codeberg.org/petrbalvin/goget/internal/format" + "codeberg.org/petrbalvin/goget/internal/hsts" + "codeberg.org/petrbalvin/goget/internal/output" + "codeberg.org/petrbalvin/goget/internal/protocol/http" + "codeberg.org/petrbalvin/goget/internal/protocol/webdav" + "codeberg.org/petrbalvin/goget/internal/warc" + + "golang.org/x/net/idna" +) + +func main() { + os.Exit(run()) +} + +// signalContext returns a context that is cancelled when SIGINT or SIGTERM +// is received. Extracted as a helper so it can be unit-tested in +// isolation (testing the real signal flow inside run() would race with +// other tests in the same package). +func signalContext() (context.Context, context.CancelFunc) { + return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) +} + +// run is a thin wrapper that delegates to the app struct's run method. +func run() int { + var a app + return a.run() +} + +// setupOutputWriter creates the output writer and progress bar for non-recursive downloads. +func setupOutputWriter(f *Flags, cfg *config.Config) (*output.Writer, *cli.ProgressBar) { + var progressBar *cli.ProgressBar + if cfg.Verbose && !*f.Recursive && !*f.NoProgress { + progressCfg := cli.DefaultProgressBarConfig() + progressCfg.Total = -1 + progressCfg.Colors = cfg.ShouldUseColors() + progressBar = cli.NewProgressBar(progressCfg) + } + + var writer *output.Writer + if !*f.Recursive { + writerCfg := output.DefaultWriterConfig() + writerCfg.Output = *f.Output + writerCfg.Resume = *f.Resume + writerCfg.Verbose = cfg.Verbose + writerCfg.Atomic = !*f.Overwrite + writerCfg.CreateDirs = *f.CreateDirs + + if progressBar != nil || *f.JSON || *f.ProgressStyle == "dot" { + writerCfg.ProgressCallback = func(current, total int64, speed float64) { + if *f.ProgressStyle == "dot" { + // Wget-style dot progress: print dot per 100KB + if current > 0 && current%(100*1024) < 32*1024 { + fmt.Fprintf(os.Stderr, ".") + } + } + if *f.JSON { + // JSON-lines progress output + eta := "" + if total > 0 && current < total && speed > 0 { + remaining := float64(total-current) / speed + eta = fmt.Sprintf(`,"eta_ms":%d`, int64(remaining*1000)) + } + totalJSON := "null" + if total > 0 { + totalJSON = fmt.Sprintf("%d", total) + } + fmt.Printf(`{"type":"progress","downloaded":%d,"total":%s,"speed":%d%s}`+"\n", + current, totalJSON, int64(speed), eta) + } + if progressBar != nil && total > 0 { + progressBar.Update(current) + } + } + } + + var err error + writer, err = output.NewWriter(writerCfg) + if err != nil { + cli.PrintError(os.Stderr, err) + return nil, progressBar + } + } + return writer, progressBar +} + +// setupHTTPConfig populates an http.Config from CLI flags, parsed config, and cookie jar. +func setupHTTPConfig(f *Flags, cfg *config.Config, cookieJar *cookie.Jar, parsedURL *url.URL) *http.Config { + httpCfg := http.DefaultConfig() + ApplyFlags(httpCfg, f, cfg, cookieJar, parsedURL) + return httpCfg +} + +// runPostDownload handles WARC, cookie saving, checksum, PGP, write-out, trace-time, +// on-complete hook, verbose output, and HSTS save. Returns the exit code. +func runPostDownload( + f *Flags, + cfg *config.Config, + result *core.DownloadResult, + cookieJar *cookie.Jar, + cookieJarPath string, + hstsCache *hsts.Cache, + writer *output.Writer, + parsedURL *url.URL, + startTime time.Time, +) int { + if !*f.Recursive { + if cerr := writer.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to close writer: %v\n", cerr) + } + + // Adjust extension: append .html to HTML files without extension + if *f.AdjustExt && *f.Output != "" { + if filepath.Ext(*f.Output) == "" { + newPath := *f.Output + ".html" + if err := os.Rename(*f.Output, newPath); err == nil { + *f.Output = newPath + result.OutputPath = newPath + if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("Adjusted extension: %s", newPath)) + } + } + } + } + } + + // Dot progress newline + if *f.ProgressStyle == "dot" { + fmt.Fprintf(os.Stderr, "\n") + } + + // WARC writing + if *f.WarcFile != "" && !*f.Recursive && *f.Output != "" { + w := warc.NewWriter(*f.WarcFile) + if err := w.WriteRecordFile(core.SafeURL(parsedURL), *f.Output); err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("Failed to write WARC: %v", err)) + } else if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("WARC record saved to %s", *f.WarcFile)) + } + } + + if cookieJar != nil && cookieJarPath != "" { + if err := cookieJar.SaveToFile(cookieJarPath); err != nil { + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "Warning: failed to save cookies: %v\n", err) + } + } else if cfg.Verbose { + cli.PrintProgressInfo(os.Stderr, fmt.Sprintf("Saved cookies to %s", cookieJarPath)) + } + } + + if !*f.Recursive && *f.Checksum != "" && *f.Output != "" { + ok, err := verifyChecksum(*f.Output, *f.Checksum, cfg.ChecksumAlgo, cfg.Verbose) + if err != nil { + cli.PrintError(os.Stderr, err) + return 1 + } + if !ok { + return 1 + } + } + + // PGP passphrase from file (more secure than CLI flag) + if *f.PGPPassphraseFile != "" && *f.PGPPassphrase == "" { + data, err := os.ReadFile(*f.PGPPassphraseFile) + if err != nil { + cli.PrintError(os.Stderr, fmt.Errorf("failed to read pgp passphrase file: %w", err)) + return 1 + } + pass := strings.TrimSpace(string(data)) + f.PGPPassphrase = &pass + } + + // PGP verification and/or decryption + if !*f.Recursive && (*f.PGPVerify || *f.PGPDecrypt) && *f.Output != "" { + ok, err := verifyPGP(*f.Output, *f.PGPVerify, *f.PGPDecrypt, f.PGPKey, f.PGPPassphrase, f.PGPSignature, cfg.Verbose) + if err != nil { + cli.PrintError(os.Stderr, err) + return 1 + } + if !ok { + return 1 + } + } + + duration := time.Since(startTime) + + // JSON completion event + if *f.JSON && !*f.Recursive { + fmt.Printf(`{"type":"complete","downloaded":%d,"total":%d,"speed":%d,"elapsed_ms":%d,"output":"%s"}`+"\n", + result.BytesDownloaded, result.TotalSize, int64(result.Speed), duration.Milliseconds(), result.OutputPath) + } + + // On-complete hook — run command directly without shell to avoid injection + if *f.OnComplete != "" && !*f.Recursive { + if cfg.Verbose { + cli.PrintInfo(os.Stderr, fmt.Sprintf("Running: %s", *f.OnComplete)) + } + // Split command into executable and arguments (basic shell-like tokenization) + tokens := splitCommand(*f.OnComplete) + if len(tokens) == 0 { + cli.PrintWarning(os.Stderr, "On-complete hook: empty command") + } else { + cmd := exec.Command(tokens[0], tokens[1:]...) + cmd.Env = append(os.Environ(), + fmt.Sprintf("GOGET_OUTPUT=%s", result.OutputPath), + fmt.Sprintf("GOGET_SIZE=%d", result.BytesDownloaded), + fmt.Sprintf("GOGET_URL=%s", core.SafeURL(parsedURL)), + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("On-complete hook failed: %v", err)) + } + } + } + + if cfg.Verbose { + fmt.Fprintf(os.Stderr, "\n") + if *f.Recursive { + cli.PrintProgressSuccess(os.Stderr, fmt.Sprintf("Recursive download complete in %v", duration.Round(time.Second))) + fmt.Fprintf(os.Stderr, " Files downloaded: %d\n", result.ChunksCount) + fmt.Fprintf(os.Stderr, " Total size: %s\n", format.Bytes(result.BytesDownloaded)) + fmt.Fprintf(os.Stderr, " Average speed: %s/s\n", format.Speed(result.Speed)) + fmt.Fprintf(os.Stderr, " Output directory: %s\n", result.OutputPath) + } else { + resumeNote := "" + if result.Resumed { + resumeNote = " [resumed]" + } + parallelNote := "" + if result.Parallel { + parallelNote = fmt.Sprintf(" [%d parallel chunks]", result.ChunksCount) + } + extractNote := "" + if *f.Extract && archive.IsArchiveFormat(*f.Output) { + extractNote = " [auto-extracted]" + } + cli.PrintProgressSuccess(os.Stderr, fmt.Sprintf("Downloaded %s in %v (%s)%s%s%s", + format.Bytes(result.BytesDownloaded), + duration.Round(time.Millisecond), + format.Speed(result.Speed), + resumeNote, + parallelNote, + extractNote)) + fmt.Fprintf(os.Stderr, " Protocol: %s | IPv%d | Saved: %s\n", + result.Protocol, result.IPVersion, result.OutputPath) + } + } + + // Write-out (curl-compatible metadata output) + if *f.WriteOut != "" && !*f.Recursive { + handleWriteOut(*f.WriteOut, result, result.BytesDownloaded, duration, result.HTTPStatusCode) + } + + // Trace timing output + if *f.TraceTime && result.TraceTimings != nil { + fmt.Fprintf(os.Stderr, "Timing breakdown: %s\n", result.TraceTimings.String()) + } + + // Save HSTS cache + if hstsCache != nil { + if err := hstsCache.Save(); err != nil && cfg.Verbose { + cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to save hsts cache: %v", err)) + } + } + + return 0 +} + +// splitCommand splits a shell-like command string into tokens respecting double quotes. +// Used to parse --on-complete and --on-complete-url without invoking a shell. +func splitCommand(raw string) []string { + var tokens []string + current := "" + inQuote := false + + for i := 0; i < len(raw); i++ { + c := raw[i] + switch { + case c == '"': + inQuote = !inQuote + case c == ' ' || c == '\t': + if inQuote { + current += string(c) + } else if current != "" { + tokens = append(tokens, current) + current = "" + } + default: + current += string(c) + } + } + if current != "" { + tokens = append(tokens, current) + } + return tokens +} + +// idnaURL converts internationalized domain names to Punycode. +func idnaURL(u *url.URL) *url.URL { + if u == nil || u.Hostname() == "" { + return u + } + host := u.Hostname() + ascii, err := idna.Lookup.ToASCII(host) + if err != nil || ascii == host { + return u + } + // Replace hostname with Punycode version + if u.Port() != "" { + ascii = ascii + ":" + u.Port() + } + result := *u + result.Host = ascii + return &result +} + +// webdavProtocol is a type alias for WebDAV protocol handler. +type webdavProtocol = webdav.Protocol + +// parseStringSlice parses comma-separated string into a slice of strings. +func parseStringSlice(s string) []string { + if s == "" { + return nil + } + var parts []string + for _, p := range strings.Split(s, ",") { + p = strings.TrimSpace(p) + if p != "" { + parts = append(parts, p) + } + } + return parts +} diff --git a/cmd/goget/main_test.go b/cmd/goget/main_test.go new file mode 100644 index 0000000..9d0a898 --- /dev/null +++ b/cmd/goget/main_test.go @@ -0,0 +1,381 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "context" + "flag" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "codeberg.org/petrbalvin/goget/internal/config" +) + +func TestExtractConfigPath(t *testing.T) { + tests := []struct { + name string + args []string + expected string + }{ + {"no config flag", []string{"--url", "https://example.com"}, ""}, + {"--config with value", []string{"--config", "/path/to/config.json"}, "/path/to/config.json"}, + {"--config=value", []string{"--config=/path/to/config.json"}, "/path/to/config.json"}, + {"config after url", []string{"--url", "https://example.com", "--config", "/custom/config.json"}, "/custom/config.json"}, + {"empty args", []string{}, ""}, + {"--config without value", []string{"--config"}, ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := extractConfigPath(tc.args) + if result != tc.expected { + t.Errorf("extractConfigPath(%v) = %q, want %q", tc.args, result, tc.expected) + } + }) + } +} + +func TestParseRateLimit(t *testing.T) { + tests := []struct { + input string + expected int64 + }{ + {"", 0}, + {"0", 0}, + {"100", 100}, + {"1KB/s", 1000}, + {"1MB/s", 1000000}, + {"1GB/s", 1000000000}, + {"500KB/s", 500000}, + {"10MB/s", 10000000}, + } + + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + result, err := parseRateLimit(tc.input) + if err != nil { + t.Errorf("parseRateLimit(%q) returned unexpected error: %v", tc.input, err) + } + if result != tc.expected { + t.Errorf("parseRateLimit(%q) = %d, want %d", tc.input, result, tc.expected) + } + }) + } +} + +// TestParseRateLimitInvalid is a regression guard for the BACKLOG entry +// "`parseRateLimit` silently returns 0 on parse error". The previous +// implementation fell back to 0 without telling the caller, so a typo +// like "1oMB/s" (letter 'o' instead of digit '0') silently disabled rate +// limiting. The fixed implementation returns a non-nil error so the caller +// can warn the user. +func TestParseRateLimitInvalid(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"letter_o_instead_of_zero", "1oMB/s"}, + {"pure_garbage", "abc"}, + {"trailing_unit_only", "MB"}, + {"only_unit_letter", "B"}, + {"whitespace_around_digits", " abc "}, + {"multiple_dots", "1.2.3MB"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := parseRateLimit(tc.input) + if err == nil { + t.Errorf("parseRateLimit(%q) = %d, nil err; want non-nil error", tc.input, result) + } + if result != 0 { + t.Errorf("parseRateLimit(%q) = %d on error, want 0", tc.input, result) + } + }) + } +} + +func TestExtractURLs(t *testing.T) { + tests := []struct { + name string + args []string + expected int + }{ + {"single url", []string{"--url", "https://example.com/file.zip"}, 1}, + {"multiple urls", []string{"--url", "https://a.com", "--url", "https://b.com"}, 2}, + {"with other flags", []string{"--url", "https://a.com", "--verbose", "--url", "https://b.com"}, 2}, + {"url= format", []string{"--url=https://a.com", "--url=https://b.com"}, 2}, + {"no url", []string{"--verbose", "--debug"}, 0}, + {"empty args", []string{}, 0}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := extractURLs(tc.args) + if len(result) != tc.expected { + t.Errorf("extractURLs(%v) returned %d URLs, want %d", tc.args, len(result), tc.expected) + } + }) + } +} + +func TestBatchFileURLParsing(t *testing.T) { + // Test that the URL parsing logic in handleBatchFile would work + // by testing the individual steps + content := `https://example.com/file1.zip +https://example.com/file2.zip +# This is a comment +// This is also a comment + +https://example.com/file3.zip` + + lines := strings.Split(strings.TrimSpace(content), "\n") + var urls []string + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { + continue + } + urls = append(urls, line) + } + + if len(urls) != 3 { + t.Errorf("Expected 3 URLs, got %d: %v", len(urls), urls) + } +} + +// TestConfigSetFilePermissions is a regression guard for the BACKLOG +// entry "os.WriteFile with 0644 on config containing OAuth tokens". +// The previous implementation used 0644, which leaves the config file +// (and any embedded access_token / refresh_token) world-readable on +// multi-user systems. After the fix, both handleConfigSet and +// handleConfigUnset must write the file with mode 0600. +func TestConfigFilePermissions(t *testing.T) { + tests := []struct { + name string + run func(t *testing.T, configPath string) + }{ + { + name: "handleConfigSet writes 0600", + run: func(t *testing.T, configPath string) { + if rc := handleConfigSet(configPath, "timeout", []string{ + "--config-set", "timeout", "30", + }); rc != 0 { + t.Fatalf("handleConfigSet returned %d", rc) + } + }, + }, + { + name: "handleConfigUnset writes 0600", + run: func(t *testing.T, configPath string) { + if rc := handleConfigSet(configPath, "timeout", []string{ + "--config-set", "timeout", "30", + }); rc != 0 { + t.Fatalf("seed handleConfigSet returned %d", rc) + } + if rc := handleConfigUnset(configPath, "timeout"); rc != 0 { + t.Fatalf("handleConfigUnset returned %d", rc) + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.toml") + // Seed a minimal config so config.Load succeeds. + seed := []byte("proxy = \"\"\nuser_agent = \"\"\ntimeout = 0\n") + if err := os.WriteFile(configPath, seed, 0600); err != nil { + t.Fatalf("seed write: %v", err) + } + + tc.run(t, configPath) + + info, err := os.Stat(configPath) + if err != nil { + t.Fatalf("stat: %v", err) + } + perm := info.Mode().Perm() + // Mask out the umask bits by checking against 0600; the file + // must not be group- or world-readable. + if perm != 0600 { + t.Errorf("config file mode = %04o, want 0600", perm) + } + }) + } +} + +func TestNextCronTime(t *testing.T) { + // Use a fixed reference time for deterministic tests (UTC) + ref, _ := time.Parse("2006-01-02 15:04", "2026-06-01 10:00") + ref = ref.UTC() + + t.Run("every minute", func(t *testing.T) { + next, err := nextCronTime("* * * * *", ref) + if err != nil { + t.Fatal(err) + } + expected := ref.Truncate(time.Minute).Add(time.Minute) + if !next.Equal(expected) { + t.Errorf("got %v, want %v", next, expected) + } + }) + + t.Run("specific hour and minute", func(t *testing.T) { + next, err := nextCronTime("30 14 * * *", ref) + if err != nil { + t.Fatal(err) + } + expected := time.Date(2026, 6, 1, 14, 30, 0, 0, time.UTC) + if !next.Equal(expected) { + t.Errorf("got %v, want %v", next, expected) + } + }) + + t.Run("daily at midnight", func(t *testing.T) { + next, err := nextCronTime("0 0 * * *", ref) + if err != nil { + t.Fatal(err) + } + expected := time.Date(2026, 6, 2, 0, 0, 0, 0, time.UTC) + if !next.Equal(expected) { + t.Errorf("got %v, want %v", next, expected) + } + }) + + t.Run("specific day of week", func(t *testing.T) { + // June 1, 2026 is Monday (1) + next, err := nextCronTime("0 9 * * 3", ref) // Wednesday + if err != nil { + t.Fatal(err) + } + if next.Weekday() != time.Wednesday { + t.Errorf("expected Wednesday, got %v", next.Weekday()) + } + }) + + t.Run("comma separated hours", func(t *testing.T) { + next, err := nextCronTime("0 8,16 * * *", ref) + if err != nil { + t.Fatal(err) + } + expected := time.Date(2026, 6, 1, 16, 0, 0, 0, time.UTC) + if !next.Equal(expected) { + t.Errorf("got %v, want %v", next, expected) + } + }) + + t.Run("step values", func(t *testing.T) { + next, err := nextCronTime("*/15 * * * *", ref) + if err != nil { + t.Fatal(err) + } + if next.Minute()%15 != 0 { + t.Errorf("minute not a multiple of 15: %d", next.Minute()) + } + }) + + t.Run("invalid fields", func(t *testing.T) { + _, err := nextCronTime("invalid", ref) + if err == nil { + t.Error("expected error for invalid cron expression") + } + }) + + t.Run("too many fields", func(t *testing.T) { + _, err := nextCronTime("1 2 3 4 5 6", ref) + if err == nil { + t.Error("expected error for 6 fields") + } + }) +} + +// TestExitCodeForRun covers the contract that graceful shutdown +// surfaces exit code 130 to the shell when the user interrupts the +// process. A clean exit (no signal) returns the inner code unchanged; +// a non-zero inner code (real error) is preserved even if the signal +// context was also cancelled. +func TestExitCodeForRun(t *testing.T) { + t.Run("clean exit on no signal returns inner code 0", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if got := exitCodeForRun(ctx, 0); got != 0 { + t.Errorf("got %d, want 0", got) + } + }) + + t.Run("cancelled signal context maps clean exit to 130", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // simulate SIGINT/SIGTERM propagation + if got := exitCodeForRun(ctx, 0); got != 130 { + t.Errorf("got %d, want 130", got) + } + }) + + t.Run("non-zero inner code is preserved when signal is also cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // Even with the signal context cancelled, a real error (e.g. + // download failure) should not be masked by the 130 mapping. + if got := exitCodeForRun(ctx, 7); got != 7 { + t.Errorf("got %d, want 7 (inner code preserved)", got) + } + }) + + t.Run("non-zero inner code is returned unchanged with no signal", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if got := exitCodeForRun(ctx, 42); got != 42 { + t.Errorf("got %d, want 42", got) + } + }) + + t.Run("nil context returns inner code unchanged", func(t *testing.T) { + if got := exitCodeForRun(context.TODO(), 5); got != 5 { + t.Errorf("got %d, want 5 (nil context safe)", got) + } + }) +} + +// TestCreateDirsFlagRegisteredAndDefaults is a regression guard for the +// BACKLOG entry "Add --create-dirs flag (curl / wget compatibility)". It +// verifies the flag is registered on the command-line FlagSet, defaults to +// true (matching curl / wget behaviour so common `goget --output +// ./downloads/file.zip` invocations just work), and can be flipped to +// false via the standard flag parser. We do not exercise the full +// download path here — that is covered by TestNewWriterCreateDirsMissingParent +// in internal/output/writer_test.go. +func TestCreateDirsFlagRegisteredAndDefaults(t *testing.T) { + cfg := config.DefaultConfig() + fs := newTestFlagSet("goget-test") + flags := defineFlags(fs, cfg) + + if flags.CreateDirs == nil { + t.Fatal("CreateDirs flag was not registered on the FlagSet") + } + if !*flags.CreateDirs { + t.Errorf("CreateDirs default = false, want true (curl / wget parity)") + } + + // Flip the flag through the standard flag parser to make sure the + // long name is wired up correctly and the value reaches the struct. + fs2 := newTestFlagSet("goget-test-2") + flags2 := defineFlags(fs2, cfg) + if err := fs2.Parse([]string{"--create-dirs=false"}); err != nil { + t.Fatalf("Parse --create-dirs=false: %v", err) + } + if *flags2.CreateDirs { + t.Errorf("CreateDirs after --create-dirs=false = true, want false") + } +} + +// newTestFlagSet builds an isolated flag.FlagSet for CLI tests. We use +// flag.ContinueOnError so a bad flag in a test fails the test instead of +// calling os.Exit and tearing down the rest of the test binary. +func newTestFlagSet(name string) *flag.FlagSet { + return flag.NewFlagSet(name, flag.ContinueOnError) +} diff --git a/cmd/goget/man_page.go b/cmd/goget/man_page.go new file mode 100644 index 0000000..d579627 --- /dev/null +++ b/cmd/goget/man_page.go @@ -0,0 +1,217 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "fmt" + + "codeberg.org/petrbalvin/goget/internal/core" +) + +func handleGenManPage() int { + fmt.Printf(`.TH GOGET 1 "2026-06" "goget %s" "User Commands" +.SH NAME +goget \- modern IPv6-first command-line download utility +.SH SYNOPSIS +.B goget +\fB\-\-url\fR \fIURL\fR [\fIOPTIONS\fR] +.SH DESCRIPTION +goget is a modern replacement for curl and wget built on Go stdlib. +Features IPv6-first connections, parallel downloads, resume support, +compression, checksum verification, and recursive mirroring. +.SH OPTIONS +.SS "Target" +.TP +\fB\-\-url\fR=\fIURL\fR +URL to download (can be specified multiple times) +.SS "Basic Options" +.TP +\fB\-\-output\fR=\fIFILE\fR +Output filename (default: auto from URL) +.TP +\fB\-\-verbose\fR +Verbose output with progress bar (default: enabled) +.TP +\fB\-\-no-progress\fR +Hide progress bar (keep other output) +.TP +\fB\-\-quiet\fR +Suppress all output except errors +.TP +\fB\-\-debug\fR +Debug mode with detailed info +.TP +\fB\-\-resume\fR +Resume interrupted download +.TP +\fB\-\-restart\fR +Delete existing file and restart download +.TP +\fB\-\-overwrite\fR +Overwrite existing file +.TP +\fB\-\-no-decompress\fR +Disable automatic decompression +.TP +\fB\-\-parallel\fR=\fIN\fR +Parallel connections (0=auto, 1=sequential) +.TP +\fB\-\-json\fR +Output progress as JSON (for scripting) +.TP +\fB\-\-info\fR +Show file information without downloading +.TP +\fB\-\-benchmark\fR=\fIURL\fR +Benchmark download speed +.SS "Network" +.TP +\fB\-\-timeout\fR=\fIDURATION\fR +Connection timeout (0 = auto based on size) +.TP +\fB\-\-proxy\fR=\fIURL\fR +Proxy server URL (http://, https://, socks5://, socks5h://) +.TP +\fB\-\-dns-servers\fR=\fIIPs\fR +Custom DNS servers (comma-separated) +.TP +\fB\-\-max-retries\fR=\fIN\fR +Maximum number of retries on failure +.TP +\fB\-\-no-ipv4\fR +Disable IPv4 fallback +.TP +\fB\-\-no-redirect\fR +Disable following HTTP redirects +.TP +\fB\-\-show-headers\fR +Print HTTP response headers +.TP +\fB\-\-keep-timestamps\fR +Set file modification time from Last-Modified header +.TP +\fB\-\-content-disposition\fR +Use server-provided filename from Content-Disposition header +.SS "Authentication" +.TP +Credentials are loaded automatically from \fB~/.netrc\fR when no \fB\-\-username\fR is given. +.TP +\fB\-\-username\fR=\fIUSER\fR +Username for HTTP Basic/Digest Auth +.TP +\fB\-\-password\fR=\fIPASS\fR +Password for HTTP Basic/Digest Auth +.TP +\fB\-\-password-file\fR=\fIFILE\fR +Read password from file +.TP +\fB\-\-auth-type\fR=\fITYPE\fR +Auth type: basic, digest, auto +.TP +\fB\-\-cookie-jar\fR=\fIFILE\fR +File for loading/saving cookies +.TP +\fB\-\-cookie\fR=\fINAME=VALUE\fR +Set cookie from CLI (repeatable, curl-compatible) +.TP +\fB\-\-retry-all-errors\fR +Retry on any HTTP 4xx/5xx response +.TP +\fB\-\-ssl-key-log\fR=\fIFILE\fR +Log TLS session keys to file (SSLKEYLOGFILE for Wireshark) +.SS "Config" +.TP +\fB\-\-config\fR=\fIFILE\fR +Path to config file +.TP +\fB\-\-config-get\fR=\fIKEY\fR +Get a config value +.TP +\fB\-\-config-set\fR=\fIKEY\fR \fIVALUE\fR +Set a config value +.TP +\fB\-\-config-list\fR +List all config values +.TP +\fB\-\-config-unset\fR=\fIKEY\fR +Unset a config value +.TP +\fB\-\-init-config\fR +Generate default config file +.SS "Mirror & Recursive" +.TP +\fB\-\-mirror\fR +Mirror entire website (infinite depth) +.TP +\fB\-\-recursive\fR +Enable recursive downloading +.TP +\fB\-\-max-depth\fR=\fIN\fR +Maximum recursion depth +.SS "Scheduling & Hooks" +.TP +\fB\-\-schedule\fR=\fITIME\fR +Schedule download ("YYYY-MM-DD HH:MM" or cron "0 2 * * *") +.TP +\fB\-\-on-complete\fR=\fICMD\fR +Run shell command after successful download +.TP +\fB\-\-bind-interface\fR=\fIIFACE\fR +Bind to specific network interface +.SS "Other" +.TP +\fB\-\-batch-file\fR=\fIFILE\fR +Download multiple URLs from a file +.TP +\fB\-\-checksum-file\fR=\fIFILE\fR +Read expected checksum from SHA256SUMS file +.TP +\fB\-\-generate-man-page\fR +Generate this man page +.TP +\fB\-\-help\fR +Show help +.TP +\fB\-\-version\fR +Show version +.SH EXAMPLES +.TP +Download a file: +goget --url https://example.com/file.zip +.TP +Parallel download: +goget --url https://example.com/large.iso --parallel 4 +.TP +Resume interrupted download: +goget --url https://example.com/file.zip --resume +.TP +Mirror website: +goget --url https://example.com --mirror --output ./mirror +.TP +Recursive download: +goget --url https://example.com --recursive --max-depth 2 +.TP +JSON progress (for scripting): +goget --url https://example.com/file.zip --json --no-progress > progress.jsonl +.TP +Scheduled download at 2 AM: +goget --url https://example.com/daily.zip --schedule "0 2 * * *" +.TP +Run import script after download: +goget --url https://example.com/data.csv --on-complete "import.sh" +.SH FILES +.TP +~/.config/goget/config.json +Configuration file +.TP +.goget.meta +Resume metadata file +.SH BUGS +Report bugs at https://codeberg.org/petrbalvin/goget/issues +.SH AUTHOR +Petr Balvin +.SH LICENSE +MIT License`, core.Version) + return 0 +} diff --git a/cmd/goget/schedule.go b/cmd/goget/schedule.go new file mode 100644 index 0000000..adad89d --- /dev/null +++ b/cmd/goget/schedule.go @@ -0,0 +1,145 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" + + "codeberg.org/petrbalvin/goget/internal/cli" + "codeberg.org/petrbalvin/goget/internal/config" +) + +// waitForSchedule blocks until the scheduled time or returns an error code. +func waitForSchedule(schedule string, cfg *config.Config, sigChan <-chan os.Signal) int { + var scheduledTime time.Time + var isCron bool + var cronSchedule string + + parsedTime, parseErr := time.Parse("2006-01-02 15:04", schedule) + if parseErr == nil { + scheduledTime = parsedTime + } else { + fields := strings.Fields(schedule) + if len(fields) == 5 { + isCron = true + cronSchedule = schedule + next, cerr := nextCronTime(cronSchedule, time.Now()) + if cerr != nil { + cli.PrintError(os.Stderr, fmt.Errorf("invalid schedule: %w (use 'YYYY-MM-DD HH:MM' or cron 'MIN HOUR DOM MON DOW')", cerr)) + return 1 + } + scheduledTime = next + } else { + cli.PrintError(os.Stderr, fmt.Errorf("invalid schedule format: %w (use 'YYYY-MM-DD HH:MM' or cron 'MIN HOUR DOM MON DOW')", parseErr)) + return 1 + } + } + + delay := time.Until(scheduledTime) + if delay <= 0 { + cli.PrintError(os.Stderr, fmt.Errorf("scheduled time is in the past")) + return 1 + } + if cfg.Verbose { + label := "Scheduled" + if isCron { + label = fmt.Sprintf("Cron (%s) scheduled", cronSchedule) + } + cli.PrintInfo(os.Stderr, fmt.Sprintf("%s download in %v (at %s)", label, delay.Round(time.Second), scheduledTime.Format(time.RFC3339))) + } + select { + case <-time.After(delay): + case <-sigChan: + return 1 + } + return 0 +} + +// nextCronTime calculates the next scheduled time from a cron expression. +func nextCronTime(expr string, from time.Time) (time.Time, error) { + fields := strings.Fields(expr) + if len(fields) != 5 { + return time.Time{}, fmt.Errorf("cron expression must have 5 fields, got %d", len(fields)) + } + + minute := fields[0] + hour := fields[1] + dom := fields[2] + month := fields[3] + dow := fields[4] + + t := from.Truncate(time.Minute).Add(time.Minute) + deadline := from.AddDate(2, 0, 0) + for t.Before(deadline) { + if matchCronField(minute, t.Minute(), 0, 59) && + matchCronField(hour, t.Hour(), 0, 23) && + matchCronField(dom, t.Day(), 1, 31) && + matchCronField(month, int(t.Month()), 1, 12) && + matchCronField(dow, int(t.Weekday()), 0, 6) { + return t, nil + } + t = t.Add(time.Minute) + } + + return time.Time{}, fmt.Errorf("no matching time found within 2 years") +} + +func matchCronField(pattern string, value, min, max int) bool { + if pattern == "*" { + return true + } + + for _, part := range strings.Split(pattern, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + if strings.Contains(part, "/") { + parts := strings.SplitN(part, "/", 2) + base := parts[0] + step, err := strconv.Atoi(parts[1]) + if err != nil || step <= 0 { + continue + } + if base == "*" { + if (value-min)%step == 0 { + return true + } + continue + } + start, err := strconv.Atoi(base) + if err != nil { + continue + } + if value >= start && (value-start)%step == 0 { + return true + } + continue + } + + if strings.Contains(part, "-") { + parts := strings.SplitN(part, "-", 2) + start, err1 := strconv.Atoi(parts[0]) + end, err2 := strconv.Atoi(parts[1]) + if err1 != nil || err2 != nil { + continue + } + if value >= start && value <= end { + return true + } + continue + } + + if v, err := strconv.Atoi(part); err == nil && v == value { + return true + } + } + + return false +} diff --git a/cmd/goget/setup.go b/cmd/goget/setup.go new file mode 100644 index 0000000..60ce23b --- /dev/null +++ b/cmd/goget/setup.go @@ -0,0 +1,262 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "fmt" + stdhttp "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "codeberg.org/petrbalvin/goget/internal/auth" + "codeberg.org/petrbalvin/goget/internal/cli" + "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/cookie" + "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/protocol/http" + "codeberg.org/petrbalvin/goget/internal/transport" +) + +// ApplyFlags populates the Config from CLI flags, global config, and cookie jar. +// It is the single function that maps all CLI flags to config fields — adding a +// new flag only requires adding one block here instead of hunting through +// scattered setup code. +func ApplyFlags(c *http.Config, f *Flags, cfg *config.Config, cookieJar *cookie.Jar, parsedURL *url.URL) { + // ── Transport defaults from global config ── + c.Transport.Timeout = cfg.GetEffectiveTimeout(-1, *f.Timeout) + c.Transport.Dialer.IPv4Fallback = cfg.IPv4Fallback + c.Transport.Dialer.Debug = cfg.Debug + c.UserAgent = cfg.UserAgent + c.Output.Verbose = cfg.Verbose + c.Parallel = &core.ParallelConfig{ + Connections: cfg.GetParallelConnections(-1), + MinSize: 100 * 1024 * 1024, + MinChunkSize: 5 * 1024 * 1024, + MaxChunkSize: 50 * 1024 * 1024, + } + if cfg.MaxSpeed > 0 { + c.RateLimiter = transport.NewTokenBucket(cfg.MaxSpeed, cfg.MaxSpeed/10) + } + + // ── DNS / interface ── + if len(cfg.DNSServers) > 0 { + c.Transport.DNSServers = cfg.DNSServers + } + if *f.BindInterface != "" { + c.Transport.BindInterface = *f.BindInterface + } + c.Transport.BlockPrivateIPs = !*f.AllowPrivate + + // ── Authentication ── + if cfg.Auth.Username == "" && cfg.Auth.OAuth.AccessToken == "" { + if user, pass, ok := auth.LookupNetrc(parsedURL.Hostname()); ok { + cfg.Auth.Username = user + cfg.Auth.Password = pass + } + } + if cfg.Auth.Username != "" || cfg.Auth.OAuth.AccessToken != "" { + c.Auth.Username = cfg.Auth.Username + c.Auth.Password = cfg.Auth.Password + c.Auth.AuthType = cfg.Auth.AuthType + c.Auth.OAuth = &cfg.Auth.OAuth + } + if cookieJar != nil { + c.CookieJar = cookieJar + } + + // ── TLS and certificates ── + if *f.Insecure && c.Transport.TLS != nil { + c.Transport.TLS.InsecureSkipVerify = true + } + if *f.CertFile != "" && c.Transport.TLS != nil { + c.Transport.TLS.CertFile = *f.CertFile + } + if *f.KeyFile != "" && c.Transport.TLS != nil { + c.Transport.TLS.KeyFile = *f.KeyFile + } + if *f.PinnedCert != "" && c.Transport.TLS != nil { + c.Transport.TLS.PinnedCertHash = *f.PinnedCert + } + if *f.CACertificate != "" && c.Transport.TLS != nil { + c.Transport.TLS.CACertFile = *f.CACertificate + } + if *f.SSLKeyLogFile != "" && c.Transport.TLS != nil { + c.Transport.TLS.KeyLogFile = *f.SSLKeyLogFile + } + if *f.ConnectTimeout > 0 && c.Transport.Dialer != nil { + c.Transport.Dialer.Timeout = *f.ConnectTimeout + } + + // ── Retry and redirects ── + if *f.NoRedirect { + c.Transport.FollowRedirects = false + } + if *f.MaxRetries > 0 { + c.Transport.MaxRetries = *f.MaxRetries + if c.Transport.Retry != nil { + c.Transport.Retry.MaxRetries = *f.MaxRetries + } + } + if *f.RetryDelay > 0 { + c.Transport.RetryDelay = *f.RetryDelay + } + if *f.RetryHTTPError != "" { + for _, code := range strings.Split(*f.RetryHTTPError, ",") { + if code, err := strconv.Atoi(strings.TrimSpace(code)); err == nil { + c.Transport.RetryHTTPCodes = append(c.Transport.RetryHTTPCodes, code) + } + } + } + if *f.RetryAllErrors { + c.Transport.RetryAllErrors = true + } + + // ── Timeouts and limits ── + if *f.ConnectTimeout > 0 { + c.Transport.ConnectTimeout = *f.ConnectTimeout + } + if *f.MaxTime > 0 { + c.Transport.MaxTime = *f.MaxTime + } + if *f.MaxFileSize != "" { + size, perr := parseRateLimit(*f.MaxFileSize) + if perr != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("ignoring --max-filesize: %v; size limit disabled", perr)) + } + c.Transport.MaxFileSize = size + } + + // ── Output flags ── + if *f.ContentDisp { + c.Output.ContentDisposition = true + } + if *f.ShowHeaders { + c.Output.ShowHeaders = true + } + if *f.KeepTimestamps { + c.Output.KeepTimestamps = true + } + if *f.JSON { + c.Output.JSONOutput = true + } + if *f.OnComplete != "" { + c.Output.OnComplete = *f.OnComplete + } + if *f.WarcFile != "" { + c.Output.WarcFile = *f.WarcFile + } + if *f.TraceTime { + c.Output.TraceTime = true + } + if *f.WriteOut != "" { + c.Output.WriteOut = *f.WriteOut + } + + // ── Request flags ── + if len(f.Headers) > 0 { + if c.CustomHeaders == nil { + c.CustomHeaders = make(map[string]string) + } + for _, h := range f.Headers { + parts := strings.SplitN(h, ":", 2) + key := strings.TrimSpace(parts[0]) + val := "" + if len(parts) > 1 { + val = strings.TrimSpace(parts[1]) + } + if strings.ContainsAny(key, "\r\n") || strings.ContainsAny(val, "\r\n") { + cli.PrintWarning(os.Stderr, fmt.Sprintf("Skipping header with CRLF: %s", key)) + continue + } + c.CustomHeaders[key] = val + } + } + if *f.Data != "" { + c.PostData = *f.Data + } + if *f.PostFile != "" { + c.PostFile = *f.PostFile + } + if *f.Range != "" { + c.Range = *f.Range + } + if *f.Compressed { + c.Compressed = true + } + if *f.Referer != "" { + c.Referer = *f.Referer + } + if *f.Fail { + c.FailOnHTTPError = true + } + if *f.NoClobber { + c.Transport.NoClobber = true + } + if len(f.Form) > 0 { + c.FormData = f.Form + } + + // ── Recursive flags ── + if *f.RecursiveParallel > 0 { + c.Recursive.Parallel = *f.RecursiveParallel + } + if *f.Wait != "" { + if d, err := time.ParseDuration(*f.Wait); err == nil { + c.Recursive.Wait = d + } + } + if *f.RandomWait { + c.Recursive.RandomWait = true + } + if *f.ConvertLinks { + c.Recursive.ConvertLinks = true + } + if *f.Accept != "" { + c.Recursive.Accept = *f.Accept + } + if *f.Domains != "" { + c.Recursive.Domains = strings.Split(*f.Domains, ",") + } + if *f.Reject != "" { + c.Recursive.Reject = strings.Split(*f.Reject, ",") + } + if *f.CutDirs > 0 { + c.Recursive.CutDirs = *f.CutDirs + } + if *f.PageRequisites { + c.Recursive.PageRequisites = true + } + if *f.SpanHosts { + c.Recursive.SpanHosts = true + } + if *f.AdjustExt { + c.Recursive.AdjustExt = true + } + + // ── Cookies from CLI flags ── + if len(f.Cookies) > 0 { + if cookieJar == nil { + var err error + cookieJar, err = cookie.NewJar() + if err != nil { + cli.PrintWarning(os.Stderr, fmt.Sprintf("failed to create cookie jar: %v", err)) + } + } + if cookieJar != nil { + for _, cstr := range f.Cookies { + parts := strings.SplitN(cstr, "=", 2) + if len(parts) < 2 { + cli.PrintWarning(os.Stderr, fmt.Sprintf("invalid cookie format (use name=value): %s", cstr)) + continue + } + ck := &stdhttp.Cookie{Name: parts[0], Value: parts[1]} + cookieJar.SetCookies(parsedURL, []*stdhttp.Cookie{ck}) + } + c.CookieJar = cookieJar + } + } +} diff --git a/cmd/goget/verify.go b/cmd/goget/verify.go new file mode 100644 index 0000000..dd86117 --- /dev/null +++ b/cmd/goget/verify.go @@ -0,0 +1,200 @@ +//go:build linux || freebsd +// +build linux freebsd + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "codeberg.org/petrbalvin/goget/internal/crypto" +) + +// parseChecksumFile reads a checksum file (SHA256SUMS format) and finds +// the expected checksum for the given filename. +func parseChecksumFile(checksumFile, outputFile string) (string, error) { + data, err := os.ReadFile(checksumFile) + if err != nil { + return "", fmt.Errorf("failed to read checksum file: %w", err) + } + + targetName := filepath.Base(outputFile) + lines := strings.Split(string(data), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + // Format: or * + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + checksum := parts[0] + filename := strings.TrimPrefix(parts[1], "*") + if filename == targetName { + return checksum, nil + } + // Also check if the output file matches + if filename == outputFile { + return checksum, nil + } + } + return "", fmt.Errorf("no checksum found for %s in %s", targetName, checksumFile) +} + +// verifyChecksum verifies the checksum of a downloaded file. +// Returns true if the checksum matches, false otherwise. +// Returns an error if verification could not be performed. +func verifyChecksum(outputFile, checksumFlag, checksumAlgo string, verbose bool) (bool, error) { + if verbose { + fmt.Fprintf(os.Stderr, "\nVerifying checksum...\n") + } + algo := crypto.SHA256 + switch strings.ToLower(checksumAlgo) { + case "sha256": + algo = crypto.SHA256 + case "sha512": + algo = crypto.SHA512 + case "blake2b": + algo = crypto.BLAKE2b + case "sha3-256", "sha3_256": + algo = crypto.SHA3_256 + case "sha3-512", "sha3_512": + algo = crypto.SHA3_512 + case "md5": + algo = crypto.MD5 + default: + return false, fmt.Errorf("unsupported checksum algorithm: %s", checksumAlgo) + } + verifier, verr := crypto.NewChecksumVerifier(algo, checksumFlag) + if verr != nil { + return false, verr + } + match, actual, verr := verifier.VerifyFile(outputFile) + if verr != nil { + return false, verr + } + if !match { + return false, fmt.Errorf("checksum mismatch\n Expected: %s\n Actual: %s", checksumFlag, actual) + } + if verbose { + fmt.Fprintf(os.Stderr, "Checksum OK: %s\n", actual) + } + return true, nil +} + +// verifyPGP handles PGP signature verification and PGP decryption. +// Returns true on success, false on failure, and any error encountered. +func verifyPGP(outputFile string, pgpVerify, pgpDecrypt bool, pgpKeyFlag, pgpPassphraseFlag, pgpSignatureFlag *string, verbose bool) (bool, error) { + // PGP signature verification + if pgpVerify && outputFile != "" { + if verbose { + fmt.Fprintf(os.Stderr, "\nVerifying PGP signature...\n") + } + + pgpCfg := &crypto.PGPConfig{ + PublicKeyPath: *pgpKeyFlag, + Passphrase: *pgpPassphraseFlag, + } + + verifier := crypto.NewPGPVerifier(pgpCfg) + + var valid bool + var identity string + var err error + + if *pgpSignatureFlag != "" { + // Detached signature + valid, identity, err = verifier.VerifyFile(outputFile, *pgpSignatureFlag) + } else { + // Try to find signature file automatically + sigFiles := []string{ + outputFile + ".asc", + outputFile + ".sig", + outputFile + ".gpg", + } + for _, sigFile := range sigFiles { + if _, statErr := os.Stat(sigFile); statErr == nil { + valid, identity, err = verifier.VerifyFile(outputFile, sigFile) + if valid { + break + } + } + } + if !valid { + err = fmt.Errorf("signature file not found or invalid (tried: %s)", strings.Join(sigFiles, ", ")) + } + } + + if err != nil { + return false, fmt.Errorf("pgp verification failed: %w", err) + } + + if !valid { + return false, fmt.Errorf("pgp signature verification failed") + } + + if verbose { + fmt.Fprintf(os.Stderr, "PGP signature verified OK (signed by: %s)\n", identity) + } + } + + // PGP decryption + if pgpDecrypt && outputFile != "" { + if verbose { + fmt.Fprintf(os.Stderr, "\nDecrypting PGP file...\n") + } + + // Read file to check if it's PGP encrypted + data, err := os.ReadFile(outputFile) + if err != nil { + return false, fmt.Errorf("failed to read file for pgp check: %w", err) + } + + if !crypto.IsPGPEncrypted(data) { + if verbose { + fmt.Fprintf(os.Stderr, "File is not PGP encrypted, skipping decryption\n") + } + return true, nil + } + + pgpCfg := &crypto.PGPConfig{ + PrivateKeyPath: *pgpKeyFlag, + Passphrase: *pgpPassphraseFlag, + } + + decryptor := crypto.NewPGPDecryptor(pgpCfg) + + // Determine output path + decryptedPath := outputFile + if strings.HasSuffix(outputFile, ".gpg") { + decryptedPath = strings.TrimSuffix(outputFile, ".gpg") + } else if strings.HasSuffix(outputFile, ".pgp") { + decryptedPath = strings.TrimSuffix(outputFile, ".pgp") + } else if strings.HasSuffix(outputFile, ".asc") { + decryptedPath = strings.TrimSuffix(outputFile, ".asc") + } else { + decryptedPath = outputFile + ".decrypted" + } + + if err := decryptor.DecryptFile(outputFile, decryptedPath); err != nil { + return false, fmt.Errorf("pgp decryption failed: %w", err) + } + + if verbose { + fmt.Fprintf(os.Stderr, "PGP decryption successful: %s\n", decryptedPath) + } + + // Remove encrypted file + if err := os.Remove(outputFile); err != nil { + if verbose { + fmt.Fprintf(os.Stderr, "Warning: failed to remove encrypted file: %v\n", err) + } + } + } + + return true, nil +} diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..28aee95 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,454 @@ +# API Reference + +**goget** exposes a public Go library in `pkg/api/` for programmatic downloads, uploads, queue management, and configuration. + +## Installation + +```go +import "codeberg.org/petrbalvin/goget/pkg/api" +``` + +## Types + +### `Client` + +The main client for downloads and uploads. Created via `NewClient()`. + +```go +type Client struct { /* unexported */ } + +func NewClient(opts ...ClientOption) *Client +func (c *Client) Download(req *DownloadRequest) (*DownloadResult, error) +func (c *Client) Upload(req *UploadRequest) (*UploadResult, error) +func (c *Client) Close() +``` + +### `ClientOption` + +Functional options for client configuration. + +```go +func WithTimeout(d time.Duration) ClientOption +func WithUserAgent(ua string) ClientOption +func WithVerbose(v bool) ClientOption +``` + +### `DownloadRequest` + +| Field | Type | Description | +|---|---|---| +| `URL` | `string` | URL to download (required) | +| `Output` | `string` | Output file path (empty = auto-detect) | +| `Resume` | `bool` | Resume interrupted download | +| `Verbose` | `bool` | Enable verbose logging | + +### `DownloadResult` + +| Field | Type | Description | +|---|---|---| +| `BytesDownloaded` | `int64` | Total bytes transferred | +| `TotalSize` | `int64` | Total file size (-1 if unknown) | +| `Speed` | `float64` | Average speed in bytes/second | +| `Duration` | `time.Duration` | Total transfer time | +| `OutputPath` | `string` | Path to saved file | +| `Protocol` | `string` | Protocol used (e.g., "http") | +| `IPVersion` | `int` | 4 or 6 | +| `Resumed` | `bool` | True if download was resumed | +| `Parallel` | `bool` | True if parallel chunks were used | +| `ChunksCount` | `int` | Number of parallel chunks | + +### `UploadRequest` + +| Field | Type | Description | +|---|---|---| +| `URL` | `string` | Destination URL (required) | +| `FilePath` | `string` | Path to file to upload (required) | +| `Method` | `string` | HTTP method: `"PUT"` or `"POST"` | +| `Verbose` | `bool` | Enable verbose logging | + +### `UploadResult` + +| Field | Type | Description | +|---|---|---| +| `BytesUploaded` | `int64` | Total bytes uploaded | +| `Duration` | `time.Duration` | Total upload time | +| `Protocol` | `string` | Protocol used | +| `ResultURL` | `string` | Result URL (may differ from input after redirects) | + +--- + +## Client + +### `NewClient` + +Creates a new goget client. Registers HTTP and FTP protocol handlers automatically. + +```go +func NewClient(opts ...ClientOption) *Client +``` + +**Options:** + +| Option | Effect | +|---|---| +| `WithTimeout(5 * time.Minute)` | Set request timeout (default: 30 minutes) | +| `WithUserAgent("MyApp/1.0")` | Custom User-Agent header | +| `WithVerbose(true)` | Enable verbose logging | + +**Example:** +```go +client := api.NewClient( + api.WithTimeout(10 * time.Minute), + api.WithUserAgent("MyDownloader/1.0"), +) +defer client.Close() +``` + +### `Client.Download` + +Downloads a file from a URL. + +```go +func (c *Client) Download(req *DownloadRequest) (*DownloadResult, error) +``` + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `req.URL` | `string` | URL to download — supports http, https, ftp, sftp, file, data, gopher, gemini | +| `req.Output` | `string` | Output path. Empty string uses the filename derived from the URL. | +| `req.Resume` | `bool` | Resume an interrupted download if resume metadata exists | + +**Returns:** `*DownloadResult` — bytes transferred, speed, duration, protocol info. + +**Throws:** +- `error` — if URL is empty or invalid +- `error` — if protocol is unsupported +- `*core.GogetError` — typed errors for network, protocol, timeout, auth, checksum failures + +**Example:** +```go +result, err := client.Download(&api.DownloadRequest{ + URL: "https://example.com/file.zip", + Output: "downloads/file.zip", + Resume: true, +}) +if err != nil { + log.Printf("Download failed: %v", err) + return +} +log.Printf("Downloaded %d bytes at %.1f KB/s", result.BytesDownloaded, result.Speed/1024) +``` + +### `Client.Upload` + +Uploads a file to a URL. The protocol handler must implement `core.Uploader`. + +```go +func (c *Client) Upload(req *UploadRequest) (*UploadResult, error) +``` + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `req.URL` | `string` | Destination URL (required) | +| `req.FilePath` | `string` | Path to local file (required) | +| `req.Method` | `string` | `"PUT"` or `"POST"` (default: `"PUT"`) | +| `req.Verbose` | `bool` | Enable verbose logging | + +**Returns:** `*UploadResult` — bytes uploaded, duration, protocol, result URL. + +**Throws:** +- `error` — if URL or file path is empty +- `error` — if protocol does not support upload + +**Example:** +```go +result, err := client.Upload(&api.UploadRequest{ + URL: "https://httpbin.org/put", + FilePath: "./data.csv", + Method: "PUT", +}) +if err != nil { + log.Printf("Upload failed: %v", err) + return +} +log.Printf("Uploaded %d bytes", result.BytesUploaded) +``` + +### `Client.Close` + +Releases resources and cancels any pending operations. + +```go +func (c *Client) Close() +``` + +--- + +## Convenience Functions + +### `Download` + +One-shot download. Creates a client, downloads, and closes. + +```go +func Download(urlStr, output string) (*DownloadResult, error) +``` + +**Example:** +```go +result, err := api.Download("https://example.com/file.zip", "output.zip") +``` + +### `Upload` + +One-shot upload. Creates a client, uploads, and closes. + +```go +func Upload(urlStr, filePath, method string) (*UploadResult, error) +``` + +**Example:** +```go +result, err := api.Upload("https://httpbin.org/put", "./data.csv", "PUT") +``` + +--- + +## Queue API + +### `QueueClient` + +Manages a download queue persisted to a JSON file. + +```go +type QueueClient struct { /* unexported */ } + +func NewQueueClient(queueFile string) (*QueueClient, error) +func (qc *QueueClient) Add(urlStr, output string, priority int) string +func (qc *QueueClient) List() []*queue.QueueItem +func (qc *QueueClient) Stats() queue.QueueStats +func (qc *QueueClient) Remove(id string) bool +func (qc *QueueClient) ClearCompleted() int +``` + +### `NewQueueClient` + +```go +func NewQueueClient(queueFile string) (*QueueClient, error) +``` + +Creates a queue client backed by a JSON file. + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `queueFile` | `string` | Path to the queue JSON file | + +**Returns:** `*QueueClient`, `error` — error if the file cannot be read or created. + +### `QueueClient.Add` + +```go +func (qc *QueueClient) Add(urlStr, output string, priority int) string +``` + +Adds a URL to the queue. + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `urlStr` | `string` | URL to download | +| `output` | `string` | Output file path | +| `priority` | `int` | Priority (1–10, higher = processed first) | + +**Returns:** `string` — unique ID for the queued item. + +### `QueueClient.List` + +```go +func (qc *QueueClient) List() []*queue.QueueItem +``` + +Returns all items in the queue. + +**Returns:** `[]*queue.QueueItem` + +### `QueueClient.Stats` + +```go +func (qc *QueueClient) Stats() queue.QueueStats +``` + +Returns queue statistics (total, pending, completed, failed counts). + +### `QueueClient.Remove` + +```go +func (qc *QueueClient) Remove(id string) bool +``` + +Removes an item from the queue by ID. + +**Returns:** `bool` — true if the item was found and removed. + +### `QueueClient.ClearCompleted` + +```go +func (qc *QueueClient) ClearCompleted() int +``` + +Removes all completed items from the queue. + +**Returns:** `int` — number of removed items. + +--- + +## Metalink API + +### `ParseMetalink` + +Parses a local Metalink file. + +```go +func ParseMetalink(path string) (*metalink.Metalink, error) +``` + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `path` | `string` | Path to a local `.meta4` or `.metalink` file | + +**Returns:** `*metalink.Metalink` — parsed metalink with sources, checksums, and metadata. + +### `FetchMetalink` + +Downloads and parses a Metalink from a URL. + +```go +func FetchMetalink(urlStr string) (*metalink.Metalink, error) +``` + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `urlStr` | `string` | URL pointing to a `.meta4` or `.metalink` file | + +**Returns:** `*metalink.Metalink` — parsed metalink with sources, checksums, and metadata. + +--- + +## Config API + +### `LoadConfig` + +```go +func LoadConfig(path string) (*config.Config, error) +``` + +Loads the TOML configuration file. Returns default config if the file does not exist. + +**Parameters:** + +| Parameter | Type | Description | +|---|---|---| +| `path` | `string` | Path to config file (empty = default location) | + +**Returns:** `*config.Config`, `error` + +### `DefaultConfig` + +```go +func DefaultConfig() *config.Config +``` + +Returns the default configuration values. + +### `ParseSpeed` + +```go +func ParseSpeed(s string) int64 +``` + +Parses a speed string like `"10MB/s"` into bytes per second. + +| Input | Output | +|---|---| +| `"1MB/s"` | `1000000` | +| `"500KB/s"` | `500000` | +| `"1GB/s"` | `1000000000` | + +--- + +## Version + +### `Version` + +```go +func Version() string +``` + +Returns the current goget version. + +--- + +## Full Example + +```go +package main + +import ( + "fmt" + "log" + "time" + + "codeberg.org/petrbalvin/goget/pkg/api" +) + +func main() { + // Create client with custom timeout + client := api.NewClient( + api.WithTimeout(5 * time.Minute), + api.WithVerbose(true), + ) + defer client.Close() + + // Download + result, err := client.Download(&api.DownloadRequest{ + URL: "https://example.com/large-file.iso", + Output: "./downloads/large-file.iso", + Resume: true, + }) + if err != nil { + log.Fatalf("download failed: %v", err) + } + + fmt.Printf("Downloaded %d bytes in %v\n", result.BytesDownloaded, result.Duration) + fmt.Printf("Average speed: %.1f MB/s\n", result.Speed/1024/1024) + fmt.Printf("Protocol: %s (IPv%d)\n", result.Protocol, result.IPVersion) + + if result.Resumed { + fmt.Println("Download was resumed from previous state") + } + if result.Parallel { + fmt.Printf("Used %d parallel chunks\n", result.ChunksCount) + } + + // Queue a download + qc, err := api.NewQueueClient("./queue.json") + if err != nil { + log.Fatalf("queue init failed: %v", err) + } + + id := qc.Add("https://example.com/another.zip", "./downloads/", 5) + fmt.Printf("Queued item ID: %s\n", id) + fmt.Printf("Queue stats: %+v\n", qc.Stats()) +} +``` diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..5ce67ef --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,222 @@ +# Architecture + +**goget** is a command-line download utility built on the Go standard library. It follows a layered architecture with a protocol abstraction layer, a transport layer, and utility modules for authentication, compression, output, and more. + +## High-Level Overview + +```mermaid +graph TD + User[User / Terminal]:::accent6 + Script[Script / CI]:::accent6 + CLI[cmd/goget/ - Main entry]:::accent0 + API[pkg/api/ - Library interface]:::accent3 + Core[internal/core/ - Types, Protocol Interface, Errors]:::accent1 + Registry[internal/protocol/ - Protocol Registry]:::accent1 + HTTP[internal/protocol/http/]:::accent1 + FTP[internal/protocol/ftp/]:::accent1 + SFTP[internal/protocol/sftp/]:::accent1 + FILE[internal/protocol/file/]:::accent1 + DataURL[internal/protocol/dataurl/]:::accent1 + Gopher[internal/protocol/gopher/]:::accent1 + Gemini[internal/protocol/gemini/]:::accent1 + WebDAV[internal/protocol/webdav/]:::accent1 + Transport[internal/transport/ - Dialer, Proxy, TLS, Rate Limit]:::accent7 + Config[internal/config/]:::accent7 + Output[internal/output/ - Atomic write, Resume, WARC]:::accent2 + Auth[internal/auth/ - Basic, Digest, OAuth, netrc]:::accent7 + Crypto[internal/crypto/ - Checksum, PGP, Certs]:::accent7 + Cookie[internal/cookie/ - Netscape jar]:::accent7 + HSTS[internal/hsts/ - RFC 6797 cache]:::accent7 + Archive[internal/archive/ - Extraction]:::accent7 + Recursive[internal/recursive/ - Crawler, Parser, Filter]:::accent2 + Mirror[internal/mirror/]:::accent2 + Metalink[internal/metalink/]:::accent2 + Queue[internal/queue/]:::accent2 + Compression[internal/compression/ - gzip, bzip2, zlib, flate, lzw]:::accent7 + Log[internal/log/ - Structured logging]:::accent7 + WARC[internal/warc/ - Archiving]:::accent7 + Register[internal/protocol/register/ - Single registration]:::accent7 + Format[internal/format/ - Bytes, Speed formatting]:::accent7 + LinkRewrite[internal/linkrewrite/ - HTML link conversion]:::accent7 + CLIUtil[internal/cli/ - Help, Progress bar, Colors]:::accent7 + + User --> CLI + Script --> API + CLI --> Log + CLI --> CLIUtil + CLI --> Config + CLI --> Core + CLI -.-> API + API --> Core + API --> Registry + + subgraph "Download Pipeline" + Core --> Registry + Registry --> HTTP + Registry --> FTP + Registry --> SFTP + Registry --> FILE + Registry --> DataURL + Registry --> Gopher + Registry --> Gemini + Registry --> WebDAV + HTTP --> Transport + HTTP --> Cookie + HTTP --> HSTS + HTTP --> Output + FTP --> Transport + SFTP --> Transport + end + + subgraph "Post-Download" + CLI --> Crypto + CLI --> Archive + CLI --> Compression + end + + subgraph "Download Management" + CLI --> Register + CLI --> WARC + CLI --> Recursive + CLI --> Mirror + CLI --> Metalink + CLI --> Queue + Recursive --> WARC + Recursive --> LinkRewrite + Mirror --> LinkRewrite + end + + Transport --> Auth + Transport --> Config + + classDef accent0 fill:#3B82F6,stroke:#2563EB,color:#fff + classDef accent1 fill:#22C55E,stroke:#16A34A,color:#fff + classDef accent2 fill:#F59E0B,stroke:#D97706,color:#000 + classDef accent3 fill:#A855F7,stroke:#7C3AED,color:#fff + classDef accent6 fill:#6366F1,stroke:#4F46E5,color:#fff + classDef accent7 fill:#64748B,stroke:#475569,color:#fff +``` + +## Layers + +### 1. Entry Points + +| Layer | Package | Role | +|---|---|---| +| **CLI** | `cmd/goget/` | Flag parsing, command routing, signal handling | +| **Library API** | `pkg/api/` | Public Go API for programmatic downloads, uploads, and queue management | + +The CLI and API share the same internal packages. The CLI is built on top of the same `internal/core` types that the API exposes. + +### 2. Protocol Abstraction + +Defined in `internal/core` and `internal/protocol`: + +```go +// core.Protocol is the interface every protocol handler implements. +type Protocol interface { + Scheme() string + CanHandle(url *url.URL) bool + Download(ctx context.Context, req *DownloadRequest) (*DownloadResult, error) + Capabilities() []string + SupportsResume() bool + SupportsCompression() bool + SupportsParallel() bool + SupportsRecursive() bool +} + +// core.Uploader is an optional interface for protocols supporting uploads. +type Uploader interface { + Upload(ctx context.Context, req *UploadRequest) (*UploadResult, error) + SupportsUpload() bool +} +``` + +Protocols register themselves by scheme in a global `Registry`. Scheme normalization maps HTTPS → HTTP, FTPS → FTP, etc. Supported protocols include HTTP, FTP, SFTP, File, DataURL, Gopher, Gemini, and WebDAV (new in `internal/protocol/webdav/`). + +### 3. Transport Layer + +`internal/transport` provides a custom HTTP transport with: + +- **IPv6-first dialer** — Resolves DNS, prefers IPv6 addresses, falls back to IPv4 only when no IPv6 record exists +- **Interface binding** — Linux-specific socket binding via `SO_BINDTODEVICE` +- **Rate limiting** — Token bucket algorithm for bandwidth capping +- **Retry logic** — Exponential backoff with configurable max retries +- **TLS configuration** — Certificate pinning, client certificates, key log file +- **Proxy support** — HTTP CONNECT and SOCKS5 (`socks5://` with local DNS, `socks5h://` with remote DNS, RFC 1929 username/password auth) + +### 4. Output Layer + +`internal/output` handles atomic file writes: + +1. Write to a `.goget.tmp` file during download +2. Store resume metadata in `.goget.meta` alongside the temp file +3. Atomically rename from `.goget.tmp` to the final filename on completion + +### 5. Utility Modules + +| Module | Responsibility | +|---|---| +| `auth` | HTTP Basic, Digest, OAuth 2.0 (client credentials, authorization code, refresh token), `.netrc` auto-reading | +| `cookie` | Netscape-format cookie jar with import/export | +| `crypto` | Checksum verification (SHA-256, SHA-512, SHA3-256/512, BLAKE2b, MD5), PGP signature verification and decryption, certificate handling | +| `compression` | Decompression wrappers for gzip, bzip2, zlib, flate, LZW | +| `hsts` | HSTS cache (RFC 6797) for automatic HTTPS upgrade | +| `archive` | Archive extraction for tar, tar.gz, tar.bz2, zip with auto-detection | +| `format` | Human-readable byte and speed formatting | +| `linkrewrite` | HTML link rewriting for offline mirror viewing | +| `log` | Structured logger with text and JSON output modes, level filtering, ANSI colors | +| `warc` | WARC (Web ARChive) format writing for legal and archival compliance | +| `cli` | ANSI color output, progress bar, help text, completion | + +## Download Flow + +```mermaid +sequenceDiagram + participant User + participant CLI as cmd/goget/ + participant Config as internal/config/ + participant Core as internal/core/ + participant Registry as internal/protocol/ + participant Proto as Protocol Handler + participant Transport as internal/transport/ + participant Output as internal/output/ + + User->>CLI: goget --url https://example.com/file.zip + CLI->>Config: Load config file + Config-->>CLI: Config (with defaults + overrides) + CLI->>Core: Create DownloadRequest + CLI->>Registry: Get protocol for URL scheme + Registry-->>CLI: HTTP protocol handler + CLI->>Proto: Download(ctx, request) + Proto->>Transport: Dial + send HTTP request + Transport-->>Proto: HTTP response stream + Proto->>Output: Write chunks atomically + Output-->>Proto: Progress + completion + Proto-->>CLI: DownloadResult + CLI-->>User: Progress bar + summary +``` + +## Config Merging + +Configuration is resolved in this priority order (highest first): + +1. CLI flags (`--timeout 5m`) +2. Config file (`~/.config/goget/config.toml`) +3. Default values + +The `config.Merge()` method applies CLI override top of the loaded config. + +## Error Handling + +All operations return structured `core.GogetError` with: + +- **Type** — Category (`NETWORK`, `PROTOCOL`, `FILE`, `TIMEOUT`, `AUTH`, `CHECKSUM`, `UNSUPPORTED`, `CANCELLED`) +- **Message** — Human-readable description +- **Cause** — Wrapped underlying error (available via `errors.Unwrap`) +- **URL** — The URL that caused the error (credentials stripped) +- **Details** — Optional key-value metadata (e.g., expected vs actual checksum) + +## Build Tags + +All source files use the build constraint `//go:build linux || freebsd`. The project targets Linux and FreeBSD only. Platform-specific code (e.g., network interface binding) lives in `internal/transport/bind_linux.go` vs `bind_other.go`. diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000..b692cc4 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,251 @@ +# Authentication + +goget supports multiple authentication mechanisms for HTTP, FTP, and SFTP connections. Credentials are resolved in priority order, with `.netrc` auto-reading as a fallback. + +## Priority Order + +1. CLI flags (`--username`, `--password`, `--password-file`) +2. OAuth 2.0 tokens (if configured via CLI or config) +3. `.netrc` auto-reading (`~/.netrc`) +4. Anonymous / no auth + +## HTTP Basic Authentication (RFC 7617) + +The simplest mechanism. Credentials are Base64-encoded in the `Authorization` header. + +```bash +goget --username admin --password secret --url https://api.example.com/data +``` + +``` +Authorization: Basic YWRtaW46c2VjcmV0 +``` + +### Security Notes + +- Basic auth sends credentials in plain text (Base64 is not encryption) +- Always use HTTPS with Basic auth +- Prefer `--password-file` for scripting to avoid credentials in shell history: + +```bash +goget --username admin --password-file ./secret.txt --url https://api.example.com +``` + +## HTTP Digest Authentication (RFC 7616) + +Digest auth never sends the password in plain text. Instead, it uses a challenge-response mechanism with MD5 or SHA-256 hashing: + +```mermaid +sequenceDiagram + participant Client + participant Server + + Client->>Server: GET /protected + Server-->>Client: 401 + WWW-Authenticate: Digest nonce="abc", realm="app" + Client->>Server: GET /protected + Authorization: Digest response="hash" + Server-->>Client: 200 OK +``` + +### Usage + +```bash +goget --username admin --password secret --auth-type digest --url https://api.example.com +goget --username admin --password secret --auth-type auto --url https://api.example.com +``` + +With `--auth-type auto`, goget first tries to make the request. If the server returns 401 with a `WWW-Authenticate: Digest` header, it automatically switches to Digest auth. + +### Implementation + +The `internal/auth` package implements Digest auth per RFC 7616: + +- **HA1** = `MD5(username:realm:password)` +- **HA2** = `MD5(method:digestURI)` +- **response** = `MD5(HA1:nonce:nc:cnonce:qop:HA2)` + +The `cnonce` (client nonce) is generated using `crypto/rand` for each request. + +## OAuth 2.0 + +goget supports four OAuth 2.0 grant types: + +### Client Credentials Grant + +For server-to-server communication without user interaction: + +```bash +goget \ + --oauth-client-id "my-client" \ + --oauth-client-secret "my-secret" \ + --oauth-token-url "https://auth.example.com/token" \ + --oauth-grant-type client_credentials \ + --oauth-scopes "read,write" \ + --url https://api.example.com/data +``` + +### Authorization Code Grant + +For web applications with user interaction: + +```bash +# Step 1: Generate authorization URL +goget \ + --oauth-client-id "my-client" \ + --oauth-auth-url "https://auth.example.com/authorize" \ + --oauth-redirect-uri "http://localhost:8080/callback" \ + --url https://api.example.com/data + +# Step 2: Exchange code for token (after user authorizes) +goget \ + --oauth-client-id "my-client" \ + --oauth-client-secret "my-secret" \ + --oauth-token-url "https://auth.example.com/token" \ + --oauth-grant-type authorization_code \ + --url https://api.example.com/data +``` + +### Refresh Token Grant + +For maintaining long-lived access: + +```bash +goget \ + --oauth-client-id "my-client" \ + --oauth-client-secret "my-secret" \ + --oauth-token-url "https://auth.example.com/token" \ + --oauth-refresh-token "ref_xxx" \ + --oauth-grant-type refresh_token \ + --url https://api.example.com/data +``` + +### Token Management + +```go +client := auth.NewOAuthClient(cfg) + +// Check if token is valid (not expired) +if !client.IsTokenValid() { + // Refresh automatically + client.RefreshToken(ctx) +} + +// Apply token to HTTP request +client.ApplyToken(httpReq) +// → Authorization: Bearer eyJhbG... +``` + +The token expiry is tracked with a 30-second grace period. + +### Security + +- **Token URL enforced to HTTPS** — plain HTTP token endpoints are rejected unless they're `localhost` or `127.0.0.1` +- **Tokens never logged** — The config `Save()` method strips `access_token`, `refresh_token`, and `client_secret` before writing to disk + +## .netrc Auto-Reading + +When no explicit credentials are provided via CLI flags or OAuth config, goget automatically checks `~/.netrc`: + +``` +machine api.example.com + login admin + password secret + +machine default + login guest + password guest123 +``` + +### Usage + +```bash +# No --username or --password needed if ~/.netrc has a matching entry +goget --url https://api.example.com/data +``` + +### Resolution Order + +1. **Exact hostname match** — `api.example.com` matches `machine api.example.com` +2. **Default entry** — `machine default` used as fallback +3. **No match** — proceeds without credentials + +### Skip .netrc + +Use `--no-private` to disable .netrc and auth file reading: + +```bash +goget --no-private --url https://api.example.com +``` + +## Cookies + +goget supports Netscape-format cookie jars: + +### CLI Cookies + +```bash +goget --cookie "session=abc123" --cookie "theme=dark" --url https://example.com +``` + +### Cookie Jar + +Persist cookies across sessions: + +```bash +# Save cookies to file +goget --cookie-jar cookies.txt --url https://example.com + +# Load cookies from file +goget --cookie-jar cookies.txt --url https://example.com/protected +``` + +The cookie jar uses the standard Netscape format: + +## SFTP Authentication + +SFTP connections authenticate via SSH: + +### SSH Key Authentication + +goget auto-detects keys from standard locations: + +- `~/.ssh/id_rsa` +- `~/.ssh/id_ecdsa` +- `~/.ssh/id_ed25519` + +```bash +goget --url sftp://user@host:/path/to/file.zip +``` + +### Password Authentication + +```bash +goget --url sftp://user@host:/path/to/file.zip --password "secret" +``` + +### Known Hosts + +Host key verification uses `~/.ssh/known_hosts` by default: + +```bash +# Custom known_hosts file +goget --url sftp://user@host:/path/to/file.zip --known-hosts ~/.ssh/known_hosts + +# Skip verification (first connection) +goget --url sftp://user@host:/path/to/file.zip --ssh-insecure +``` + +## FTP Authentication + +FTP connections default to anonymous login. For authenticated FTP: + +```bash +# Username/password +goget --url ftp://ftp.example.com/file.zip --username user --password pass + +# Anonymous (default) +goget --url ftp://anonymous@ftp.example.com/file.zip +``` + +## Access Tokens in URLs + +While `https://user:pass@example.com` URLs are supported, prefer `--username` and `--password` CLI flags. When a URL contains credentials, they're automatically stripped from error messages and logs via `core.SafeURL()`. diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..3548cd1 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,225 @@ +# CLI Reference + +Complete reference for all goget command-line flags, organized by category. + +```bash +goget --url [OPTIONS] +``` + +## Target + +| Flag | Argument | Description | +|---|---|---| +| `--url` | `` | URL to download (repeatable for multiple URLs) | + +## Output + +| Flag | Argument | Description | +|---|---|---| +| `--output` | `` | Output filename or directory (default: auto-derived from URL) | +| `--restart` | — | Delete existing file and restart download from scratch | +| `--overwrite` | — | Overwrite existing file without atomic rename | +| `--content-disposition` | — | Use filename from server's `Content-Disposition` header | +| `--create-dirs` | — | Create missing parent directories of `--output` (default: true). Use `--create-dirs=false` to disable (curl `--create-dirs` / wget behavior). Ignored for stdout output | +| `--keep-timestamps` | — | Set file modification time to server's `Last-Modified` value | + +## Progress + +| Flag | Argument | Description | +|---|---|---| +| `--verbose` | — | Verbose output with Unicode progress bar (default enabled) | +| `--no-progress` | — | Hide progress bar, keep other output | +| `--progress` | `dot` | Dot progress bar style | +| `--quiet` | — | Suppress all output except errors | +| `--json` | — | Output progress as JSON lines for scripting (format: `{"type":"progress","current":X,"total":X,"speed":X}`) | +| `--debug` | — | Enable debug output with detailed connection info | + +## Network + +| Flag | Argument | Description | +|---|---|---| +| `--timeout` | `` | Connection timeout (`0` = smart auto-calculation based on file size) | +| `--connect-timeout` | `` | TCP connection establishment timeout | +| `--max-time` | `` | Maximum total transfer time (abort if exceeded) | +| `--max-retries` | `` | Maximum retry attempts on failure | +| `--max-filesize` | `` | Abort if file exceeds size limit (e.g. `100MB`, `1GB`) | +| `--proxy` | `` | Proxy server URL — `http://` for HTTP CONNECT, `socks5://` (local DNS) or `socks5h://` (remote DNS) for SOCKS5, with optional `user:pass@` authentication | +| `--no-ipv4` | — | Disable IPv4 fallback (IPv6 only) | +| `--no-redirect` | — | Disable following HTTP redirects (3xx) | +| `--dns-servers` | `` | Custom DNS servers, comma-separated (e.g. `1.1.1.1,8.8.8.8`) | +| `--bind-interface` | `` | Bind outgoing connections to a specific network interface (Linux only) | +| `--allow-private` | — | Allow connections to private/internal IPs (disabled by default for SSRF protection) | +| `--pinned-cert` | `` | Certificate pinning — abort if server cert SHA-256 doesn't match | + +## HTTP / Network Flags + +| Flag | Argument | Description | +|---|---|---| +| `--header` | `"K: V"` | Custom HTTP header (repeatable) | +| `--insecure` | — | Skip TLS certificate verification | +| `--cert` | `` | Client certificate in PEM format (mTLS) | +| `--key` | `` | Client private key in PEM format (mTLS) | +| `--data` | `"key=value"` | Send POST data (implies POST) | +| `--form` | `"field=value"` | Multipart form field (repeatable; `--form "file=@./path"` for file upload) | +| `--spider` | — | Check URL existence via HEAD request — no body downloaded | +| `--write-out` | `` | Print metadata after transfer (format: `%{http_code}`, `%{size_download}`, `%{time_total}`, `%{speed_download}`) | +| `--trace-time` | — | Add HTTP timing breakdown to output (DNS, TCP, TLS, TTFB, Total) | +| `--fail` | — | Exit non-zero on HTTP 4xx/5xx response | +| `--compressed` | — | Accept-Encoding header for compressed transfer | +| `--range` | `-` | Download a byte range (e.g. `--range 0-999`) | +| `--post-file` | `` | POST file contents as request body | +| `--referer` | `` | Set Referer header | + +## Recursive / Spider Flags + +| Flag | Argument | Description | +|---|---|---| +| `--accept` | `"*.pdf"` | Accept patterns — filename glob or MIME type (comma-separated) | +| `--no-parent` | — | Do not ascend to parent directories during recursive download | +| `--adjust-extension` | — | Append `.html` extension to text/html files without one | +| `--wait` | `` | Delay between requests in recursive mode | +| `--random-wait` | — | Randomize wait time (0.5x–1.5x of `--wait` value) | +| `--page-requisites` | — | Download CSS, JS, and images required to render HTML pages | +| `--span-hosts` | — | Follow links to external domains in recursive mode | +| `--convert-links` | — | Rewrite links in downloaded HTML for local offline viewing | +| `--warc-file` | `` | Write WARC (Web ARChive) output for archival | +| `--dry-run` | — | List all URLs that would be downloaded in recursive/mirror mode without saving files to disk | +| `--progress` | `dot` | Dot progress bar style | +| `--no-clobber` | — | Skip download if file already exists | + +## Recursive / Mirror + +| Flag | Argument | Description | +|---|---|---| +| `--recursive` | — | Enable recursive downloading — follow links from HTML pages | +| `--mirror` | — | Mirror entire website (infinite depth, convert links) — shorthand for `--recursive --convert-links --page-requisites` | +| `--max-depth` | `` | Maximum recursion depth | +| `--follow-external` | — | Follow links to domains other than the starting domain | +| `--exclude-pattern` | `

` | Exclude URL patterns (glob, comma-separated) | +| `--no-convert-links` | — | Don't convert links in HTML during mirror (faster, but not offline-ready) | +| `--no-mirror-assets` | — | Don't download CSS, JS, and images during mirror | +| `--no-robots` | — | Don't respect `robots.txt` during mirroring | +| `--rate-limit` | `` | Maximum download speed (e.g. `1MB/s`, `500KB/s`, `0` = unlimited) | +| `--domains` | `` | Restrict recursion to listed domains (comma-separated) | +| `--reject` | `` | Reject URL patterns (glob) | +| `--cut-dirs` | `` | Ignore N directory components when creating local filenames | +| `--proto-dirs` | — | Create protocol-prefixed directories (http/, ftp/ etc.) | +| `--adjust-extension` | — | Append `.html` to text/html files missing an extension | +| `--timestamping` | — | Only download files newer than the local copy | +| `--backup-converted` | — | Back up original `.orig` files before link conversion | +| `--recursive-parallel` | `` | Number of concurrent file downloads in recursive mode across all protocols (HTTP, WebDAV, FTP, SFTP). `0` = sequential (default) | +| `--continue` | — | Continue getting partially-downloaded file | + +## Download Management + +| Flag | Argument | Description | +|---|---|---| +| `--queue` | — | Add URL to the download queue (see `--queue-file`) | +| `--queue-list` | — | List all items in the queue | +| `--queue-process` | — | Process (download) all pending items in the queue | +| `--queue-clear` | — | Remove completed items from the queue | +| `--queue-file` | `` | Custom queue file path (default: `~/.config/goget/queue.json`) | +| `--queue-priority` | `` | Priority for queued item (1–10, higher = processed first) | +| `--batch-file` | `` | Read URLs from a file, one URL per line | +| `--input-file` | `` | Read URLs from a file (alias for `--batch-file`) | +| `--schedule` | `

Hello World

+

No links here!

+ + + ` + + links, err := ExtractLinks(baseURL, []byte(html)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(links) != 0 { + t.Errorf("expected 0 links from HTML without links, got %d", len(links)) + } +} + +func TestExtractLinksBaseURLWithPath(t *testing.T) { + baseURL, _ := url.Parse("https://example.com/docs/guide/") + + t.Run("relative URL resolves relative to base path", func(t *testing.T) { + html := `Chapter 1` + links, err := ExtractLinks(baseURL, []byte(html)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(links) != 1 { + t.Fatalf("expected 1 link, got %d", len(links)) + } + expected := "https://example.com/docs/guide/chapter1.html" + if links[0].String() != expected { + t.Errorf("expected %s, got %s", expected, links[0].String()) + } + }) + + t.Run("parent path resolution", func(t *testing.T) { + html := `Logo` + links, err := ExtractLinks(baseURL, []byte(html)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(links) != 1 { + t.Fatalf("expected 1 link, got %d", len(links)) + } + expected := "https://example.com/docs/images/logo.png" + if links[0].String() != expected { + t.Errorf("expected %s, got %s", expected, links[0].String()) + } + }) +} + +func TestExtractLinksDuplicateRemoval(t *testing.T) { + baseURL, _ := url.Parse("https://example.com/") + + html := ` + Page 1 + Page 2 + Page 3 + Other + ` + + links, err := ExtractLinks(baseURL, []byte(html)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(links) != 2 { + t.Fatalf("expected 2 unique links, got %d", len(links)) + } + + expectedURLs := []string{ + "https://example.com/page.html", + "https://example.com/other.html", + } + + for _, expected := range expectedURLs { + found := false + for _, link := range links { + if link.String() == expected { + found = true + break + } + } + if !found { + t.Errorf("expected URL %s not found", expected) + } + } +} + +func TestExtractLinksFromHTML(t *testing.T) { + baseURL, _ := url.Parse("https://example.com/") + html := `Test` + + links1, err1 := ExtractLinks(baseURL, []byte(html)) + links2, err2 := ExtractLinksFromHTML(baseURL, []byte(html)) + + if err1 != nil || err2 != nil { + t.Fatal("both functions should return no error") + } + + if len(links1) != len(links2) { + t.Fatalf("ExtractLinks returned %d links, ExtractLinksFromHTML returned %d", len(links1), len(links2)) + } + + if len(links1) > 0 && links1[0].String() != links2[0].String() { + t.Errorf("ExtractLinks returned %s, ExtractLinksFromHTML returned %s", links1[0].String(), links2[0].String()) + } +} + +// ============================================================ +// Crawler tests +// ============================================================ + +func TestNewCrawler(t *testing.T) { + cfg := &CrawlerConfig{ + MaxDepth: 3, + FollowExternal: false, + OutputDir: "/tmp/test", + Parallel: 2, + UserAgent: "TestBot/1.0", + } + + crawler := NewCrawler(cfg, nil) + + if crawler == nil { + t.Fatal("NewCrawler returned nil") + } + if crawler.config.MaxDepth != 3 { + t.Errorf("expected MaxDepth 3, got %d", crawler.config.MaxDepth) + } + if crawler.config.FollowExternal != false { + t.Errorf("expected FollowExternal false, got %v", crawler.config.FollowExternal) + } + if crawler.config.OutputDir != "/tmp/test" { + t.Errorf("expected OutputDir /tmp/test, got %s", crawler.config.OutputDir) + } + if crawler.config.UserAgent != "TestBot/1.0" { + t.Errorf("expected UserAgent TestBot/1.0, got %s", crawler.config.UserAgent) + } + if crawler.config.Parallel != 2 { + t.Errorf("expected Parallel 2, got %d", crawler.config.Parallel) + } + if crawler.filter == nil { + t.Error("expected filter to be initialized") + } + if crawler.visited == nil { + t.Error("expected visited map to be initialized") + } + if crawler.queue == nil { + t.Error("expected queue to be initialized") + } + if crawler.stats == nil { + t.Error("expected stats to be initialized") + } + if crawler.ctx == nil { + t.Error("expected context to be initialized") + } +} + +func TestNewCrawlerNilURLFilterConfig(t *testing.T) { + cfg := &CrawlerConfig{ + MaxDepth: 0, + FollowExternal: false, + ExcludePatterns: nil, + IncludePatterns: nil, + } + + crawler := NewCrawler(cfg, nil) + + if crawler == nil { + t.Fatal("NewCrawler returned nil") + } + if crawler.filter == nil { + t.Fatal("expected filter to be initialized") + } + if len(crawler.filter.excludePatterns) != 0 { + t.Errorf("expected 0 exclude patterns, got %d", len(crawler.filter.excludePatterns)) + } + if len(crawler.filter.includePatterns) != 0 { + t.Errorf("expected 0 include patterns, got %d", len(crawler.filter.includePatterns)) + } +} + +func TestNewCrawlerDefaultConfig(t *testing.T) { + cfg := &CrawlerConfig{} + crawler := NewCrawler(cfg, nil) + + if crawler == nil { + t.Fatal("NewCrawler returned nil") + } + if crawler.config.MaxDepth != 0 { + t.Errorf("expected MaxDepth 0 as default, got %d", crawler.config.MaxDepth) + } + if crawler.config.Parallel != 0 { + t.Errorf("expected Parallel 0 as default, got %d", crawler.config.Parallel) + } + if crawler.stats == nil { + t.Error("expected stats to be initialized") + } + if crawler.stats.StartTime.IsZero() { + t.Error("expected StartTime to be set") + } +} + +func TestGetStats(t *testing.T) { + cfg := &CrawlerConfig{} + crawler := NewCrawler(cfg, nil) + + stats := crawler.GetStats() + + if stats == nil { + t.Fatal("GetStats returned nil") + } + if stats.TotalURLs != 0 { + t.Errorf("expected TotalURLs 0, got %d", stats.TotalURLs) + } + if stats.DownloadedURLs != 0 { + t.Errorf("expected DownloadedURLs 0, got %d", stats.DownloadedURLs) + } + if stats.FailedURLs != 0 { + t.Errorf("expected FailedURLs 0, got %d", stats.FailedURLs) + } + if stats.SkippedURLs != 0 { + t.Errorf("expected SkippedURLs 0, got %d", stats.SkippedURLs) + } + if stats.TotalBytes != 0 { + t.Errorf("expected TotalBytes 0, got %d", stats.TotalBytes) + } + if stats.StartTime.IsZero() { + t.Error("expected StartTime to be set") + } + + // Verify it returns the same pointer (not a copy) + if stats != crawler.stats { + t.Error("GetStats should return the same stats instance") + } +} + +func TestGetStatsSummary(t *testing.T) { + stats := &CrawlerStats{ + DownloadedURLs: 10, + TotalBytes: 1500000, + StartTime: time.Now().Add(-5 * time.Second), + EndTime: time.Now(), + } + + summary := stats.GetStatsSummary() + + if !strings.Contains(summary, "10") { + t.Errorf("summary should contain 10 downloaded files, got: %s", summary) + } + if !strings.Contains(summary, "1.5 MB") { + t.Errorf("summary should contain 1.5 MB, got: %s", summary) + } + if !strings.Contains(summary, "Downloaded:") { + t.Errorf("summary should start with 'Downloaded:', got: %s", summary) + } + + // Test with EndTime set + stats.EndTime = time.Now() + summary = stats.GetStatsSummary() + if !strings.Contains(summary, "Downloaded:") { + t.Errorf("summary should be valid when EndTime is set, got: %s", summary) + } + + // Test zero values + emptyStats := &CrawlerStats{} + summary = emptyStats.GetStatsSummary() + if summary == "" { + t.Error("summary should not be empty even for zero stats") + } +} + +func TestHttpDetectContentType(t *testing.T) { + t.Run("HTML declaration", func(t *testing.T) { + data := []byte("") + contentType := httpDetectContentType(data) + if contentType != "text/html" { + t.Errorf("expected text/html, got %s", contentType) + } + }) + + t.Run("HTML tag", func(t *testing.T) { + data := []byte("") + contentType := httpDetectContentType(data) + if contentType != "text/html" { + t.Errorf("expected text/html, got %s", contentType) + } + }) + + t.Run("uppercase HTML", func(t *testing.T) { + data := []byte("") + contentType := httpDetectContentType(data) + if contentType != "text/html" { + t.Errorf("expected text/html, got %s", contentType) + } + }) + + t.Run("non-HTML content", func(t *testing.T) { + data := []byte("") + contentType := httpDetectContentType(data) + if contentType != "" { + t.Errorf("expected empty string, got %s", contentType) + } + }) + + t.Run("empty data", func(t *testing.T) { + contentType := httpDetectContentType([]byte{}) + if contentType != "" { + t.Errorf("expected empty string, got %s", contentType) + } + }) + + t.Run("nil data", func(t *testing.T) { + contentType := httpDetectContentType(nil) + if contentType != "" { + t.Errorf("expected empty string, got %s", contentType) + } + }) + + t.Run("single byte", func(t *testing.T) { + contentType := httpDetectContentType([]byte{'x'}) + if contentType != "" { + t.Errorf("expected empty string, got %s", contentType) + } + }) + + t.Run("binary data", func(t *testing.T) { + data := []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A} + contentType := httpDetectContentType(data) + if contentType != "" { + t.Errorf("expected empty string for binary data, got %s", contentType) + } + }) +} + +// TestExtractCSSURLs tests CSS URL extraction. +func TestExtractCSSURLs(t *testing.T) { + baseURL, _ := url.Parse("https://example.com/css/") + + t.Run("url function", func(t *testing.T) { + css := []byte(`body { background: url("bg.png"); }`) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 1 { + t.Fatalf("expected 1 URL, got %d", len(urls)) + } + expected := "https://example.com/css/bg.png" + if urls[0].String() != expected { + t.Errorf("expected %s, got %s", expected, urls[0].String()) + } + }) + + t.Run("import rule", func(t *testing.T) { + css := []byte(`@import "style.css";`) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 1 { + t.Fatalf("expected 1 URL, got %d", len(urls)) + } + expected := "https://example.com/css/style.css" + if urls[0].String() != expected { + t.Errorf("expected %s, got %s", expected, urls[0].String()) + } + }) + + t.Run("multiple urls", func(t *testing.T) { + css := []byte(` + body { background: url("bg.png"); } + div { background: url('icon.svg'); } + @import "theme.css"; + `) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 3 { + t.Fatalf("expected 3 URLs, got %d", len(urls)) + } + }) + + t.Run("deduplicates", func(t *testing.T) { + css := []byte(`body { background: url("bg.png"); background: url("bg.png"); }`) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 1 { + t.Fatalf("expected 1 unique URL, got %d", len(urls)) + } + }) + + t.Run("empty", func(t *testing.T) { + urls := ExtractCSSURLs(baseURL, []byte(``)) + if len(urls) != 0 { + t.Errorf("expected 0 URLs, got %d", len(urls)) + } + }) + + t.Run("no urls", func(t *testing.T) { + css := []byte(`body { color: red; }`) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 0 { + t.Errorf("expected 0 URLs, got %d", len(urls)) + } + }) + + t.Run("skips non-http", func(t *testing.T) { + css := []byte(`body { background: url("data:image/png;base64,abc"); }`) + urls := ExtractCSSURLs(baseURL, css) + if len(urls) != 0 { + t.Errorf("expected 0 URLs for data: URI, got %d", len(urls)) + } + }) +} diff --git a/internal/transport/bind_linux.go b/internal/transport/bind_linux.go new file mode 100644 index 0000000..28405c2 --- /dev/null +++ b/internal/transport/bind_linux.go @@ -0,0 +1,24 @@ +//go:build linux +// +build linux + +package transport + +import ( + "net" + "syscall" +) + +// bindToInterface returns a Control function that binds connections +// to the specified network interface using SO_BINDTODEVICE. +func bindToInterface(iface string) func(string, string, syscall.RawConn) error { + return func(network, address string, c syscall.RawConn) error { + var err error + c.Control(func(fd uintptr) { + err = syscall.BindToDevice(int(fd), iface) + }) + return err + } +} + +// Ensure net import is used +var _ net.Conn diff --git a/internal/transport/bind_other.go b/internal/transport/bind_other.go new file mode 100644 index 0000000..60d1b5f --- /dev/null +++ b/internal/transport/bind_other.go @@ -0,0 +1,15 @@ +//go:build !linux +// +build !linux + +package transport + +import ( + "net" + "syscall" +) + +// bindToInterface returns nil on non-Linux platforms +// since SO_BINDTODEVICE is a Linux-specific socket option. +func bindToInterface(_ string) func(string, string, syscall.RawConn) error { return nil } + +var _ net.Conn diff --git a/internal/transport/dialer.go b/internal/transport/dialer.go new file mode 100644 index 0000000..fc0dc3c --- /dev/null +++ b/internal/transport/dialer.go @@ -0,0 +1,285 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "context" + "fmt" + "net" + "net/url" + "sort" + "strings" + "time" +) + +// DialerConfig configures the dialer +type DialerConfig struct { + // Timeout for total connection + Timeout time.Duration + + // Timeout for IPv6 attempt before fallback to IPv4 + IPv6Timeout time.Duration + + // Enable IPv4 fallback + IPv4Fallback bool + + // Debug mode for logging + Debug bool + + // BindInterface specifies the network interface to bind to + BindInterface string + + // BlockPrivateIPs blocks connections to private/internal IP ranges + BlockPrivateIPs bool + + // Callback for reporting used IP version + OnConnect func(ip string, version int) +} + +// DefaultDialerConfig returns the default configuration +func DefaultDialerConfig() *DialerConfig { + return &DialerConfig{ + Timeout: 30 * time.Second, + IPv6Timeout: 5 * time.Second, + IPv4Fallback: true, + Debug: false, + BlockPrivateIPs: true, // SSRF protection on by default + } +} + +// Dialer is an IPv6-first dialer with fallback +type Dialer struct { + config *DialerConfig + resolver *net.Resolver + dialer *net.Dialer +} + +// NewDialer creates a new dialer +func NewDialer(cfg *DialerConfig) *Dialer { + if cfg == nil { + cfg = DefaultDialerConfig() + } + + d := &Dialer{ + config: cfg, + resolver: &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + dial := net.Dialer{ + Timeout: 5 * time.Second, + DualStack: true, + } + return dial.DialContext(ctx, network, address) + }, + }, + } + + // Configure bind interface if specified + if cfg.BindInterface != "" { + d.dialer = &net.Dialer{ + Timeout: cfg.Timeout, + DualStack: false, + Control: bindToInterface(cfg.BindInterface), + } + } + + return d +} + +// DialContext connects to an address with IPv6-first strategy +func (d *Dialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("invalid address: %w", err) + } + + // Resolve all IP addresses + ips, err := d.resolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("dns lookup failed: %w", err) + } + + // Split into IPv6 and IPv4 + var ipv6Addrs, ipv4Addrs []net.IPAddr + for _, ip := range ips { + if ip.IP.To4() == nil { + ipv6Addrs = append(ipv6Addrs, ip) + } else { + ipv4Addrs = append(ipv4Addrs, ip) + } + } + + // Sort: IPv6 first, then IPv4 + var orderedAddrs []net.IPAddr + orderedAddrs = append(orderedAddrs, ipv6Addrs...) + if d.config.IPv4Fallback { + orderedAddrs = append(orderedAddrs, ipv4Addrs...) + } + + if len(orderedAddrs) == 0 { + return nil, fmt.Errorf("no ip addresses found for %s", host) + } + + // SSRF protection: block private/internal IPs + if d.config.BlockPrivateIPs { + for _, addr := range orderedAddrs { + if isPrivateIP(addr.IP) { + return nil, fmt.Errorf("ssrf protection: blocked private ip %s for host %s", addr.IP, host) + } + } + } + + // Try to connect in order (IPv6 first) + var lastErr error + for _, ipAddr := range orderedAddrs { + ip := ipAddr.IP.String() + address := net.JoinHostPort(ip, port) + + // Determine timeout based on IP version + dialTimeout := d.config.Timeout + if ipAddr.IP.To4() == nil && d.config.IPv4Fallback { + // IPv6 with fallback: shorter timeout + dialTimeout = d.config.IPv6Timeout + } + + dialCtx, cancel := context.WithTimeout(ctx, dialTimeout) + + var conn net.Conn + var err error + if d.dialer != nil { + // Use pre-configured dialer (e.g., with bind interface) + dialer := *d.dialer + dialer.Timeout = dialTimeout + conn, err = dialer.DialContext(dialCtx, network, address) + } else { + conn, err = (&net.Dialer{ + Timeout: dialTimeout, + DualStack: false, // Explicitly disabled for deterministic behavior + }).DialContext(dialCtx, network, address) + } + + cancel() + + if err == nil { + // Connection successful + ipVersion := 6 + if ipAddr.IP.To4() != nil { + ipVersion = 4 + } + + if d.config.Debug { + fmt.Printf("[DEBUG] Connected to %s (IPv%d)\n", ip, ipVersion) + } + + if d.config.OnConnect != nil { + d.config.OnConnect(ip, ipVersion) + } + + return conn, nil + } + + lastErr = err + + if d.config.Debug { + fmt.Printf("[DEBUG] Failed to connect to %s: %v\n", ip, err) + } + } + + return nil, fmt.Errorf("all connection attempts failed: %w", lastErr) +} + +// LookupIPAddr performs DNS lookup with IPv6 preference +func (d *Dialer) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) { + ips, err := d.resolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, err + } + + // Sort: IPv6 first + sort.Slice(ips, func(i, j int) bool { + iIsV6 := ips[i].IP.To4() == nil + jIsV6 := ips[j].IP.To4() == nil + if iIsV6 && !jIsV6 { + return true + } + if !iIsV6 && jIsV6 { + return false + } + return false + }) + + return ips, nil +} + +// ParseProxyURL parses the proxy URL and returns host:port +func ParseProxyURL(proxyURL string) (*url.URL, error) { + if proxyURL == "" { + return nil, nil + } + + u, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy url: %w", err) + } + + if u.Host == "" { + return nil, fmt.Errorf("proxy url missing host") + } + + return u, nil +} + +// SetCustomDNS configures custom DNS servers for the dialer +func (d *Dialer) SetCustomDNS(servers []string) { + if d == nil || len(servers) == 0 { + return + } + + d.resolver = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + dialer := &net.Dialer{Timeout: d.config.Timeout} + server := servers[0] + if !strings.Contains(server, ":") { + server = net.JoinHostPort(server, "53") + } + return dialer.DialContext(ctx, "udp", server) + }, + } +} + +// ParseDNSServers parses a comma-separated list of DNS servers +func ParseDNSServers(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + var servers []string + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + servers = append(servers, p) + } + } + return servers +} + +// isPrivateIP checks if an IP address is in a private/internal range. +// Always allows localhost (127.0.0.0/8, ::1). +// IPv4-mapped IPv6 addresses (e.g. ::ffff:10.0.0.1) are normalized to IPv4 +// first, otherwise net.IP.IsPrivate returns false for them and an attacker +// controlling DNS could bypass SSRF protection. +func isPrivateIP(ip net.IP) bool { + if ip.IsLoopback() { + return false // always allow localhost + } + if v4 := ip.To4(); v4 != nil { + ip = v4 + } + if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsPrivate() || ip.IsUnspecified() { + return true + } + return false +} diff --git a/internal/transport/dialer_test.go b/internal/transport/dialer_test.go new file mode 100644 index 0000000..c5572dc --- /dev/null +++ b/internal/transport/dialer_test.go @@ -0,0 +1,60 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "net" + "testing" +) + +// TestIsPrivateIP verifies that isPrivateIP correctly blocks private and +// reserved ranges, allows loopback and public addresses, and — critically — +// detects IPv4-mapped IPv6 representations of private addresses. +func TestIsPrivateIP(t *testing.T) { + cases := []struct { + name string + ip string + want bool + }{ + // Loopback — must always pass (localhost is allowed). + {"ipv4 loopback", "127.0.0.1", false}, + {"ipv6 loopback", "::1", false}, + + // Plain IPv4 private/reserved — must be blocked. + {"ipv4 rfc1918 10/8", "10.0.0.1", true}, + {"ipv4 rfc1918 172.16/12", "172.16.0.1", true}, + {"ipv4 rfc1918 192.168/16", "192.168.1.1", true}, + {"ipv4 link-local", "169.254.0.1", true}, + {"ipv4 unspecified", "0.0.0.0", true}, + + // IPv4-mapped IPv6 — must be normalized and blocked. + // This is the SSRF bypass the fix addresses. + {"ipv4-mapped private", "::ffff:10.0.0.1", true}, + {"ipv4-mapped rfc1918 172.16", "::ffff:172.16.0.1", true}, + {"ipv4-mapped rfc1918 192.168", "::ffff:192.168.1.1", true}, + {"ipv4-mapped link-local", "::ffff:169.254.0.1", true}, + {"ipv4-mapped loopback", "::ffff:127.0.0.1", false}, + + // Plain IPv6 private/link-local — must be blocked. + {"ipv6 unique-local fc00::/7", "fc00::1", true}, + {"ipv6 link-local fe80::/10", "fe80::1", true}, + + // Public addresses — must pass. + {"ipv4 public", "8.8.8.8", false}, + {"ipv4-mapped public", "::ffff:8.8.8.8", false}, + {"ipv6 public", "2606:4700:4700::1111", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ip := net.ParseIP(tc.ip) + if ip == nil { + t.Fatalf("net.ParseIP(%q) returned nil", tc.ip) + } + if got := isPrivateIP(ip); got != tc.want { + t.Errorf("isPrivateIP(%s) = %v, want %v", tc.ip, got, tc.want) + } + }) + } +} diff --git a/internal/transport/proxy.go b/internal/transport/proxy.go new file mode 100644 index 0000000..798fdaa --- /dev/null +++ b/internal/transport/proxy.go @@ -0,0 +1,405 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "bufio" + "context" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" +) + +// ProxyType defines the proxy type +type ProxyType string + +const ( + ProxyHTTP ProxyType = "http" + ProxyHTTPS ProxyType = "https" + ProxySOCKS5 ProxyType = "socks5" +) + +// ProxyConfig configures the proxy +type ProxyConfig struct { + // Proxy server URL + URL *url.URL + + // Proxy type + Type ProxyType + + // Authentication + Username string + Password string + + // SOCKS5LocalResolve controls DNS resolution behavior for SOCKS5 proxies. + // When true (socks5:// scheme), hostnames are resolved locally before being + // sent to the proxy. When false (socks5h:// scheme), the hostname is sent + // to the proxy for remote resolution. + SOCKS5LocalResolve bool +} + +// ParseProxyConfig parses proxy URL into configuration +func ParseProxyConfig(proxyURL string) (*ProxyConfig, error) { + if proxyURL == "" { + return nil, nil + } + + u, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy url: %w", err) + } + + var proxyType ProxyType + socks5LocalResolve := false + switch u.Scheme { + case "http", "https": + proxyType = ProxyHTTP + case "socks5": + proxyType = ProxySOCKS5 + socks5LocalResolve = true + case "socks5h": + proxyType = ProxySOCKS5 + // socks5LocalResolve stays false + default: + proxyType = ProxyHTTP + } + + username := u.User.Username() + password, _ := u.User.Password() + + // Remove authentication from URL for further use + u.User = nil + + return &ProxyConfig{ + URL: u, + Type: proxyType, + Username: username, + Password: password, + SOCKS5LocalResolve: socks5LocalResolve, + }, nil +} + +// NewProxyTransport creates HTTP transport with proxy +func NewProxyTransport(cfg *ProxyConfig, baseTransport *http.Transport) (*http.Transport, error) { + if cfg == nil { + return baseTransport, nil + } + + // SOCKS5 must be implemented as a custom DialContext — Go's stdlib + // http.ProxyURL only supports HTTP CONNECT proxies. + if cfg.Type == ProxySOCKS5 { + return newSOCKS5Transport(cfg, baseTransport) + } + + // Create a copy of the transport + transport := baseTransport.Clone() + + // Set up the proxy + proxyURL := cfg.URL.String() + if cfg.Username != "" { + // Add authentication back for http.ProxyURL + if cfg.Password != "" { + proxyURL = fmt.Sprintf("%s://%s:%s@%s", + cfg.URL.Scheme, + cfg.Username, + cfg.Password, + cfg.URL.Host) + } else { + proxyURL = fmt.Sprintf("%s://%s@%s", + cfg.URL.Scheme, + cfg.Username, + cfg.URL.Host) + } + } + + parsedProxy, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy url: %w", err) + } + + transport.Proxy = http.ProxyURL(parsedProxy) + + return transport, nil +} + +// newSOCKS5Transport creates an http.Transport that routes connections +// through a SOCKS5 proxy by overriding DialContext. The transport's Proxy +// field is cleared so the stdlib HTTP proxy machinery does not interfere. +func newSOCKS5Transport(cfg *ProxyConfig, baseTransport *http.Transport) (*http.Transport, error) { + transport := baseTransport.Clone() + transport.Proxy = nil + + // Preserve the existing dialer if one was configured (e.g., bind interface). + // SOCKS5 dialing uses net.Dialer directly because the proxy itself handles + // connection establishment to the target. + prevDial := transport.DialContext + transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + // When the caller already provided a custom dialer (e.g., bind-to-device), + // use it to reach the SOCKS5 proxy. The proxy then negotiates the + // connection to addr. + if prevDial != nil { + return prevDial(ctx, network, cfg.URL.Host) + } + return (&net.Dialer{}).DialContext(ctx, network, cfg.URL.Host) + } + + // Wrap so the actual target connection happens via SOCKS5 handshake. + transport.DialContext = socks5DialContext(transport.DialContext, cfg) + return transport, nil +} + +// socks5DialContext wraps a base dialer (which connects to the SOCKS5 proxy +// host) and returns a DialContext that performs the SOCKS5 handshake to reach +// the requested target address. +func socks5DialContext(proxyDialer DialContextFunc, cfg *ProxyConfig) DialContextFunc { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + proxyConn, err := proxyDialer(ctx, network, cfg.URL.Host) + if err != nil { + return nil, fmt.Errorf("connect to socks5 proxy: %w", err) + } + // Perform SOCKS5 handshake on top of the proxy connection. + return socks5Handshake(ctx, proxyConn, network, addr, cfg) + } +} + +// DialContextFunc is the shape of net.Dialer.DialContext — used internally +// for wrapping proxy dialers. +type DialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// DialProxy creates a connection through the proxy +func DialProxy(ctx context.Context, network, addr string, cfg *ProxyConfig) (net.Conn, error) { + if cfg == nil { + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + + switch cfg.Type { + case ProxyHTTP, ProxyHTTPS: + return dialHTTPProxy(ctx, network, addr, cfg) + case ProxySOCKS5: + return dialSOCKS5Proxy(ctx, network, addr, cfg) + default: + return nil, fmt.Errorf("unsupported proxy type: %s", cfg.Type) + } +} + +// dialHTTPProxy creates a connection via HTTP CONNECT proxy +func dialHTTPProxy(ctx context.Context, network, addr string, cfg *ProxyConfig) (net.Conn, error) { + if network != "tcp" && network != "tcp4" && network != "tcp6" { + return nil, fmt.Errorf("unsupported network: %s", network) + } + + // Connect to the proxy server + proxyAddr := cfg.URL.Host + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", proxyAddr) + if err != nil { + return nil, fmt.Errorf("failed to connect to proxy: %w", err) + } + + // Validate addr for CRLF injection + if strings.ContainsAny(addr, "\r\n") { + return nil, fmt.Errorf("invalid proxy target address") + } + + // Send HTTP CONNECT request + connectReq := fmt.Sprintf("CONNECT %s HTTP/1.1\r\nHost: %s\r\n", addr, addr) + + if cfg.Username != "" { + auth := base64.StdEncoding.EncodeToString([]byte(cfg.Username + ":" + cfg.Password)) + connectReq += fmt.Sprintf("Proxy-Authorization: Basic %s\r\n", auth) + } + + connectReq += "\r\n" + + _, err = conn.Write([]byte(connectReq)) + if err != nil { + conn.Close() + return nil, fmt.Errorf("failed to send connect request: %w", err) + } + + // Read the proxy response + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + conn.Close() + return nil, fmt.Errorf("failed to read proxy response: %w", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + conn.Close() + return nil, fmt.Errorf("proxy connect returned %s", resp.Status) + } + + return conn, nil +} + +// dialSOCKS5Proxy creates a connection via SOCKS5 proxy +func dialSOCKS5Proxy(ctx context.Context, network, addr string, cfg *ProxyConfig) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", cfg.URL.Host) + if err != nil { + return nil, fmt.Errorf("failed to connect to socks5 proxy: %w", err) + } + return socks5Handshake(ctx, conn, network, addr, cfg) +} + +// socks5Handshake performs the SOCKS5 negotiation on top of an already-open +// connection to the proxy. It implements RFC 1928 (SOCKS5) and RFC 1929 +// (username/password auth). If cfg.SOCKS5LocalResolve is true and addr contains +// a hostname, the hostname is resolved locally before being sent to the proxy. +func socks5Handshake(ctx context.Context, conn net.Conn, network, addr string, cfg *ProxyConfig) (retConn net.Conn, retErr error) { + // Ensure the connection is closed on any error path below. + defer func() { + if retErr != nil { + conn.Close() + } + }() + + // SOCKS5 greeting + authMethods := []byte{0x00} // No auth + if cfg.Username != "" { + authMethods = []byte{0x02} // Username/password auth + } + greeting := append([]byte{0x05, byte(len(authMethods))}, authMethods...) + + if _, err := conn.Write(greeting); err != nil { + return nil, fmt.Errorf("socks5 greeting failed: %w", err) + } + + // Read greeting response + resp := make([]byte, 2) + if _, err := io.ReadFull(conn, resp); err != nil { + return nil, fmt.Errorf("socks5 greeting response failed: %w", err) + } + + if resp[0] != 0x05 { + return nil, fmt.Errorf("socks5: invalid version: %d", resp[0]) + } + + if resp[1] == 0x02 && cfg.Username != "" { + // Username/password auth + authMsg := []byte{0x01, byte(len(cfg.Username))} + authMsg = append(authMsg, []byte(cfg.Username)...) + authMsg = append(authMsg, byte(len(cfg.Password))) + authMsg = append(authMsg, []byte(cfg.Password)...) + + if _, err := conn.Write(authMsg); err != nil { + return nil, fmt.Errorf("socks5 auth failed: %w", err) + } + + authResp := make([]byte, 2) + if _, err := io.ReadFull(conn, authResp); err != nil { + return nil, fmt.Errorf("socks5 auth response failed: %w", err) + } + if authResp[1] != 0x00 { + return nil, fmt.Errorf("socks5 authentication failed") + } + } else if resp[1] != 0x00 { + return nil, fmt.Errorf("socks5: no acceptable auth method") + } + + // SOCKS5 CONNECT request + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("invalid target address: %w", err) + } + + port, _ := strconv.Atoi(portStr) + + var connectReq []byte + connectReq = append(connectReq, 0x05, 0x01, 0x00) // VER=5, CMD=CONNECT, RSV=0 + + // Decide what to send: IPv4 literal, IPv6 literal, or domain name. + // When SOCKS5LocalResolve is true, hostnames are resolved locally and + // sent as IPs; otherwise the hostname is passed through to the proxy. + target := host + if cfg.SOCKS5LocalResolve { + if ip := net.ParseIP(host); ip == nil { + resolved, err := resolveSOCKS5Hostname(ctx, host) + if err != nil { + return nil, fmt.Errorf("socks5: resolve %s: %w", host, err) + } + target = resolved + } + } + + if ip := net.ParseIP(target); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + connectReq = append(connectReq, 0x01) // IPv4 + connectReq = append(connectReq, ip4...) + } else { + connectReq = append(connectReq, 0x04) // IPv6 + connectReq = append(connectReq, ip.To16()...) + } + } else { + connectReq = append(connectReq, 0x03) // Domain name + connectReq = append(connectReq, byte(len(target))) + connectReq = append(connectReq, []byte(target)...) + } + + connectReq = append(connectReq, byte(port>>8), byte(port)) + + if _, err := conn.Write(connectReq); err != nil { + return nil, fmt.Errorf("socks5 connect request failed: %w", err) + } + + // Read CONNECT response + resp6 := make([]byte, 4) + if _, err := io.ReadFull(conn, resp6); err != nil { + return nil, fmt.Errorf("socks5 connect response failed: %w", err) + } + + if resp6[0] != 0x05 || resp6[1] != 0x00 { + return nil, fmt.Errorf("socks5 connection rejected: code=%d", resp6[1]) + } + + // Read remaining address from response (bnd.addr + bnd.port) + addrType := resp6[3] + switch addrType { + case 0x01: // IPv4 + _, err = io.ReadFull(conn, make([]byte, 4+2)) + case 0x03: // Domain name + var nameLen [1]byte + _, err = io.ReadFull(conn, nameLen[:]) + if err == nil { + _, err = io.ReadFull(conn, make([]byte, int(nameLen[0])+2)) + } + case 0x04: // IPv6 + _, err = io.ReadFull(conn, make([]byte, 16+2)) + } + if err != nil { + return nil, fmt.Errorf("socks5: failed to read bind address: %w", err) + } + + return conn, nil +} + +// resolveSOCKS5Hostname resolves a hostname to a single IP literal. It uses +// the system resolver — the SOCKS5 proxy is the bottleneck for this path, so +// IPv6-first dialer integration is not applied here. +func resolveSOCKS5Hostname(ctx context.Context, host string) (string, error) { + ips, err := (&net.Resolver{}).LookupIPAddr(ctx, host) + if err != nil { + return "", err + } + if len(ips) == 0 { + return "", fmt.Errorf("no addresses for %s", host) + } + return ips[0].IP.String(), nil +} + +// GetProxyFromEnv gets proxy from environment +func GetProxyFromEnv() string { + // Order: HTTPS_PROXY > HTTP_PROXY > http_proxy + for _, env := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"} { + if proxy := strings.TrimSpace(os.Getenv(env)); proxy != "" { + return proxy + } + } + return "" +} diff --git a/internal/transport/retry.go b/internal/transport/retry.go new file mode 100644 index 0000000..2d0c0c8 --- /dev/null +++ b/internal/transport/retry.go @@ -0,0 +1,153 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "context" + "fmt" + "strings" + "time" +) + +// RetryConfig configures the retry logic +type RetryConfig struct { + // Maximum number of attempts + MaxRetries int + + // Initial delay between attempts + InitialDelay time.Duration + + // Maximum delay between attempts + MaxDelay time.Duration + + // Multiplier for exponential backoff + Multiplier float64 + + // Retry on these HTTP status codes + RetryableStatusCodes []int + + // Callback before each retry + OnRetry func(attempt int, err error, delay time.Duration) +} + +// DefaultRetryConfig returns the default configuration +func DefaultRetryConfig() *RetryConfig { + return &RetryConfig{ + MaxRetries: 3, + InitialDelay: 1 * time.Second, + MaxDelay: 30 * time.Second, + Multiplier: 2.0, + RetryableStatusCodes: []int{408, 429, 500, 502, 503, 504}, + } +} + +// RetryFunc is a function that can be retried +type RetryFunc func(ctx context.Context, attempt int) error + +// Retry performs an operation with retry logic +func Retry(ctx context.Context, fn RetryFunc, cfg *RetryConfig) error { + if cfg == nil { + cfg = DefaultRetryConfig() + } + + var lastErr error + delay := cfg.InitialDelay + + for attempt := 1; attempt <= cfg.MaxRetries+1; attempt++ { + // Check the context + if ctx.Err() != nil { + return ctx.Err() + } + + // Perform the operation + err := fn(ctx, attempt) + if err == nil { + return nil + } + + lastErr = err + + // Last attempt - return error + if attempt > cfg.MaxRetries { + return fmt.Errorf("all %d attempts failed: %w", cfg.MaxRetries+1, err) + } + + // Callback before retry + if cfg.OnRetry != nil { + cfg.OnRetry(attempt, err, delay) + } + + // Wait before the next attempt + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // Exponential backoff + delay = time.Duration(float64(delay) * cfg.Multiplier) + if delay > cfg.MaxDelay { + delay = cfg.MaxDelay + } + } + } + + return lastErr +} + +// IsRetryableError determines whether the error is retryable +func IsRetryableError(err error) bool { + if err == nil { + return false + } + + errStr := err.Error() + + // Network errors + retryableErrors := []string{ + "connection refused", + "connection reset", + "connection timed out", + "i/o timeout", + "temporary failure", + "no route to host", + } + + for _, re := range retryableErrors { + if containsIgnoreCase(errStr, re) { + return true + } + } + + return false +} + +// IsRetryableStatusCode determines whether the HTTP status is retryable +func IsRetryableStatusCode(status int, cfg *RetryConfig) bool { + if cfg == nil { + cfg = DefaultRetryConfig() + } + + for _, code := range cfg.RetryableStatusCodes { + if status == code { + return true + } + } + + return false +} + +// WithMaxRetries creates a RetryConfig with the specified maximum retries +func WithMaxRetries(maxRetries int) *RetryConfig { + cfg := DefaultRetryConfig() + if maxRetries > 0 { + cfg.MaxRetries = maxRetries + } + return cfg +} + +// containsIgnoreCase helper function +func containsIgnoreCase(s, substr string) bool { + s = strings.ToLower(s) + substr = strings.ToLower(substr) + return strings.Contains(s, substr) +} diff --git a/internal/transport/tls.go b/internal/transport/tls.go new file mode 100644 index 0000000..9391a74 --- /dev/null +++ b/internal/transport/tls.go @@ -0,0 +1,146 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "fmt" + "os" + "strings" +) + +// TLSConfig configures TLS +type TLSConfig struct { + // Minimum TLS version + MinVersion uint16 + + // Maximum TLS version + MaxVersion uint16 + + // Path to CA certificates + CACertFile string + + // Skip certificate verification + InsecureSkipVerify bool + + // Client certificate for mTLS + CertFile string + + // Client private key for mTLS + KeyFile string + + // Certificate pinning (SHA-256 hash of certificate) + PinnedCertHash string + + // TLSServerNameOverride forces the SNI server name and the hostname used + // for certificate verification. Leave empty (the default) so that Go's + // stdlib derives the server name from the dial target, which is the safe + // behavior. Setting this to a value that does not match the host you are + // connecting to enables a man-in-the-middle attacker with a valid cert + // for the overridden name to impersonate the target. Only set this when + // you genuinely need to override SNI (e.g. connecting to an IP address + // whose certificate carries a different hostname). + TLSServerNameOverride string + + // Path to SSLKEYLOGFILE for Wireshark debugging + KeyLogFile string +} + +// DefaultTLSConfig returns a safe default configuration +func DefaultTLSConfig() *TLSConfig { + return &TLSConfig{ + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + InsecureSkipVerify: false, + } +} + +// ToTLSConfig converts to tls.Config +func (c *TLSConfig) ToTLSConfig() (*tls.Config, error) { + tlsCfg := &tls.Config{ + MinVersion: c.MinVersion, + MaxVersion: c.MaxVersion, + InsecureSkipVerify: c.InsecureSkipVerify, + } + + // Only apply the override if explicitly set. When empty, Go's net/http + // transport derives ServerName from the dial address, which matches the + // certificate the server is expected to present. + if c.TLSServerNameOverride != "" { + tlsCfg.ServerName = c.TLSServerNameOverride + } + + // Load custom CA certificates + if c.CACertFile != "" { + caCert, err := os.ReadFile(c.CACertFile) + if err != nil { + return nil, fmt.Errorf("failed to read ca cert: %w", err) + } + + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse ca cert") + } + + tlsCfg.RootCAs = caCertPool + } + + // Certificate pinning: verify SHA-256 hash of server certificate + if c.PinnedCertHash != "" { + expectedHash := strings.ToLower(strings.TrimSpace(c.PinnedCertHash)) + tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + for _, rawCert := range rawCerts { + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + continue + } + hash := sha256.Sum256(cert.Raw) + actual := hex.EncodeToString(hash[:]) + if actual == expectedHash { + return nil + } + } + return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expectedHash) + } + } + + // Load client certificate for mTLS + if c.CertFile != "" && c.KeyFile != "" { + cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate: %w", err) + } + tlsCfg.Certificates = []tls.Certificate{cert} + } + + // SSLKEYLOGFILE for Wireshark/TLS debugging + if c.KeyLogFile != "" { + f, err := os.OpenFile(c.KeyLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, fmt.Errorf("failed to open keylog file: %w", err) + } + tlsCfg.KeyLogWriter = f + } + + return tlsCfg, nil +} + +// GetTLSVersionName returns a human-readable TLS version name +func GetTLSVersionName(version uint16) string { + switch version { + case tls.VersionTLS10: + return "TLS 1.0" + case tls.VersionTLS11: + return "TLS 1.1" + case tls.VersionTLS12: + return "TLS 1.2" + case tls.VersionTLS13: + return "TLS 1.3" + default: + return fmt.Sprintf("TLS 0x%04X", version) + } +} diff --git a/internal/transport/token_bucket.go b/internal/transport/token_bucket.go new file mode 100644 index 0000000..df54d3d --- /dev/null +++ b/internal/transport/token_bucket.go @@ -0,0 +1,83 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "io" + "sync" + "time" +) + +// TokenBucket implements the token bucket algorithm +type TokenBucket struct { + mu sync.Mutex + capacity int64 + tokens float64 + refillRate float64 + lastRefill time.Time +} + +// NewTokenBucket creates new rate limiter +func NewTokenBucket(maxBytesPerSec int64, burstSize int64) *TokenBucket { + if maxBytesPerSec <= 0 { + return nil + } + return &TokenBucket{ + capacity: burstSize, + tokens: float64(burstSize), + refillRate: float64(maxBytesPerSec), + lastRefill: time.Now(), + } +} + +// Allow waits for enough tokens +func (tb *TokenBucket) Allow(n int64) { + if tb == nil { + return + } + tb.mu.Lock() + defer tb.mu.Unlock() + + now := time.Now() + elapsed := now.Sub(tb.lastRefill).Seconds() + tb.tokens += elapsed * tb.refillRate + if tb.tokens > float64(tb.capacity) { + tb.tokens = float64(tb.capacity) + } + tb.lastRefill = now + + for tb.tokens < float64(n) { + needed := float64(n) - tb.tokens + waitTime := time.Duration(needed/tb.refillRate*1e9) * time.Nanosecond + tb.mu.Unlock() + time.Sleep(waitTime) + tb.mu.Lock() + now := time.Now() + elapsed := now.Sub(tb.lastRefill).Seconds() + tb.tokens += elapsed * tb.refillRate + if tb.tokens > float64(tb.capacity) { + tb.tokens = float64(tb.capacity) + } + tb.lastRefill = now + } + tb.tokens -= float64(n) +} + +// WrapReader returns a reader with rate limiting +func (tb *TokenBucket) WrapReader(r io.Reader) io.Reader { + if tb == nil { + return r + } + return &limitedReader{r: r, limiter: tb} +} + +type limitedReader struct { + r io.Reader + limiter *TokenBucket +} + +func (lr *limitedReader) Read(p []byte) (int, error) { + lr.limiter.Allow(int64(len(p))) + return lr.r.Read(p) +} diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go new file mode 100644 index 0000000..142dd2e --- /dev/null +++ b/internal/transport/transport_test.go @@ -0,0 +1,2026 @@ +//go:build linux || freebsd +// +build linux freebsd + +package transport + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// Dialer tests +// --------------------------------------------------------------------------- + +func TestDefaultDialerConfig(t *testing.T) { + t.Parallel() + + cfg := DefaultDialerConfig() + + if cfg.Timeout != 30*time.Second { + t.Errorf("expected Timeout=30s, got %v", cfg.Timeout) + } + if cfg.IPv6Timeout != 5*time.Second { + t.Errorf("expected IPv6Timeout=5s, got %v", cfg.IPv6Timeout) + } + if !cfg.IPv4Fallback { + t.Error("expected IPv4Fallback=true") + } + if cfg.Debug { + t.Error("expected Debug=false") + } + if cfg.OnConnect != nil { + t.Error("expected OnConnect=nil") + } +} + +func TestNewDialer(t *testing.T) { + t.Parallel() + + t.Run("with nil config uses defaults", func(t *testing.T) { + d := NewDialer(nil) + if d == nil { + t.Fatal("expected non-nil dialer") + } + if d.config == nil { + t.Fatal("expected non-nil config") + } + if d.config.Timeout != 30*time.Second { + t.Errorf("expected default timeout, got %v", d.config.Timeout) + } + }) + + t.Run("with custom config", func(t *testing.T) { + cfg := &DialerConfig{ + Timeout: 10 * time.Second, + IPv6Timeout: 2 * time.Second, + IPv4Fallback: false, + Debug: true, + } + d := NewDialer(cfg) + if d == nil { + t.Fatal("expected non-nil dialer") + } + if d.config.Timeout != 10*time.Second { + t.Errorf("expected 10s, got %v", d.config.Timeout) + } + if d.config.IPv6Timeout != 2*time.Second { + t.Errorf("expected 2s, got %v", d.config.IPv6Timeout) + } + if d.config.IPv4Fallback { + t.Error("expected IPv4Fallback=false") + } + if !d.config.Debug { + t.Error("expected Debug=true") + } + }) + + t.Run("resolver is initialized", func(t *testing.T) { + d := NewDialer(DefaultDialerConfig()) + if d.resolver == nil { + t.Fatal("expected resolver to be initialized") + } + }) +} + +func TestNewDialerPreservesOnConnectCallback(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + called bool + gotIP string + gotVer int + ) + + cfg := &DialerConfig{ + Timeout: 5 * time.Second, + IPv6Timeout: 1 * time.Second, + IPv4Fallback: true, + OnConnect: func(ip string, version int) { + mu.Lock() + called = true + gotIP = ip + gotVer = version + mu.Unlock() + }, + } + + d := NewDialer(cfg) + if d == nil { + t.Fatal("expected non-nil dialer") + } + + // Verify the callback is stored (actual connection is not tested here) + if d.config.OnConnect == nil { + t.Fatal("expected OnConnect callback to be preserved") + } + + // Call it manually to verify it works + d.config.OnConnect("192.0.2.1", 4) + + mu.Lock() + if !called { + t.Fatal("expected OnConnect to have been called") + } + if gotIP != "192.0.2.1" { + t.Errorf("expected IP 192.0.2.1, got %s", gotIP) + } + if gotVer != 4 { + t.Errorf("expected version 4, got %d", gotVer) + } + mu.Unlock() +} + +func TestParseProxyURL(t *testing.T) { + t.Parallel() + + t.Run("empty string returns nil, nil", func(t *testing.T) { + u, err := ParseProxyURL("") + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + if u != nil { + t.Errorf("expected nil URL, got %v", u) + } + }) + + t.Run("valid HTTP proxy URL", func(t *testing.T) { + u, err := ParseProxyURL("http://proxy.example.com:8080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if u == nil { + t.Fatal("expected non-nil URL") + } + if u.Scheme != "http" { + t.Errorf("expected scheme http, got %s", u.Scheme) + } + if u.Host != "proxy.example.com:8080" { + t.Errorf("expected host proxy.example.com:8080, got %s", u.Host) + } + }) + + t.Run("valid HTTPS proxy URL with auth", func(t *testing.T) { + u, err := ParseProxyURL("https://user:pass@proxy.example.com:443") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if u == nil { + t.Fatal("expected non-nil URL") + } + if u.Scheme != "https" { + t.Errorf("expected scheme https, got %s", u.Scheme) + } + if u.User == nil { + t.Fatal("expected user info") + } + password, _ := u.User.Password() + if u.User.Username() != "user" || password != "pass" { + t.Errorf("expected user:pass, got %s:%s", u.User.Username(), password) + } + }) + + t.Run("invalid URL returns error", func(t *testing.T) { + _, err := ParseProxyURL("://invalid") + if err == nil { + t.Error("expected error for invalid URL") + } + }) + + t.Run("URL without host returns error", func(t *testing.T) { + _, err := ParseProxyURL("http://") + if err == nil { + t.Error("expected error for URL without host") + } + }) + + t.Run("socks5 proxy URL", func(t *testing.T) { + u, err := ParseProxyURL("socks5://127.0.0.1:1080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if u == nil { + t.Fatal("expected non-nil URL") + } + if u.Scheme != "socks5" { + t.Errorf("expected scheme socks5, got %s", u.Scheme) + } + if u.Host != "127.0.0.1:1080" { + t.Errorf("expected host 127.0.0.1:1080, got %s", u.Host) + } + }) +} + +// --------------------------------------------------------------------------- +// Proxy tests +// --------------------------------------------------------------------------- + +func TestParseProxyConfig(t *testing.T) { + t.Parallel() + + t.Run("empty string returns nil, nil", func(t *testing.T) { + cfg, err := ParseProxyConfig("") + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + if cfg != nil { + t.Errorf("expected nil config, got %+v", cfg) + } + }) + + t.Run("HTTP proxy URL", func(t *testing.T) { + cfg, err := ParseProxyConfig("http://proxy.example.com:3128") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Type != ProxyHTTP { + t.Errorf("expected ProxyHTTP, got %s", cfg.Type) + } + if cfg.URL.Host != "proxy.example.com:3128" { + t.Errorf("expected host proxy.example.com:3128, got %s", cfg.URL.Host) + } + }) + + t.Run("HTTPS proxy URL maps to ProxyHTTP type", func(t *testing.T) { + cfg, err := ParseProxyConfig("https://secure-proxy.example.com:443") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Type != ProxyHTTP { + t.Errorf("expected ProxyHTTP for https scheme, got %s", cfg.Type) + } + }) + + t.Run("SOCKS5 proxy URL", func(t *testing.T) { + cfg, err := ParseProxyConfig("socks5://127.0.0.1:1080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Type != ProxySOCKS5 { + t.Errorf("expected ProxySOCKS5, got %s", cfg.Type) + } + if !cfg.SOCKS5LocalResolve { + t.Error("expected SOCKS5LocalResolve=true for socks5:// scheme") + } + }) + + t.Run("SOCKS5h proxy URL", func(t *testing.T) { + cfg, err := ParseProxyConfig("socks5h://127.0.0.1:1080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Type != ProxySOCKS5 { + t.Errorf("expected ProxySOCKS5 for socks5h scheme, got %s", cfg.Type) + } + if cfg.SOCKS5LocalResolve { + t.Error("expected SOCKS5LocalResolve=false for socks5h:// scheme") + } + }) + + t.Run("unknown scheme defaults to ProxyHTTP", func(t *testing.T) { + cfg, err := ParseProxyConfig("ftp://proxy.example.com:21") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Type != ProxyHTTP { + t.Errorf("expected ProxyHTTP for unknown scheme, got %s", cfg.Type) + } + }) + + t.Run("with authentication", func(t *testing.T) { + cfg, err := ParseProxyConfig("http://alice:secret@proxy.example.com:8080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Username != "alice" { + t.Errorf("expected username alice, got %s", cfg.Username) + } + if cfg.Password != "secret" { + t.Errorf("expected password secret, got %s", cfg.Password) + } + // Auth should be stripped from URL + if cfg.URL.User != nil { + t.Error("expected User to be nil in stored URL") + } + }) + + t.Run("with username only (no password)", func(t *testing.T) { + cfg, err := ParseProxyConfig("http://alice@proxy.example.com:8080") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Username != "alice" { + t.Errorf("expected username alice, got %s", cfg.Username) + } + if cfg.Password != "" { + t.Errorf("expected empty password, got %s", cfg.Password) + } + }) + + t.Run("invalid URL returns error", func(t *testing.T) { + _, err := ParseProxyConfig("://bad") + if err == nil { + t.Error("expected error for invalid URL") + } + }) +} + +func TestGetProxyFromEnv(t *testing.T) { + // Start with a clean environment + origHTTPS := os.Getenv("HTTPS_PROXY") + origHTTP := os.Getenv("HTTP_PROXY") + origHTTPLower := os.Getenv("http_proxy") + origHTTPSSecure := os.Getenv("https_proxy") + + defer func() { + os.Setenv("HTTPS_PROXY", origHTTPS) + os.Setenv("HTTP_PROXY", origHTTP) + os.Setenv("http_proxy", origHTTPLower) + os.Setenv("https_proxy", origHTTPSSecure) + }() + + t.Run("no proxy env vars returns empty", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + result := GetProxyFromEnv() + if result != "" { + t.Errorf("expected empty string, got %s", result) + } + }) + + t.Run("reads HTTPS_PROXY first", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + os.Setenv("HTTPS_PROXY", "https://secure-proxy:443") + os.Setenv("HTTP_PROXY", "http://proxy:8080") + + result := GetProxyFromEnv() + if result != "https://secure-proxy:443" { + t.Errorf("expected https://secure-proxy:443, got %s", result) + } + }) + + t.Run("reads HTTP_PROXY when HTTPS_PROXY is empty", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + os.Setenv("HTTP_PROXY", "http://proxy:8080") + + result := GetProxyFromEnv() + if result != "http://proxy:8080" { + t.Errorf("expected http://proxy:8080, got %s", result) + } + }) + + t.Run("reads http_proxy (lowercase) when others are empty", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + os.Setenv("http_proxy", "http://lower-proxy:3128") + + result := GetProxyFromEnv() + if result != "http://lower-proxy:3128" { + t.Errorf("expected http://lower-proxy:3128, got %s", result) + } + }) + + t.Run("trims whitespace from proxy value", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + os.Setenv("HTTP_PROXY", " http://proxy:8080 ") + + result := GetProxyFromEnv() + if result != "http://proxy:8080" { + t.Errorf("expected http://proxy:8080 (trimmed), got %q", result) + } + }) + + t.Run("priority: HTTPS_PROXY > https_proxy > HTTP_PROXY > http_proxy", func(t *testing.T) { + os.Unsetenv("HTTPS_PROXY") + os.Unsetenv("https_proxy") + os.Unsetenv("HTTP_PROXY") + os.Unsetenv("http_proxy") + + os.Setenv("https_proxy", "https://lower-secure:443") + os.Setenv("HTTP_PROXY", "http://proxy:8080") + + result := GetProxyFromEnv() + if result != "https://lower-secure:443" { + t.Errorf("expected https://lower-secure:443, got %s", result) + } + }) +} + +func TestNewProxyTransport(t *testing.T) { + t.Parallel() + + t.Run("nil config returns base transport unchanged", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + result, err := NewProxyTransport(nil, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result != base { + t.Error("expected same transport back") + } + }) + + t.Run("sets HTTP proxy", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "proxy.example.com:8080"}, + Type: ProxyHTTP, + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == nil { + t.Fatal("expected non-nil transport") + } + + // Verify proxy function works + req := &http.Request{ + URL: &url.URL{Scheme: "http", Host: "example.com"}, + } + proxyURL, err := result.Proxy(req) + if err != nil { + t.Fatalf("proxy function error: %v", err) + } + if proxyURL == nil { + t.Fatal("expected non-nil proxy URL") + } + if proxyURL.Host != "proxy.example.com:8080" { + t.Errorf("expected proxy host proxy.example.com:8080, got %s", proxyURL.Host) + } + }) + + t.Run("sets proxy with authentication", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "proxy.example.com:8080"}, + Type: ProxyHTTP, + Username: "user", + Password: "pass", + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == nil { + t.Fatal("expected non-nil transport") + } + + req := &http.Request{ + URL: &url.URL{Scheme: "http", Host: "example.com"}, + } + proxyURL, err := result.Proxy(req) + if err != nil { + t.Fatalf("proxy function error: %v", err) + } + if proxyURL == nil { + t.Fatal("expected non-nil proxy URL") + } + if proxyURL.User == nil { + t.Fatal("expected user info in proxy URL") + } + password, _ := proxyURL.User.Password() + if proxyURL.User.Username() != "user" || password != "pass" { + t.Errorf("expected user:pass, got %s:%s", proxyURL.User.Username(), password) + } + }) + + t.Run("sets proxy with username only (no password)", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "proxy.example.com:8080"}, + Type: ProxyHTTP, + Username: "alice", + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == nil { + t.Fatal("expected non-nil transport") + } + + req := &http.Request{ + URL: &url.URL{Scheme: "http", Host: "example.com"}, + } + proxyURL, err := result.Proxy(req) + if err != nil { + t.Fatalf("proxy function error: %v", err) + } + if proxyURL == nil { + t.Fatal("expected non-nil proxy URL") + } + if proxyURL.User == nil { + t.Fatal("expected user info in proxy URL") + } + if proxyURL.User.Username() != "alice" { + t.Errorf("expected username alice, got %s", proxyURL.User.Username()) + } + }) + + t.Run("proxy URL parse failure returns error", func(t *testing.T) { + // This is hard to trigger with normal inputs; skip edge case. + // The url.Parse function is very lenient, so this test is for completeness. + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "proxy:8080"}, + Type: ProxyHTTP, + Username: "user\x00", // null byte may cause issues + } + base := http.DefaultTransport.(*http.Transport).Clone() + _, err := NewProxyTransport(cfg, base) + // We allow error or no error here; just make sure it doesn't panic + _ = err + }) + + t.Run("HTTP proxy preserves TLSClientConfig", func(t *testing.T) { + // Regression guard for the BACKLOG entry "TLS config lost through + // SOCKS5 proxy transport". The original concern was that + // NewProxyTransport might construct a fresh transport without + // copying the base's TLSClientConfig, silently dropping + // InsecureSkipVerify, custom CAs, and pinned certs. Verify that + // is *not* the case for HTTP proxy too. + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // test-only + MinVersion: tls.VersionTLS12, + } + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "proxy.example.com:8080"}, + Type: ProxyHTTP, + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.TLSClientConfig == nil { + t.Fatal("TLSClientConfig was dropped from the resulting transport") + } + if !result.TLSClientConfig.InsecureSkipVerify { + t.Error("InsecureSkipVerify was dropped from TLSClientConfig") + } + if result.TLSClientConfig.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %d, want %d", result.TLSClientConfig.MinVersion, tls.VersionTLS12) + } + }) +} + +func TestProxyTypes(t *testing.T) { + t.Parallel() + + if string(ProxyHTTP) != "http" { + t.Errorf("expected ProxyHTTP to be 'http', got %q", string(ProxyHTTP)) + } + if string(ProxyHTTPS) != "https" { + t.Errorf("expected ProxyHTTPS to be 'https', got %q", string(ProxyHTTPS)) + } + if string(ProxySOCKS5) != "socks5" { + t.Errorf("expected ProxySOCKS5 to be 'socks5', got %q", string(ProxySOCKS5)) + } +} + +func TestDialProxy(t *testing.T) { + t.Parallel() + + t.Run("nil config uses direct dialer", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + // This will fail quickly because there's no server, but it should + // attempt a direct connection rather than returning an unsupported error. + _, err := DialProxy(ctx, "tcp", "127.0.0.1:1", nil) + // Should NOT get "unsupported proxy type" + if err != nil && strings.Contains(err.Error(), "unsupported proxy type") { + t.Errorf("unexpected 'unsupported proxy type' error: %v", err) + } + }) + + t.Run("unsupported network returns error", func(t *testing.T) { + ctx := context.Background() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}, + Type: ProxyHTTP, + } + _, err := DialProxy(ctx, "udp", "127.0.0.1:80", cfg) + if err == nil { + t.Error("expected error for unsupported network") + } + }) +} + +// --------------------------------------------------------------------------- +// SOCKS5 tests +// --------------------------------------------------------------------------- + +// mockSOCKS5Server accepts a single SOCKS5 connection, performs (or skips) +// the auth negotiation, then verifies the CONNECT request and replies with +// success. The connection is then handed off to the caller via the returned +// channel. +type mockSOCKS5Result struct { + host string + port uint16 + addrType byte + gotUsername string + gotPassword string + requestedAuth byte + conn net.Conn +} + +func startMockSOCKS5(t *testing.T, requireAuth bool) (addr string, results <-chan mockSOCKS5Result, stop func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + out := make(chan mockSOCKS5Result, 1) + done := make(chan struct{}) + go func() { + defer close(done) + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + res := mockSOCKS5Result{conn: conn} + + // Greeting: read VER + NMETHODS, then NMETHODS method bytes. + // The client offers at least no-auth (0x00), and additionally + // user/pass (0x02) when a username is configured. + greet := make([]byte, 2) + if _, err := io.ReadFull(conn, greet); err != nil { + return + } + if greet[0] != 0x05 || int(greet[1]) > 4 { + return + } + methods := make([]byte, greet[1]) + if _, err := io.ReadFull(conn, methods); err != nil { + return + } + + // Pick the strongest method we support. requireAuth means we + // require user/pass; otherwise we accept no-auth if offered. + offered := make(map[byte]bool, len(methods)) + for _, m := range methods { + offered[m] = true + } + if requireAuth { + if !offered[0x02] { + conn.Write([]byte{0x05, 0xFF}) // no acceptable method + return + } + conn.Write([]byte{0x05, 0x02}) + res.requestedAuth = 0x02 + } else { + if !offered[0x00] { + conn.Write([]byte{0x05, 0xFF}) + return + } + conn.Write([]byte{0x05, 0x00}) + res.requestedAuth = 0x00 + } + + // User/pass sub-negotiation (only when 0x02 was selected). + if res.requestedAuth == 0x02 { + ver := make([]byte, 1) + if _, err := io.ReadFull(conn, ver); err != nil { + return + } + if ver[0] != 0x01 { + return + } + ulen := make([]byte, 1) + if _, err := io.ReadFull(conn, ulen); err != nil { + return + } + uname := make([]byte, ulen[0]) + if _, err := io.ReadFull(conn, uname); err != nil { + return + } + plen := make([]byte, 1) + if _, err := io.ReadFull(conn, plen); err != nil { + return + } + pass := make([]byte, plen[0]) + if _, err := io.ReadFull(conn, pass); err != nil { + return + } + res.gotUsername = string(uname) + res.gotPassword = string(pass) + conn.Write([]byte{0x01, 0x00}) + } + + // CONNECT request: VER, CMD, RSV, ATYP + hdr := make([]byte, 4) + if _, err := io.ReadFull(conn, hdr); err != nil { + return + } + if hdr[0] != 0x05 || hdr[1] != 0x01 { + return + } + res.addrType = hdr[3] + switch hdr[3] { + case 0x01: + ip := make([]byte, 4) + if _, err := io.ReadFull(conn, ip); err != nil { + return + } + res.host = net.IP(ip).String() + case 0x04: + ip := make([]byte, 16) + if _, err := io.ReadFull(conn, ip); err != nil { + return + } + res.host = net.IP(ip).String() + case 0x03: + l := make([]byte, 1) + if _, err := io.ReadFull(conn, l); err != nil { + return + } + name := make([]byte, l[0]) + if _, err := io.ReadFull(conn, name); err != nil { + return + } + res.host = string(name) + } + portBuf := make([]byte, 2) + if _, err := io.ReadFull(conn, portBuf); err != nil { + return + } + res.port = uint16(portBuf[0])<<8 | uint16(portBuf[1]) + + // Reply success with 0.0.0.0:0 bind address. + conn.Write([]byte{0x05, 0x00, 0x00, 0x01, 0, 0, 0, 0, 0, 0}) + out <- res + // Keep the connection open until the test closes it. + // Block here so the conn is not garbage-collected. + buf := make([]byte, 1) + conn.Read(buf) + }() + stop = func() { + ln.Close() + <-done + } + return ln.Addr().String(), out, stop +} + +func TestNewProxyTransport_SOCKS5(t *testing.T) { + t.Parallel() + + t.Run("SOCKS5 sets DialContext and clears Proxy field", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: "127.0.0.1:1080"}, + Type: ProxySOCKS5, + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == nil { + t.Fatal("expected non-nil transport") + } + if result.Proxy != nil { + t.Errorf("expected Proxy=nil for SOCKS5 (handled via DialContext), got %T", result.Proxy) + } + if result.DialContext == nil { + t.Error("expected DialContext to be set for SOCKS5") + } + }) + + t.Run("SOCKS5 with auth", func(t *testing.T) { + base := http.DefaultTransport.(*http.Transport).Clone() + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: "127.0.0.1:1080"}, + Type: ProxySOCKS5, + Username: "alice", + Password: "secret", + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.DialContext == nil { + t.Error("expected DialContext to be set for SOCKS5 with auth") + } + }) + + t.Run("SOCKS5 proxy preserves TLSClientConfig", func(t *testing.T) { + // Regression guard for the BACKLOG entry "TLS config lost through + // SOCKS5 proxy transport". After Clone(), the new transport + // must still carry the caller's TLSClientConfig (with + // InsecureSkipVerify, custom CAs, pinned certs, etc.). + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // test-only + MinVersion: tls.VersionTLS12, + } + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: "127.0.0.1:1080"}, + Type: ProxySOCKS5, + } + result, err := NewProxyTransport(cfg, base) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.TLSClientConfig == nil { + t.Fatal("TLSClientConfig was dropped from the SOCKS5 transport") + } + if !result.TLSClientConfig.InsecureSkipVerify { + t.Error("InsecureSkipVerify was dropped from TLSClientConfig") + } + if result.TLSClientConfig.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %d, want %d", result.TLSClientConfig.MinVersion, tls.VersionTLS12) + } + }) +} + +func TestDialProxy_SOCKS5NoAuth(t *testing.T) { + t.Parallel() + + addr, results, stop := startMockSOCKS5(t, false) + defer stop() + + host, port, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("split addr: %v", err) + } + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: addr}, + Type: ProxySOCKS5, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, err := DialProxy(ctx, "tcp", net.JoinHostPort("192.0.2.1", "8080"), cfg) + if err != nil { + t.Fatalf("DialProxy failed: %v", err) + } + defer conn.Close() + + select { + case res := <-results: + if res.addrType != 0x01 { + t.Errorf("expected IPv4 ATYP=0x01, got 0x%02x", res.addrType) + } + if res.host != "192.0.2.1" { + t.Errorf("expected host 192.0.2.1, got %s", res.host) + } + if res.port != 8080 { + t.Errorf("expected port 8080, got %d", res.port) + } + if res.requestedAuth != 0x00 { + t.Errorf("expected no-auth selection, got 0x%02x", res.requestedAuth) + } + _ = host + _ = port + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for SOCKS5 server to receive request") + } +} + +func TestDialProxy_SOCKS5WithAuth(t *testing.T) { + t.Parallel() + + addr, results, stop := startMockSOCKS5(t, true) + defer stop() + + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: addr}, + Type: ProxySOCKS5, + Username: "alice", + Password: "s3cret", + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, err := DialProxy(ctx, "tcp", "192.0.2.1:8443", cfg) + if err != nil { + t.Fatalf("DialProxy failed: %v", err) + } + defer conn.Close() + + select { + case res := <-results: + if res.requestedAuth != 0x02 { + t.Errorf("expected user/pass auth selection, got 0x%02x", res.requestedAuth) + } + if res.gotUsername != "alice" { + t.Errorf("expected username alice, got %s", res.gotUsername) + } + if res.gotPassword != "s3cret" { + t.Errorf("expected password s3cret, got %s", res.gotPassword) + } + if res.port != 8443 { + t.Errorf("expected port 8443, got %d", res.port) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for SOCKS5 server to receive request") + } +} + +func TestDialProxy_SOCKS5HostnameRemoteResolve(t *testing.T) { + t.Parallel() + + addr, results, stop := startMockSOCKS5(t, false) + defer stop() + + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5h", Host: addr}, + Type: ProxySOCKS5, + SOCKS5LocalResolve: false, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, err := DialProxy(ctx, "tcp", "example.com:443", cfg) + if err != nil { + t.Fatalf("DialProxy failed: %v", err) + } + defer conn.Close() + + select { + case res := <-results: + if res.addrType != 0x03 { + t.Errorf("expected domain ATYP=0x03, got 0x%02x", res.addrType) + } + if res.host != "example.com" { + t.Errorf("expected host example.com, got %s", res.host) + } + if res.port != 443 { + t.Errorf("expected port 443, got %d", res.port) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for SOCKS5 server to receive request") + } +} + +func TestDialProxy_SOCKS5LocalResolve(t *testing.T) { + t.Parallel() + + addr, results, stop := startMockSOCKS5(t, false) + defer stop() + + cfg := &ProxyConfig{ + URL: &url.URL{Scheme: "socks5", Host: addr}, + Type: ProxySOCKS5, + SOCKS5LocalResolve: true, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Use a hostname that always resolves to a real IP. localhost works. + conn, err := DialProxy(ctx, "tcp", "localhost:8080", cfg) + if err != nil { + t.Fatalf("DialProxy failed: %v", err) + } + defer conn.Close() + + select { + case res := <-results: + // The hostname is resolved locally, so the SOCKS5 server must + // receive an IP literal — never a bare hostname like "localhost". + // The exact address family depends on the host resolver: most + // systems return 127.0.0.1, but IPv6-preferring environments + // (some CI runners) return ::1 instead. Accept either as long + // as it is a loopback IP and the ATYP field matches. + if res.host == "localhost" { + t.Fatal("expected hostname to be resolved to IP literal, got hostname") + } + ip := net.ParseIP(res.host) + if ip == nil { + t.Fatalf("expected IP literal after local resolve, got %q", res.host) + } + if !ip.IsLoopback() { + t.Errorf("expected loopback IP, got %s", res.host) + } + + wantATYP := byte(0x01) // IPv4 + if ip.To4() == nil { + wantATYP = 0x04 // IPv6 + } + if res.addrType != wantATYP { + t.Errorf("ATYP = 0x%02x, want 0x%02x for host %s", res.addrType, wantATYP, res.host) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for SOCKS5 server to receive request") + } +} + +// --------------------------------------------------------------------------- +// Retry tests +// --------------------------------------------------------------------------- + +func TestDefaultRetryConfig(t *testing.T) { + t.Parallel() + + cfg := DefaultRetryConfig() + + if cfg.MaxRetries != 3 { + t.Errorf("expected MaxRetries=3, got %d", cfg.MaxRetries) + } + if cfg.InitialDelay != 1*time.Second { + t.Errorf("expected InitialDelay=1s, got %v", cfg.InitialDelay) + } + if cfg.MaxDelay != 30*time.Second { + t.Errorf("expected MaxDelay=30s, got %v", cfg.MaxDelay) + } + if cfg.Multiplier != 2.0 { + t.Errorf("expected Multiplier=2.0, got %f", cfg.Multiplier) + } + if len(cfg.RetryableStatusCodes) != 6 { + t.Errorf("expected 6 retryable status codes, got %d", len(cfg.RetryableStatusCodes)) + } + if cfg.OnRetry != nil { + t.Error("expected OnRetry=nil") + } + + // Verify specific status codes + expectedCodes := []int{408, 429, 500, 502, 503, 504} + for i, code := range expectedCodes { + if cfg.RetryableStatusCodes[i] != code { + t.Errorf("expected status code %d at index %d, got %d", code, i, cfg.RetryableStatusCodes[i]) + } + } +} + +func TestRetrySuccess(t *testing.T) { + t.Parallel() + + t.Run("succeeds on first attempt", func(t *testing.T) { + ctx := context.Background() + attempts := 0 + err := Retry(ctx, func(ctx context.Context, attempt int) error { + attempts++ + return nil + }, &RetryConfig{ + MaxRetries: 3, + InitialDelay: 10 * time.Millisecond, + MaxDelay: 100 * time.Millisecond, + Multiplier: 2.0, + }) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if attempts != 1 { + t.Errorf("expected 1 attempt, got %d", attempts) + } + }) + + t.Run("succeeds after retries", func(t *testing.T) { + ctx := context.Background() + var attempts int + err := Retry(ctx, func(ctx context.Context, attempt int) error { + attempts++ + if attempt < 3 { + return fmt.Errorf("attempt %d failed", attempt) + } + return nil + }, &RetryConfig{ + MaxRetries: 3, + InitialDelay: 5 * time.Millisecond, + MaxDelay: 50 * time.Millisecond, + Multiplier: 1.5, + }) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if attempts != 3 { + t.Errorf("expected 3 attempts, got %d", attempts) + } + }) + + t.Run("uses default config when nil", func(t *testing.T) { + ctx := context.Background() + err := Retry(ctx, func(ctx context.Context, attempt int) error { + return nil + }, nil) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + }) +} + +func TestRetryFailAll(t *testing.T) { + t.Parallel() + + ctx := context.Background() + var attempts int + expectedFailures := 4 // maxRetries(3) + initial(1) = 4 + + err := Retry(ctx, func(ctx context.Context, attempt int) error { + attempts++ + return fmt.Errorf("persistent error on attempt %d", attempt) + }, &RetryConfig{ + MaxRetries: 3, + InitialDelay: 5 * time.Millisecond, + MaxDelay: 50 * time.Millisecond, + Multiplier: 1.5, + }) + + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "all 4 attempts failed") { + t.Errorf("expected 'all 4 attempts failed' in error, got %q", err.Error()) + } + if attempts != expectedFailures { + t.Errorf("expected %d attempts, got %d", expectedFailures, attempts) + } +} + +func TestRetryWithContextCancel(t *testing.T) { + t.Parallel() + + t.Run("cancel before retry begins", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + err := Retry(ctx, func(ctx context.Context, attempt int) error { + return fmt.Errorf("should not be called") + }, &RetryConfig{ + MaxRetries: 3, + InitialDelay: 5 * time.Millisecond, + MaxDelay: 50 * time.Millisecond, + Multiplier: 1.5, + }) + + if err == nil { + t.Fatal("expected context canceled error") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got %v", err) + } + }) + + t.Run("cancel during retry delay", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + var attempts int + errCh := make(chan error, 1) + + go func() { + errCh <- Retry(ctx, func(ctx context.Context, attempt int) error { + attempts++ + // Cancel on second attempt + if attempt == 2 { + cancel() + } + return fmt.Errorf("failing attempt %d", attempt) + }, &RetryConfig{ + MaxRetries: 5, + InitialDelay: 1 * time.Second, // long enough for cancel to take effect + MaxDelay: 2 * time.Second, + Multiplier: 1.0, + }) + }() + + select { + case err := <-errCh: + if err == nil { + t.Fatal("expected an error") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("test timed out") + } + }) +} + +func TestIsRetryableError(t *testing.T) { + t.Parallel() + + t.Run("nil error is not retryable", func(t *testing.T) { + if IsRetryableError(nil) { + t.Error("expected false for nil error") + } + }) + + t.Run("connection refused", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("connection refused")) { + t.Error("expected true for 'connection refused'") + } + }) + + t.Run("connection reset", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("connection reset by peer")) { + t.Error("expected true for 'connection reset'") + } + }) + + t.Run("connection timed out", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("connection timed out")) { + t.Error("expected true for 'connection timed out'") + } + }) + + t.Run("i/o timeout", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("i/o timeout")) { + t.Error("expected true for 'i/o timeout'") + } + }) + + t.Run("temporary failure", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("temporary failure in name resolution")) { + t.Error("expected true for 'temporary failure'") + } + }) + + t.Run("no route to host", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("no route to host")) { + t.Error("expected true for 'no route to host'") + } + }) + + t.Run("case insensitive matching", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("connection refused")) { + t.Error("expected true for 'connection refused' (case insensitive)") + } + if !IsRetryableError(fmt.Errorf("connection timed out")) { + t.Error("expected true for 'connection timed out' (case insensitive)") + } + }) + + t.Run("non-retryable error", func(t *testing.T) { + if IsRetryableError(fmt.Errorf("404 not found")) { + t.Error("expected false for non-retryable error") + } + if IsRetryableError(fmt.Errorf("permission denied")) { + t.Error("expected false for permission denied") + } + }) + + t.Run("retryable substring in longer message", func(t *testing.T) { + if !IsRetryableError(fmt.Errorf("dial tcp 192.168.1.1:80: connect: connection refused")) { + t.Error("expected true for longer message containing 'connection refused'") + } + }) +} + +func TestIsRetryableStatusCode(t *testing.T) { + t.Parallel() + + t.Run("uses default status codes with nil config", func(t *testing.T) { + if !IsRetryableStatusCode(408, nil) { + t.Error("expected 408 to be retryable with nil config") + } + if !IsRetryableStatusCode(429, nil) { + t.Error("expected 429 to be retryable with nil config") + } + if !IsRetryableStatusCode(500, nil) { + t.Error("expected 500 to be retryable with nil config") + } + if !IsRetryableStatusCode(502, nil) { + t.Error("expected 502 to be retryable with nil config") + } + if !IsRetryableStatusCode(503, nil) { + t.Error("expected 503 to be retryable with nil config") + } + if !IsRetryableStatusCode(504, nil) { + t.Error("expected 504 to be retryable with nil config") + } + }) + + t.Run("non-retryable status codes", func(t *testing.T) { + if IsRetryableStatusCode(200, nil) { + t.Error("expected 200 not to be retryable") + } + if IsRetryableStatusCode(301, nil) { + t.Error("expected 301 not to be retryable") + } + if IsRetryableStatusCode(400, nil) { + t.Error("expected 400 not to be retryable") + } + if IsRetryableStatusCode(401, nil) { + t.Error("expected 401 not to be retryable") + } + if IsRetryableStatusCode(403, nil) { + t.Error("expected 403 not to be retryable") + } + if IsRetryableStatusCode(404, nil) { + t.Error("expected 404 not to be retryable") + } + }) + + t.Run("custom retryable codes", func(t *testing.T) { + cfg := &RetryConfig{ + RetryableStatusCodes: []int{429, 503}, + } + if !IsRetryableStatusCode(429, cfg) { + t.Error("expected 429 to be retryable with custom config") + } + if !IsRetryableStatusCode(503, cfg) { + t.Error("expected 503 to be retryable with custom config") + } + if IsRetryableStatusCode(500, cfg) { + t.Error("expected 500 not to be retryable with custom config") + } + if IsRetryableStatusCode(502, cfg) { + t.Error("expected 502 not to be retryable with custom config") + } + }) + + t.Run("empty retryable codes list", func(t *testing.T) { + cfg := &RetryConfig{ + RetryableStatusCodes: []int{}, + } + if IsRetryableStatusCode(503, cfg) { + t.Error("expected 503 not to be retryable with empty codes list") + } + }) +} + +func TestOnRetryCallback(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + callbackCount int + lastAttempt int + lastDelay time.Duration + ) + + cfg := &RetryConfig{ + MaxRetries: 2, + InitialDelay: 5 * time.Millisecond, + MaxDelay: 50 * time.Millisecond, + Multiplier: 1.0, + OnRetry: func(attempt int, err error, delay time.Duration) { + mu.Lock() + callbackCount++ + lastAttempt = attempt + lastDelay = delay + mu.Unlock() + }, + } + + ctx := context.Background() + err := Retry(ctx, func(ctx context.Context, attempt int) error { + return fmt.Errorf("failing attempt %d", attempt) + }, cfg) + + if err == nil { + t.Fatal("expected an error") + } + + mu.Lock() + if callbackCount != 2 { + t.Errorf("expected 2 OnRetry callbacks, got %d", callbackCount) + } + // Last callback should be for attempt 2 (the retry before final attempt) + if lastAttempt != 2 { + t.Errorf("expected last callback attempt=2, got %d", lastAttempt) + } + if lastDelay != 5*time.Millisecond { + t.Errorf("expected last delay=5ms, got %v", lastDelay) + } + mu.Unlock() +} + +// --------------------------------------------------------------------------- +// TLS tests +// --------------------------------------------------------------------------- + +func TestDefaultTLSConfig(t *testing.T) { + t.Parallel() + + cfg := DefaultTLSConfig() + + if cfg.MinVersion != tls.VersionTLS12 { + t.Errorf("expected MinVersion=TLS 1.2, got 0x%04X", cfg.MinVersion) + } + if cfg.MaxVersion != tls.VersionTLS13 { + t.Errorf("expected MaxVersion=TLS 1.3, got 0x%04X", cfg.MaxVersion) + } + if cfg.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=false") + } + if cfg.CACertFile != "" { + t.Errorf("expected empty CACertFile, got %s", cfg.CACertFile) + } + if cfg.PinnedCertHash != "" { + t.Errorf("expected empty PinnedCertHash, got %s", cfg.PinnedCertHash) + } + if cfg.TLSServerNameOverride != "" { + t.Errorf("expected empty TLSServerNameOverride, got %s", cfg.TLSServerNameOverride) + } +} + +func TestToTLSConfig(t *testing.T) { + t.Parallel() + + t.Run("basic conversion", func(t *testing.T) { + cfg := &TLSConfig{ + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + } + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg == nil { + t.Fatal("expected non-nil tls.Config") + } + if tlsCfg.MinVersion != tls.VersionTLS12 { + t.Errorf("expected MinVersion=TLS 1.2, got 0x%04X", tlsCfg.MinVersion) + } + if tlsCfg.MaxVersion != tls.VersionTLS13 { + t.Errorf("expected MaxVersion=TLS 1.3, got 0x%04X", tlsCfg.MaxVersion) + } + if tlsCfg.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=false") + } + if tlsCfg.ServerName != "" { + t.Errorf("expected empty ServerName, got %s", tlsCfg.ServerName) + } + }) + + t.Run("with InsecureSkipVerify set", func(t *testing.T) { + cfg := &TLSConfig{ + InsecureSkipVerify: true, + } + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !tlsCfg.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=true") + } + }) + + t.Run("with TLSServerNameOverride set", func(t *testing.T) { + cfg := &TLSConfig{ + TLSServerNameOverride: "example.com", + } + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg.ServerName != "example.com" { + t.Errorf("expected ServerName=example.com, got %s", tlsCfg.ServerName) + } + }) + + t.Run("with empty TLSServerNameOverride leaves ServerName empty", func(t *testing.T) { + // Sanity check: an empty override must not be propagated into the + // tls.Config, otherwise Go's stdlib would treat it as an explicit + // (empty) SNI value and certificate verification would fail. + cfg := &TLSConfig{} + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg.ServerName != "" { + t.Errorf("expected empty ServerName, got %s", tlsCfg.ServerName) + } + }) + + t.Run("with custom TLS versions", func(t *testing.T) { + cfg := &TLSConfig{ + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS12, + } + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg.MinVersion != tls.VersionTLS10 { + t.Errorf("expected MinVersion=TLS 1.0, got 0x%04X", tlsCfg.MinVersion) + } + if tlsCfg.MaxVersion != tls.VersionTLS12 { + t.Errorf("expected MaxVersion=TLS 1.2, got 0x%04X", tlsCfg.MaxVersion) + } + }) + + t.Run("non-existent CA cert file returns error", func(t *testing.T) { + cfg := &TLSConfig{ + CACertFile: "/nonexistent/path/ca.pem", + } + _, err := cfg.ToTLSConfig() + if err == nil { + t.Error("expected error for non-existent CA cert file") + } + }) + + t.Run("with PinnedCertHash returns config without error", func(t *testing.T) { + // PinnedCertHash is not yet implemented, but it should not cause an error + cfg := &TLSConfig{ + PinnedCertHash: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + } + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg == nil { + t.Fatal("expected non-nil tls.Config") + } + }) + + t.Run("default TLSConfig via ToTLSConfig", func(t *testing.T) { + cfg := DefaultTLSConfig() + tlsCfg, err := cfg.ToTLSConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tlsCfg.MinVersion != tls.VersionTLS12 { + t.Errorf("expected MinVersion=TLS 1.2, got 0x%04X", tlsCfg.MinVersion) + } + if tlsCfg.MaxVersion != tls.VersionTLS13 { + t.Errorf("expected MaxVersion=TLS 1.3, got 0x%04X", tlsCfg.MaxVersion) + } + }) +} + +func TestGetTLSVersionName(t *testing.T) { + t.Parallel() + + tests := []struct { + version uint16 + want string + }{ + {tls.VersionTLS10, "TLS 1.0"}, + {tls.VersionTLS11, "TLS 1.1"}, + {tls.VersionTLS12, "TLS 1.2"}, + {tls.VersionTLS13, "TLS 1.3"}, + {0x0000, "TLS 0x0000"}, + {0x0300, "TLS 0x0300"}, + {0xFFFF, "TLS 0xFFFF"}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("version_0x%04X", tt.version), func(t *testing.T) { + got := GetTLSVersionName(tt.version) + if got != tt.want { + t.Errorf("expected %q for version 0x%04X, got %q", tt.want, tt.version, got) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Token bucket tests +// --------------------------------------------------------------------------- + +func TestNewTokenBucket(t *testing.T) { + t.Parallel() + + t.Run("valid parameters", func(t *testing.T) { + tb := NewTokenBucket(1024, 2048) + if tb == nil { + t.Fatal("expected non-nil token bucket") + } + // Check initial state via Allow and subsequent behavior + if tb.capacity != 2048 { + t.Errorf("expected capacity=2048, got %d", tb.capacity) + } + if tb.tokens != float64(2048) { + t.Errorf("expected tokens=2048, got %f", tb.tokens) + } + if tb.refillRate != float64(1024) { + t.Errorf("expected refillRate=1024, got %f", tb.refillRate) + } + }) + + t.Run("equal rate and burst", func(t *testing.T) { + tb := NewTokenBucket(5000, 5000) + if tb == nil { + t.Fatal("expected non-nil token bucket") + } + if tb.capacity != 5000 { + t.Errorf("expected capacity=5000, got %d", tb.capacity) + } + }) +} + +func TestNewTokenBucketNil(t *testing.T) { + t.Parallel() + + t.Run("zero rate returns nil", func(t *testing.T) { + tb := NewTokenBucket(0, 1024) + if tb != nil { + t.Error("expected nil for zero rate") + } + }) + + t.Run("negative rate returns nil", func(t *testing.T) { + tb := NewTokenBucket(-100, 1024) + if tb != nil { + t.Error("expected nil for negative rate") + } + }) +} + +func TestTokenBucketAllow(t *testing.T) { + t.Parallel() + + t.Run("nil bucket Allow is no-op", func(t *testing.T) { + // Should not panic + var tb *TokenBucket + tb.Allow(100) + // No assertion needed - if it didn't panic, we're good + }) + + t.Run("allows up to capacity immediately", func(t *testing.T) { + tb := NewTokenBucket(1000, 500) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + // Should allow up to burst size without blocking + start := time.Now() + tb.Allow(300) + tb.Allow(200) + elapsed := time.Since(start) + if elapsed > 100*time.Millisecond { + t.Errorf("expected near-instant Allow for <= capacity, took %v", elapsed) + } + }) + + t.Run("blocks when exceeding rate", func(t *testing.T) { + // Rate: 100 bytes/sec, burst: 100 bytes + tb := NewTokenBucket(100, 100) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + + // Consume all tokens + tb.Allow(100) + + // Next Allow should block for some time + start := time.Now() + tb.Allow(50) // needs ~500ms to refill at 100 bytes/sec + elapsed := time.Since(start) + if elapsed < 200*time.Millisecond { + t.Errorf("expected Allow(50) to block for ~500ms, took %v", elapsed) + } + }) + + t.Run("allows after waiting for refill", func(t *testing.T) { + tb := NewTokenBucket(1000, 500) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + + // Consume all tokens + tb.Allow(500) + + // Wait for refill + time.Sleep(200 * time.Millisecond) // ~200 tokens refilled + + start := time.Now() + tb.Allow(150) // should have enough tokens + elapsed := time.Since(start) + if elapsed > 50*time.Millisecond { + t.Errorf("expected Allow(150) to be near-instant after waiting, took %v", elapsed) + } + }) +} + +func TestTokenBucketSmallBurst(t *testing.T) { + t.Parallel() + + t.Run("burst of 1 works with rate limiting", func(t *testing.T) { + // Rate: 1000 bytes/sec, burst: 1 byte — each byte needs ~1ms refill + tb := NewTokenBucket(1000, 1) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + if tb.capacity != 1 { + t.Errorf("expected capacity=1, got %d", tb.capacity) + } + // First byte should be allowed immediately (burst capacity) + start := time.Now() + tb.Allow(1) + elapsed := time.Since(start) + if elapsed > 50*time.Millisecond { + t.Errorf("expected Allow(1) to be instant, took %v", elapsed) + } + // Second byte should block for ~1ms at 1000 bytes/sec + start = time.Now() + tb.Allow(1) + elapsed = time.Since(start) + if elapsed < 500*time.Microsecond { + t.Errorf("expected Allow(1) to block briefly for refill, took %v", elapsed) + } + }) + + t.Run("zero burst with valid rate returns non-nil but Allow blocks indefinitely", func(t *testing.T) { + // capacity=0 means tokens are always capped to 0, so no token can ever be consumed. + // This is an inherent limitation — the test verifies the bucket is created without + // panic and that Allow(1) would block (but we don't actually call Allow to avoid timeout). + tb := NewTokenBucket(1000, 0) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + if tb.capacity != 0 { + t.Errorf("expected capacity=0, got %d", tb.capacity) + } + // Note: Allow(1) with capacity=0 would loop forever because tokens + // are always capped back to 0 after refill. + }) +} + +func TestWrapReader(t *testing.T) { + t.Parallel() + + t.Run("nil bucket returns original reader", func(t *testing.T) { + var tb *TokenBucket + r := strings.NewReader("hello") + wrapped := tb.WrapReader(r) + if wrapped != r { + t.Error("expected original reader when bucket is nil") + } + }) + + t.Run("wraps reader with rate limiter", func(t *testing.T) { + tb := NewTokenBucket(10000, 10000) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + r := strings.NewReader("hello world") + wrapped := tb.WrapReader(r) + if wrapped == nil { + t.Fatal("expected non-nil wrapped reader") + } + if _, ok := wrapped.(*limitedReader); !ok { + t.Error("expected *limitedReader type") + } + }) +} + +func TestLimitedReader(t *testing.T) { + t.Parallel() + + t.Run("reads all data through limiter", func(t *testing.T) { + tb := NewTokenBucket(100000, 100000) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + + input := []byte("the quick brown fox jumps over the lazy dog") + r := tb.WrapReader(bytes.NewReader(input)) + + output, err := io.ReadAll(r) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !bytes.Equal(output, input) { + t.Errorf("output doesn't match input: got %q, want %q", string(output), string(input)) + } + }) + + t.Run("reads with small buffer", func(t *testing.T) { + tb := NewTokenBucket(50000, 50000) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + + input := []byte("hello world") + r := tb.WrapReader(bytes.NewReader(input)) + + // Read byte by byte + var output []byte + buf := make([]byte, 1) + for { + n, err := r.Read(buf) + if n > 0 { + output = append(output, buf[:n]...) + } + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if !bytes.Equal(output, input) { + t.Errorf("output doesn't match input: got %q, want %q", string(output), string(input)) + } + }) + + t.Run("empty reader returns immediately", func(t *testing.T) { + tb := NewTokenBucket(1000, 1000) + if tb == nil { + t.Fatal("expected non-nil bucket") + } + + r := tb.WrapReader(bytes.NewReader(nil)) + n, err := r.Read(make([]byte, 10)) + if n != 0 || err != io.EOF { + t.Errorf("expected (0, EOF), got (%d, %v)", n, err) + } + }) +} + +// --------------------------------------------------------------------------- +// Integration test: retry with OnRetry counts +// --------------------------------------------------------------------------- + +func TestRetryExponentialBackoff(t *testing.T) { + t.Parallel() + + var delays []time.Duration + cfg := &RetryConfig{ + MaxRetries: 3, + InitialDelay: 10 * time.Millisecond, + MaxDelay: 100 * time.Millisecond, + Multiplier: 3.0, // aggressive multiplier for testing + OnRetry: func(attempt int, err error, delay time.Duration) { + delays = append(delays, delay) + }, + } + + ctx := context.Background() + Retry(ctx, func(ctx context.Context, attempt int) error { + return fmt.Errorf("fail") + }, cfg) + + // We should have had 3 retries: delays: 10ms, 30ms, 90ms + if len(delays) != 3 { + t.Fatalf("expected 3 delays, got %d: %v", len(delays), delays) + } + if delays[0] != 10*time.Millisecond { + t.Errorf("expected first delay=10ms, got %v", delays[0]) + } + if delays[1] != 30*time.Millisecond { + t.Errorf("expected second delay=30ms, got %v", delays[1]) + } + if delays[2] != 90*time.Millisecond { + t.Errorf("expected third delay=90ms, got %v", delays[2]) + } +} + +func TestRetryExponentialBackoffMaxDelay(t *testing.T) { + t.Parallel() + + var delays []time.Duration + cfg := &RetryConfig{ + MaxRetries: 4, + InitialDelay: 10 * time.Millisecond, + MaxDelay: 25 * time.Millisecond, + Multiplier: 3.0, + OnRetry: func(attempt int, err error, delay time.Duration) { + delays = append(delays, delay) + }, + } + + ctx := context.Background() + Retry(ctx, func(ctx context.Context, attempt int) error { + return fmt.Errorf("fail") + }, cfg) + + // delays: 10ms, 30ms (but capped at 25ms), 75ms (capped at 25ms), ... + if len(delays) != 4 { + t.Fatalf("expected 4 delays, got %d: %v", len(delays), delays) + } + if delays[1] != 25*time.Millisecond { + t.Errorf("expected second delay capped at 25ms, got %v", delays[1]) + } + if delays[2] != 25*time.Millisecond { + t.Errorf("expected third delay capped at 25ms, got %v", delays[2]) + } + if delays[3] != 25*time.Millisecond { + t.Errorf("expected fourth delay capped at 25ms, got %v", delays[3]) + } +} + +// --------------------------------------------------------------------------- +// Dialer edge cases (unit-level, no network required) +// --------------------------------------------------------------------------- + +func TestDialerLookupIPAddr(t *testing.T) { + t.Parallel() + + d := NewDialer(DefaultDialerConfig()) + if d == nil { + t.Fatal("expected non-nil dialer") + } + + // LookupIPAddr sorts IPv6 first - we can't easily test with real DNS + // without network, but we can verify the method exists and returns + // appropriate error for invalid hostname. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + _, err := d.LookupIPAddr(ctx, "invalid-host-name-that-does-not-exist.example.com") + // Should get some kind of DNS error + if err == nil { + t.Log("DNS lookup succeeded (unexpected on offline test)") + } +} + +func TestDialerDialContextInvalidAddress(t *testing.T) { + t.Parallel() + + d := NewDialer(DefaultDialerConfig()) + if d == nil { + t.Fatal("expected non-nil dialer") + } + + ctx := context.Background() + _, err := d.DialContext(ctx, "tcp", "invalid-address") + if err == nil { + t.Error("expected error for invalid address") + } + if !strings.Contains(err.Error(), "invalid address") { + t.Errorf("expected 'invalid address' in error, got %v", err) + } +} + +func TestDialerDialContextEmptyHost(t *testing.T) { + t.Parallel() + + d := NewDialer(DefaultDialerConfig()) + if d == nil { + t.Fatal("expected non-nil dialer") + } + + ctx := context.Background() + // ":80" will fail SplitHostPort with "missing host" or similar + _, err := d.DialContext(ctx, "tcp", ":80") + if err == nil { + t.Error("expected error for empty host with port") + } +} + +func TestContainsIgnoreCase(t *testing.T) { + t.Parallel() + + tests := []struct { + s, substr string + want bool + }{ + {"Hello World", "world", true}, + {"Hello World", "hello", true}, + {"Hello World", "WORLD", true}, + {"Hello World", "xyz", false}, + {"", "", true}, + {"abc", "", true}, + {"", "abc", false}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got := containsIgnoreCase(tt.s, tt.substr) + if got != tt.want { + t.Errorf("containsIgnoreCase(%q, %q) = %v, want %v", tt.s, tt.substr, got, tt.want) + } + }) + } +} diff --git a/internal/warc/writer.go b/internal/warc/writer.go new file mode 100644 index 0000000..1f5464b --- /dev/null +++ b/internal/warc/writer.go @@ -0,0 +1,50 @@ +//go:build linux || freebsd +// +build linux freebsd + +// Package warc provides Web ARChive (WARC) format reading and writing. +// It implements WARC/1.0 as defined by ISO 28500. +package warc + +import ( + "fmt" + "os" + "time" +) + +// Writer writes WARC records to a file. +type Writer struct { + path string +} + +// NewWriter creates a new WARC writer that writes records to the given file. +// Each call to WriteRecord or WriteRecordFile overwrites the file. +func NewWriter(path string) *Writer { + return &Writer{path: path} +} + +// WriteRecord writes a WARC response record from raw byte data. +// targetURI is the original URL of the downloaded resource. +func (w *Writer) WriteRecord(targetURI string, data []byte) error { + now := time.Now().UTC().Format(time.RFC3339) + warcData := fmt.Sprintf( + "WARC/1.0\r\n"+ + "WARC-Type: response\r\n"+ + "WARC-Date: %s\r\n"+ + "WARC-Target-URI: %s\r\n"+ + "Content-Length: %d\r\n"+ + "\r\n"+ + "%s\r\n\r\n", + now, targetURI, len(data), string(data)) + + return os.WriteFile(w.path, []byte(warcData), 0644) +} + +// WriteRecordFile reads the file at filePath and writes it as a WARC record. +// targetURI is the original URL of the downloaded resource. +func (w *Writer) WriteRecordFile(targetURI, filePath string) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read file for WARC: %w", err) + } + return w.WriteRecord(targetURI, data) +} diff --git a/justfile b/justfile new file mode 100644 index 0000000..bb328f9 --- /dev/null +++ b/justfile @@ -0,0 +1,24 @@ +_default: + @just --list + +# Install dependencies (Go modules) +install: + go mod download + +# Run from source (development) +run: + go run ./cmd/goget + +# Build the binary +build: + go vet ./... + gofmt -l $(find cmd internal pkg -name '*.go') | grep . && exit 1 || true + go build -ldflags "-s -w" -o bin/goget ./cmd/goget + +# Run tests (race detector + fuzz seed corpus) +test: + go test -race -count=1 ./... + +# Remove build artifacts +uninstall: + rm -rf bin/ coverage.out diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..7ce8cd6 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,341 @@ +//go:build linux || freebsd +// +build linux freebsd + +package api + +import ( + "context" + "fmt" + "net/url" + "time" + + "codeberg.org/petrbalvin/goget/internal/config" + "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/metalink" + "codeberg.org/petrbalvin/goget/internal/protocol" + "codeberg.org/petrbalvin/goget/internal/queue" +) + +// Version returns the current goget version. +func Version() string { + return core.Version +} + +// Client is the public API client for goget downloads and uploads. +type Client struct { + ctx context.Context + cancel context.CancelFunc + cfg *clientConfig +} + +// ClientOption configures a Client. +type ClientOption func(*clientConfig) + +type clientConfig struct { + timeout time.Duration + userAgent string + verbose bool +} + +// WithTimeout sets the request timeout. +func WithTimeout(d time.Duration) ClientOption { + return func(c *clientConfig) { + c.timeout = d + } +} + +// WithUserAgent sets the User-Agent header. +func WithUserAgent(ua string) ClientOption { + return func(c *clientConfig) { + c.userAgent = ua + } +} + +// WithVerbose enables verbose output. +func WithVerbose(v bool) ClientOption { + return func(c *clientConfig) { + c.verbose = v + } +} + +// NewClient creates a new goget client with the given options. +// The client registers HTTP and FTP protocol handlers automatically. +func NewClient(opts ...ClientOption) *Client { + cfg := &clientConfig{ + timeout: 30 * time.Minute, + userAgent: "Goget/" + Version() + " (Library)", + verbose: false, + } + for _, opt := range opts { + opt(cfg) + } + + registerDefaultProtocols() + + // The client-scoped context is cancellable but has no deadline. It + // lives until Close() is called and is the parent of every per-request + // context created by requestContext(). A deadline here would cap the + // *lifetime* of the client, so a long-lived Client could only serve + // requests for cfg.timeout before every subsequent call failed with + // "context deadline exceeded". The per-request deadline is applied + // below in Download and Upload. + ctx, cancel := context.WithCancel(context.Background()) + return &Client{ctx: ctx, cancel: cancel, cfg: cfg} +} + +// requestContext returns a child context of the client scope with the +// configured per-request timeout applied. If the timeout is zero or +// negative, the request inherits the client scope without a deadline. +func (c *Client) requestContext() (context.Context, context.CancelFunc) { + if c.cfg.timeout > 0 { + return context.WithTimeout(c.ctx, c.cfg.timeout) + } + return context.WithCancel(c.ctx) +} + +// DownloadRequest represents a download request. +type DownloadRequest struct { + URL string + Output string + Resume bool + Verbose bool +} + +// DownloadResult contains the result of a download. +type DownloadResult struct { + BytesDownloaded int64 + TotalSize int64 + Speed float64 + Duration time.Duration + OutputPath string + Protocol string + IPVersion int + Resumed bool + Parallel bool + ChunksCount int +} + +// Download downloads a file from the given URL. +func (c *Client) Download(req *DownloadRequest) (*DownloadResult, error) { + if req == nil || req.URL == "" { + return nil, fmt.Errorf("url is required") + } + + parsedURL, err := url.Parse(req.URL) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + + proto, err := protocol.Get(parsedURL) + if err != nil { + return nil, fmt.Errorf("unsupported protocol: %w", err) + } + + // Apply client defaults if request doesn't override + verbose := req.Verbose || c.cfg.verbose + headers := make(map[string]string) + if c.cfg.userAgent != "" { + headers["User-Agent"] = c.cfg.userAgent + } + + coreReq := &core.DownloadRequest{ + URL: parsedURL, + Output: req.Output, + Resume: req.Resume, + Verbose: verbose, + Headers: headers, + Timeout: c.cfg.timeout, + } + + // Derive a per-request context from the client scope so a long-lived + // Client can serve many requests and each one gets its own deadline. + reqCtx, reqCancel := c.requestContext() + defer reqCancel() + coreReq.Ctx = reqCtx + + result, err := proto.Download(reqCtx, coreReq) + if err != nil { + return nil, err + } + + return &DownloadResult{ + BytesDownloaded: result.BytesDownloaded, + TotalSize: result.TotalSize, + Speed: result.Speed, + Duration: result.Duration, + OutputPath: result.OutputPath, + Protocol: result.Protocol, + IPVersion: result.IPVersion, + Resumed: result.Resumed, + Parallel: result.Parallel, + ChunksCount: result.ChunksCount, + }, nil +} + +// UploadRequest represents an upload request. +type UploadRequest struct { + URL string + FilePath string + Method string // PUT or POST + Verbose bool +} + +// UploadResult contains the result of an upload. +type UploadResult struct { + BytesUploaded int64 + Duration time.Duration + Protocol string + ResultURL string +} + +// Upload uploads a file to the given URL. +func (c *Client) Upload(req *UploadRequest) (*UploadResult, error) { + if req == nil || req.URL == "" || req.FilePath == "" { + return nil, fmt.Errorf("url and file path are required") + } + + parsedURL, err := url.Parse(req.URL) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + + proto, err := protocol.Get(parsedURL) + if err != nil { + return nil, fmt.Errorf("unsupported protocol: %w", err) + } + + uploader, ok := proto.(core.Uploader) + if !ok { + return nil, fmt.Errorf("protocol does not support upload: %s", parsedURL.Scheme) + } + + coreReq := &core.UploadRequest{ + URL: parsedURL, + Input: req.FilePath, + Verbose: req.Verbose, + } + + // Per-request context so a long-lived Client can serve many uploads + // without each one being bounded by the time elapsed since client + // creation. See NewClient for the rationale. + reqCtx, reqCancel := c.requestContext() + defer reqCancel() + + coreResult, err := uploader.Upload(reqCtx, coreReq) + if err != nil { + return nil, err + } + + return &UploadResult{ + BytesUploaded: coreResult.BytesUploaded, + Duration: coreResult.Duration, + Protocol: coreResult.Protocol, + ResultURL: coreResult.ResultURL, + }, nil +} + +// Close releases resources associated with the client. +func (c *Client) Close() { + c.cancel() +} + +// Download is a convenience function for a one-shot download. +func Download(urlStr, output string) (*DownloadResult, error) { + client := NewClient() + defer client.Close() + + return client.Download(&DownloadRequest{ + URL: urlStr, + Output: output, + Resume: true, + }) +} + +// Upload is a convenience function for a one-shot upload. +func Upload(urlStr, filePath, method string) (*UploadResult, error) { + client := NewClient() + defer client.Close() + + return client.Upload(&UploadRequest{ + URL: urlStr, + FilePath: filePath, + Method: method, + }) +} + +// ============================================================================ +// Queue API +// ============================================================================ + +// QueueClient wraps the download queue for programmatic use. +type QueueClient struct { + q *queue.Queue +} + +// NewQueueClient creates a new queue client. +func NewQueueClient(queueFile string) (*QueueClient, error) { + q, err := queue.NewQueue(&queue.QueueConfig{QueueFile: queueFile}) + if err != nil { + return nil, err + } + return &QueueClient{q: q}, nil +} + +// Add adds a URL to the download queue. +func (qc *QueueClient) Add(urlStr, output string, priority int) string { + item := qc.q.Add(urlStr, output, priority) + return item.ID +} + +// List returns all items in the queue. +func (qc *QueueClient) List() []*queue.QueueItem { + return qc.q.GetAll() +} + +// Stats returns queue statistics. +func (qc *QueueClient) Stats() queue.QueueStats { + return qc.q.GetStats() +} + +// Remove removes an item from the queue by ID. +func (qc *QueueClient) Remove(id string) bool { + return qc.q.Remove(id) +} + +// ClearCompleted removes all completed items. +func (qc *QueueClient) ClearCompleted() int { + return qc.q.ClearCompleted() +} + +// ============================================================================ +// Metalink API +// ============================================================================ + +// ParseMetalink parses a Metalink file. +func ParseMetalink(path string) (*metalink.Metalink, error) { + return metalink.ParseFile(path) +} + +// FetchMetalink downloads and parses a Metalink from a URL. +func FetchMetalink(urlStr string) (*metalink.Metalink, error) { + return metalink.ParseURL(urlStr) +} + +// ============================================================================ +// Config API +// ============================================================================ + +// LoadConfig loads the configuration file. +func LoadConfig(path string) (*config.Config, error) { + return config.Load(path) +} + +// DefaultConfig returns the default configuration. +func DefaultConfig() *config.Config { + return config.DefaultConfig() +} + +// ParseSpeed parses a speed string like "10MB/s" to bytes/sec. +func ParseSpeed(s string) int64 { + return config.ParseSpeed(s) +} diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go new file mode 100644 index 0000000..72f3be9 --- /dev/null +++ b/pkg/api/api_test.go @@ -0,0 +1,197 @@ +//go:build linux || freebsd +// +build linux freebsd + +package api + +import ( + "strings" + "testing" + "time" +) + +func TestVersion(t *testing.T) { + v := Version() + if v == "" { + t.Error("Version() returned an empty string") + } +} + +func TestVersionFormat(t *testing.T) { + v := Version() + if !strings.Contains(v, ".") { + t.Errorf("Version() = %q; expected semver-like format", v) + } +} + +func TestNewClient(t *testing.T) { + client := NewClient() + if client == nil { + t.Fatal("NewClient() returned nil") + } + defer client.Close() +} + +func TestNewClientWithOptions(t *testing.T) { + client := NewClient(WithVerbose(true), WithUserAgent("Test/1.0")) + if client == nil { + t.Fatal("NewClient() returned nil") + } + defer client.Close() +} + +func TestDownloadRequestValidation(t *testing.T) { + client := NewClient() + defer client.Close() + + if _, err := client.Download(nil); err == nil { + t.Error("expected error for nil request") + } + if _, err := client.Download(&DownloadRequest{}); err == nil { + t.Error("expected error for empty URL") + } +} + +func TestUploadRequestValidation(t *testing.T) { + client := NewClient() + defer client.Close() + + if _, err := client.Upload(nil); err == nil { + t.Error("expected error for nil request") + } + if _, err := client.Upload(&UploadRequest{URL: "http://example.com"}); err == nil { + t.Error("expected error for missing FilePath") + } + if _, err := client.Upload(&UploadRequest{FilePath: "/tmp/test"}); err == nil { + t.Error("expected error for missing URL") + } +} + +func TestDownloadResultFields(t *testing.T) { + result := &DownloadResult{ + BytesDownloaded: 1024, + Protocol: "HTTP/2", + } + if result.BytesDownloaded != 1024 { + t.Errorf("BytesDownloaded = %d", result.BytesDownloaded) + } +} + +func TestUploadResultFields(t *testing.T) { + result := &UploadResult{ + BytesUploaded: 512, + } + if result.BytesUploaded != 512 { + t.Errorf("BytesUploaded = %d", result.BytesUploaded) + } +} + +func TestDownloadConvenience(t *testing.T) { + _, err := Download("", "") + if err == nil { + t.Error("expected error for empty URL") + } +} + +func TestUploadConvenience(t *testing.T) { + _, err := Upload("", "", "") + if err == nil { + t.Error("expected error for empty fields") + } +} + +// TestRequestContextTimeout is a regression guard for the BACKLOG entry +// "`pkg/api` Client timeout is lifetime, not per-request". The previous +// implementation created a single context.WithTimeout in NewClient, so a +// Client that lived longer than cfg.timeout could no longer serve any +// request. The fixed implementation uses a cancellable background context +// for the Client scope and derives a per-request child with the timeout +// applied at call time. +func TestRequestContextTimeout(t *testing.T) { + client := NewClient(WithTimeout(5 * time.Second)) + defer client.Close() + + reqCtx, cancel := client.requestContext() + defer cancel() + + deadline, ok := reqCtx.Deadline() + if !ok { + t.Fatal("per-request context has no deadline; want cfg.timeout") + } + + remaining := time.Until(deadline) + // Allow generous slack to avoid flakes on a busy CI runner, but + // enforce that the deadline is at most the configured timeout and + // has not already passed. + if remaining <= 0 { + t.Errorf("per-request deadline already passed: %v remaining", remaining) + } + if remaining > 5*time.Second { + t.Errorf("per-request deadline = %v from now; want <= 5s (cfg.timeout)", remaining) + } +} + +// TestRequestContextNoTimeout verifies that a Client constructed with +// WithTimeout(0) yields a per-request context without a deadline. This +// preserves the "no timeout" semantics that callers relied on before the +// per-request refactor, when the only way to disable the lifetime cap +// was to never call Download at all. +func TestRequestContextNoTimeout(t *testing.T) { + client := NewClient(WithTimeout(0)) + defer client.Close() + + reqCtx, cancel := client.requestContext() + defer cancel() + + if _, ok := reqCtx.Deadline(); ok { + t.Error("per-request context has a deadline; want none when cfg.timeout == 0") + } +} + +// TestRequestContextCancelledByClientClose verifies that cancelling the +// Client scope propagates to per-request children. A long-running +// Download must abort when Close() is called, even if its own deadline +// has not been reached. +func TestRequestContextCancelledByClientClose(t *testing.T) { + client := NewClient(WithTimeout(time.Hour)) + defer client.Close() + + reqCtx, cancel := client.requestContext() + defer cancel() + + // Sanity: nothing is cancelled yet. + if err := reqCtx.Err(); err != nil { + t.Fatalf("reqCtx unexpectedly cancelled before Close: %v", err) + } + + client.Close() + + // Close cancels the client scope, which propagates to the child. + select { + case <-reqCtx.Done(): + case <-time.After(2 * time.Second): + t.Fatal("per-request context not cancelled after Client.Close()") + } + if err := reqCtx.Err(); err == nil { + t.Error("reqCtx.Err() = nil after Client.Close(); want non-nil") + } +} + +// TestClientScopeHasNoDeadline is a regression guard for the original +// bug. The previous implementation wrapped context.Background() in +// WithTimeout at NewClient time, so a Client that lived past cfg.timeout +// had a context that was already past its deadline — and any +// context.WithTimeout child would be expired immediately. +func TestClientScopeHasNoDeadline(t *testing.T) { + client := NewClient(WithTimeout(50 * time.Millisecond)) + defer client.Close() + + if _, ok := client.ctx.Deadline(); ok { + t.Error("client-scope context has a deadline; lifetime cap would re-introduce the bug") + } +} + +// TestRequestContextCancelledByClientClose above already proves that +// per-request contexts are derived from the client scope: cancelling the +// client cancels the per-request child. A direct parent-chain walk is not +// possible from outside the context package because the `parent` field +// is unexported. diff --git a/pkg/api/register.go b/pkg/api/register.go new file mode 100644 index 0000000..8047ef0 --- /dev/null +++ b/pkg/api/register.go @@ -0,0 +1,19 @@ +//go:build linux || freebsd +// +build linux freebsd + +package api + +import ( + "sync" + + "codeberg.org/petrbalvin/goget/internal/protocol" + protocolreg "codeberg.org/petrbalvin/goget/internal/protocol/register" +) + +var registerOnce sync.Once + +func registerDefaultProtocols() { + registerOnce.Do(func() { + _, _ = protocolreg.RegisterAll(protocol.GlobalRegistry) + }) +}