From fc65ea8340e07663cb4e2949602067a45bbdd1fb Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 29 Jun 2026 19:23:10 +0200 Subject: [PATCH] feat: extend certificate pinning to all TLS protocols and add SSH host-key pinning --- BACKLOG.md | 2 +- CHANGELOG.md | 15 +++++ cmd/goget/app.go | 2 + cmd/goget/flags.go | 4 +- docs/cli-reference.md | 3 +- docs/protocols.md | 12 ++++ docs/security.md | 21 ++++++- internal/cli/help.txt | 3 +- internal/core/types.go | 11 ++++ internal/protocol/ftp/client.go | 41 +++++++++++- internal/protocol/ftp/ftp_test.go | 18 ++++++ internal/protocol/ftp/protocol.go | 5 +- internal/protocol/gemini/gemini.go | 35 +++++++++-- internal/protocol/gemini/gemini_test.go | 38 +++++++++++ internal/protocol/sftp/hostkey.go | 83 ++++++++++++++++++++++++- internal/protocol/sftp/hostkey_test.go | 60 +++++++++++++++++- internal/protocol/sftp/sftp.go | 10 +-- internal/protocol/sftp/sftp_test.go | 2 +- internal/protocol/webdav/protocol.go | 15 ++++- 19 files changed, 355 insertions(+), 25 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 29e5050..ac55d84 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -16,7 +16,7 @@ certificate pinning and metadata debugging to all TLS protocols. - [x] **Graceful shutdown for all protocols** — persist `.goget.meta` on SIGINT for FTP single-file and SFTP parallel recursive paths. -- [ ] **`--pinned-cert` for all TLS protocols** — extend certificate +- [x] **`--pinned-cert` for all TLS protocols** — extend certificate pinning to FTPS, Gemini, and WebDAVS (currently HTTP/HTTPS only). Also add SSH host-key pinning for SFTP as `--pinned-host-key`. - [x] **Unified worker pool** — a single protocol-agnostic worker pool diff --git a/CHANGELOG.md b/CHANGELOG.md index 4747bed..82e6747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt for SFTP single-file downloads, ensuring progress is saved for resume. +**Security** + +- **Certificate pinning across all TLS protocols** — `--pinned-cert` now + applies to FTPS, Gemini, and WebDAVS in addition to HTTP/HTTPS. The leaf + certificate SHA-256 hash is verified via `tls.Config.VerifyPeerCertificate` + on every TLS-bearing protocol. +- **SSH host-key pinning for SFTP** — new `--pinned-host-key` flag pins the + SSH host key by its OpenSSH fingerprint (`SHA256:`, as produced by + `ssh-keygen -lf`). Raw lowercase-hex SHA-256 of the key wire format is also + accepted. Pinning takes precedence over `--ssh-insecure` and `known_hosts`. +- **WebDAV TLS config fix** — `webdav.Protocol.getHTTPTransport` now builds + its `tls.Config` from the caller-provided `transport.TLSConfig` instead of + a fresh config that silently dropped CA certs, mTLS, keylog, and pinned-cert + settings. + **Recursive downloading & mirroring** - `--adjust-extension` extended to non-HTTP protocols — applies `.html` diff --git a/cmd/goget/app.go b/cmd/goget/app.go index ce75e8e..5268eda 100644 --- a/cmd/goget/app.go +++ b/cmd/goget/app.go @@ -606,6 +606,8 @@ func (a *app) execute() int { Ctx: ctx, SSHKnownHosts: *f.KnownHosts, SSHInsecure: *f.SSHInsecure, + PinnedCertHash: *f.PinnedCert, + PinnedHostKey: *f.PinnedHostKey, } // Configure TLS for WebDAV if supported. diff --git a/cmd/goget/flags.go b/cmd/goget/flags.go index bbb340e..d26083c 100644 --- a/cmd/goget/flags.go +++ b/cmd/goget/flags.go @@ -150,6 +150,7 @@ type Flags struct { Referer *string RetryDelay *time.Duration PinnedCert *string + PinnedHostKey *string AllowPrivate *bool Range *string ConnectTimeout *time.Duration @@ -289,7 +290,8 @@ func defineFlags(fs *flag.FlagSet, cfg *config.Config) *Flags { Compressed: fs.Bool("compressed", false, "Send Accept-Encoding header for auto-decompression"), Referer: fs.String("referer", "", "Send Referer header"), RetryDelay: fs.Duration("retry-delay", 0, "Delay between retry attempts"), - PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning"), + PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning (TLS protocols: HTTPS, FTPS, Gemini, WebDAVS)"), + PinnedHostKey: fs.String("pinned-host-key", "", "SSH host key fingerprint for pinning (SFTP). Accepts 'SHA256:' (ssh-keygen -lf) or raw hex SHA-256."), AllowPrivate: fs.Bool("allow-private", false, "Allow connections to private/internal IPs (disables SSRF protection)"), Range: fs.String("range", "", "Download only a range of bytes (e.g., 0-1000)"), ConnectTimeout: fs.Duration("connect-timeout", 0, "Timeout for connection establishment"), diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3548cd1..70b6f64 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -49,7 +49,8 @@ goget --url [OPTIONS] | `--dns-servers` | `` | Custom DNS servers, comma-separated (e.g. `1.1.1.1,8.8.8.8`) | | `--bind-interface` | `` | Bind outgoing connections to a specific network interface (Linux only) | | `--allow-private` | — | Allow connections to private/internal IPs (disabled by default for SSRF protection) | -| `--pinned-cert` | `` | Certificate pinning — abort if server cert SHA-256 doesn't match | +| `--pinned-cert` | `` | Certificate pinning (TLS protocols: HTTPS, FTPS, Gemini, WebDAVS) — abort if leaf cert SHA-256 doesn't match | +| `--pinned-host-key` | `` | SSH host-key pinning (SFTP) — accepts `SHA256:` (`ssh-keygen -lf`) or raw hex SHA-256 | ## HTTP / Network Flags diff --git a/docs/protocols.md b/docs/protocols.md index f8c8005..39b23c0 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -78,6 +78,7 @@ Custom FTP implementation supporting active and passive modes. - **Explicit TLS (FTPS)** — AUTH TLS command on port 21 - **Resume** — REST command for partial transfers - **Directory listing** — For recursive operations +- **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match on FTPS connections ### Limitations @@ -95,6 +96,9 @@ Custom FTP implementation supporting active and passive modes. # Anonymous (default) --url ftp://anonymous@ftp.example.com/file.zip + +# Certificate pinning (FTPS only) +--url ftps://ftp.example.com/file.zip --pinned-cert a1b2c3d4e5f6... ``` ## SFTP (`sftp://`) @@ -108,6 +112,7 @@ SFTP over SSH protocol. - **Known hosts verification** — Via `~/.ssh/known_hosts` - **Resume** — Partial transfer support; parallel recursive downloads available via `--recursive-parallel N` with connection pooling - **`known_hosts` verification** — Reads `~/.ssh/known_hosts`; does not write to it. First-connection accept via `--ssh-insecure` +- **Host-key pinning** — `--pinned-host-key` enforces an exact SSH host-key match (takes precedence over `--ssh-insecure` and `known_hosts`) ### Configuration @@ -123,6 +128,9 @@ SFTP over SSH protocol. # With password --url sftp://user@host:/path/to/file.zip --password "secret" + +# Host-key pinning (OpenSSH fingerprint from ssh-keygen -lf) +--url sftp://user@host:/path/to/file.zip --pinned-host-key SHA256:AAAAC3NzaC1lZDI1NTE5AAAAIE... ``` ## Local File (`file://`) @@ -202,6 +210,7 @@ Implements the Gemini protocol (a lightweight alternative to HTTP/HTTPS). - **Redirect following** — Respects Gemini redirect status codes (3x), up to 10 redirects - **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled - **Text and binary downloads** — 2x success responses stream content directly +- **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match ### Limitations @@ -217,6 +226,9 @@ goget --url gemini://gemini.example.com/file.zip # Browse a capsule goget --url gemini://gemini.example.com/ + +# Pin the server certificate +goget --url gemini://gemini.example.com/ --pinned-cert a1b2c3d4e5f6... ``` ## Protocol Resolution Flow diff --git a/docs/security.md b/docs/security.md index 7bbd019..ba1b689 100644 --- a/docs/security.md +++ b/docs/security.md @@ -12,13 +12,30 @@ goget implements multiple security features to protect both the client and serve ### Certificate Pinning -Pin a specific certificate by its SHA-256 hash: +Pin a specific certificate by its SHA-256 hash. This works across every +TLS-bearing protocol goget supports: HTTPS, FTPS, Gemini, and WebDAVS. ```bash goget --pinned-cert "a1b2c3d4e5f6..." --url https://example.com +goget --pinned-cert "a1b2c3d4e5f6..." --url ftps://ftp.example.com/file +goget --pinned-cert "a1b2c3d4e5f6..." --url gemini://geminiprotocol.net +goget --pinned-cert "a1b2c3d4e5f6..." --url webdavs://dav.example.com/file ``` -If the server certificate's SHA-256 doesn't match, the connection is aborted. +If the leaf certificate's SHA-256 doesn't match, the connection is aborted. + +### SSH Host-Key Pinning (SFTP) + +Pin the SSH host key for SFTP by its OpenSSH fingerprint (the same value +printed by `ssh-keygen -lf ~/.ssh/known_hosts`). Pinning takes precedence +over `--ssh-insecure` and the `known_hosts` file. + +```bash +goget --pinned-host-key "SHA256:AAAAC3NzaC1lZDI1NTE5AAAAIE+HHi0MtRj4VPl8mdP8gniGZDRb0SZTdI2TPxRyCm1H" \ + --url sftp://example.com/file +``` + +The raw lowercase-hex SHA-256 of the key wire format is also accepted. ### Client Certificates (mTLS) diff --git a/internal/cli/help.txt b/internal/cli/help.txt index effaebd..5dbe795 100644 --- a/internal/cli/help.txt +++ b/internal/cli/help.txt @@ -37,7 +37,8 @@ NETWORK --dns-servers Custom DNS servers (comma-separated) --bind-interface Bind to specific network interface --allow-private Allow connections to private/internal IPs - --pinned-cert Certificate pinning (SHA-256 hash) + --pinned-cert Certificate pinning (TLS: HTTPS, FTPS, Gemini, WebDAVS) + --pinned-host-key SSH host-key pinning (SFTP): SHA256: or hex CURL-COMPATIBLE --header "K: V" Custom HTTP header (repeatable) diff --git a/internal/core/types.go b/internal/core/types.go index 67a5f4a..9f16a4a 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -87,6 +87,17 @@ type DownloadRequest struct { Ctx context.Context SSHKnownHosts string // path to known_hosts file (SFTP) SSHInsecure bool // skip SSH host key verification (SFTP) + + // PinnedCertHash pins a TLS leaf certificate by its SHA-256 hash + // (lowercase hex). Applied to every TLS-bearing protocol: HTTPS, + // FTPS, Gemini, WebDAVS. Empty disables pinning. + PinnedCertHash string + + // PinnedHostKey pins an SSH host key. Accepted formats: + // "SHA256:" — OpenSSH fingerprint (ssh-keygen -lf) + // "" — raw SHA-256 hex of the key wire format + // Applied to SFTP only. Empty disables pinning. + PinnedHostKey string } // DownloadResult represents a download result diff --git a/internal/protocol/ftp/client.go b/internal/protocol/ftp/client.go index 7a53f9e..b1a8088 100644 --- a/internal/protocol/ftp/client.go +++ b/internal/protocol/ftp/client.go @@ -6,7 +6,10 @@ package ftp import ( "bufio" "context" + "crypto/sha256" "crypto/tls" + "crypto/x509" + "encoding/hex" "fmt" "io" "net" @@ -35,6 +38,11 @@ type Client struct { useTLS bool passive bool timeout time.Duration + + // pinnedCertHash, when non-empty, requires the server's leaf + // certificate to match this SHA-256 hash (lowercase hex). Only + // meaningful for FTPS. + pinnedCertHash string } func NewClient(rawURL string, timeout time.Duration) (*Client, error) { @@ -86,6 +94,13 @@ func NewClient(rawURL string, timeout time.Duration) (*Client, error) { return c, nil } +// SetPinnedCert configures certificate pinning for FTPS connections. +// The expectedHash is a SHA-256 hash of the leaf certificate, expressed +// as lowercase hex. Calling this on a plain-FTP client is a no-op. +func (c *Client) SetPinnedCert(expectedHash string) { + c.pinnedCertHash = strings.ToLower(strings.TrimSpace(expectedHash)) +} + func (c *Client) Connect() error { addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port)) conn, err := net.DialTimeout("tcp", addr, c.timeout) @@ -130,6 +145,23 @@ func (c *Client) Connect() error { } // Wrap connection with TLS + if c.pinnedCertHash != "" { + expected := c.pinnedCertHash + c.tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + for _, rawCert := range rawCerts { + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + continue + } + hash := sha256.Sum256(cert.Raw) + if hex.EncodeToString(hash[:]) == expected { + return nil + } + } + return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected) + } + } + tlsConn := tls.Client(c.conn, c.tlsConfig) if err := tlsConn.Handshake(); err != nil { return fmt.Errorf("tls handshake failed: %w", err) @@ -647,7 +679,7 @@ type clientPool struct { // newClientPool opens `size` independent FTP connections to the same // server. Connections are kept open until close() is called. -func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) { +func newClientPool(rawURL string, timeout time.Duration, size int, pinnedCertHash string) (*clientPool, error) { if size < 1 { size = 1 } @@ -661,6 +693,9 @@ func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, } return nil, err } + if pinnedCertHash != "" { + c.SetPinnedCert(pinnedCertHash) + } if err := c.Connect(); err != nil { for _, existing := range all { _ = existing.Close() @@ -774,11 +809,11 @@ func ftpTaskHandler(cpool *clientPool, progressFunc func(file string, current, t // on the server side. The returned error is the first error encountered // by any worker; other workers continue in the background until they // hit ctx cancellation or finish their tasks. -func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error { +func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, pinnedCertHash string, progressFunc func(file string, current, total int64)) error { if parallel < 1 { parallel = 1 } - cpool, err := newClientPool(rawURL, timeout, parallel) + cpool, err := newClientPool(rawURL, timeout, parallel, pinnedCertHash) if err != nil { return err } diff --git a/internal/protocol/ftp/ftp_test.go b/internal/protocol/ftp/ftp_test.go index 3d7ef47..ca7ebfd 100644 --- a/internal/protocol/ftp/ftp_test.go +++ b/internal/protocol/ftp/ftp_test.go @@ -380,3 +380,21 @@ func TestNewClientTLSConfig(t *testing.T) { t.Errorf("tlsConfig.InsecureSkipVerify should be false by default") } } + +func TestSetPinnedCert(t *testing.T) { + client, err := NewClient("ftps://secure.example.com/file.txt", 30*time.Second) + if err != nil { + t.Fatalf("NewClient() returned error: %v", err) + } + + // Pinning is off by default. + if client.pinnedCertHash != "" { + t.Errorf("pinnedCertHash should default to empty, got %q", client.pinnedCertHash) + } + + // SetPinnedCert normalises to lowercase and trims whitespace. + client.SetPinnedCert(" ABCDEF0123456789 ") + if client.pinnedCertHash != "abcdef0123456789" { + t.Errorf("pinnedCertHash = %q, want %q", client.pinnedCertHash, "abcdef0123456789") + } +} diff --git a/internal/protocol/ftp/protocol.go b/internal/protocol/ftp/protocol.go index b1404ae..4d3c842 100644 --- a/internal/protocol/ftp/protocol.go +++ b/internal/protocol/ftp/protocol.go @@ -79,6 +79,9 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co if err != nil { return nil, core.NewProtocolError(fmt.Sprintf("failed to create FTP client: %v", err), nil, core.SafeURL(req.URL)) } + if req.PinnedCertHash != "" { + client.SetPinnedCert(req.PinnedCertHash) + } if err := client.Connect(); err != nil { return nil, core.NewProtocolError(fmt.Sprintf("failed to connect: %v", err), nil, core.SafeURL(req.URL)) @@ -169,7 +172,7 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co // opens N control connections to avoid stepping on CWD state and // to keep the existing client-side protocol serialisation. if req.RecursiveParallel > 1 { - if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, progressCallback); err != nil { + if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, req.PinnedCertHash, progressCallback); err != nil { return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL)) } } else { diff --git a/internal/protocol/gemini/gemini.go b/internal/protocol/gemini/gemini.go index 327cf38..bb040bd 100644 --- a/internal/protocol/gemini/gemini.go +++ b/internal/protocol/gemini/gemini.go @@ -6,7 +6,10 @@ package gemini import ( "bufio" "context" + "crypto/sha256" "crypto/tls" + "crypto/x509" + "encoding/hex" "fmt" "io" "net" @@ -63,6 +66,27 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download InsecureSkipVerify: p.TLSInsecure, } + // Certificate pinning: require the leaf certificate to match the + // caller-provided SHA-256 hash. The check runs after the standard + // verification chain so a pinned self-signed cert still needs + // InsecureSkipVerify (or a matching CA) to clear the normal path. + if req != nil && req.PinnedCertHash != "" { + expected := strings.ToLower(strings.TrimSpace(req.PinnedCertHash)) + tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + for _, rawCert := range rawCerts { + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + continue + } + hash := sha256.Sum256(cert.Raw) + if hex.EncodeToString(hash[:]) == expected { + return nil + } + } + return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected) + } + } + dialer := &net.Dialer{Timeout: 30 * time.Second} conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsCfg) if err != nil { @@ -123,11 +147,12 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download } // Follow redirect with incremented depth redirectReq := &core.DownloadRequest{ - URL: resolved, - Output: req.Output, - Verbose: req.Verbose, - Writer: req.Writer, - Ctx: ctx, + URL: resolved, + Output: req.Output, + Verbose: req.Verbose, + Writer: req.Writer, + PinnedCertHash: req.PinnedCertHash, + Ctx: ctx, } return p.downloadWithRedirects(ctx, redirectReq, redirectCount+1) diff --git a/internal/protocol/gemini/gemini_test.go b/internal/protocol/gemini/gemini_test.go index a1759d6..0d445e4 100644 --- a/internal/protocol/gemini/gemini_test.go +++ b/internal/protocol/gemini/gemini_test.go @@ -132,3 +132,41 @@ func TestGeminiPermanentFailure(t *testing.T) { t.Error("expected permanent failure error") } } + +// TestGeminiPinnedCertMismatch exercises the certificate-pinning path. +// The server presents a self-signed cert whose hash does NOT match the +// pinned value, so the download must fail with a pinning error. +func TestGeminiPinnedCertMismatch(t *testing.T) { + cfg := testTLSConfig() + listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + go func() { + conn, _ := listener.Accept() + if conn == nil { + return + } + defer conn.Close() + r := bufio.NewReader(conn) + r.ReadString('\n') + conn.Write([]byte("20 text/gemini\r\nhello\n")) + }() + + proto := NewProtocol() + proto.TLSInsecure = true // accept self-signed, then let pinning reject + u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/") + _, err = proto.Download(context.Background(), &core.DownloadRequest{ + URL: u, + PinnedCertHash: "0000000000000000000000000000000000000000000000000000000000000000", + }) + if err == nil { + t.Fatal("expected pinning failure") + } + if !strings.Contains(err.Error(), "pinning failed") { + t.Errorf("expected pinning error, got: %v", err) + } +} diff --git a/internal/protocol/sftp/hostkey.go b/internal/protocol/sftp/hostkey.go index 8034d60..9a10e56 100644 --- a/internal/protocol/sftp/hostkey.go +++ b/internal/protocol/sftp/hostkey.go @@ -4,15 +4,39 @@ package sftp import ( + "crypto/sha256" + "encoding/base64" "fmt" + "net" "os" "path/filepath" + "strings" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" ) -func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, error) { +// hostKeyCallback builds the ssh.HostKeyCallback used for dialing. +// +// Precedence: +// +// 1. pinnedHostKey — when set, only a host whose key matches the +// given fingerprint is accepted. known_hosts and --ssh-insecure +// are both ignored because pinning is a stricter guarantee. +// 2. insecure --ssh-insecure skips verification entirely. +// 3. known_hosts file (default ~/.ssh/known_hosts). +func hostKeyCallback(knownHostsPath string, insecure bool, pinnedHostKey string) (ssh.HostKeyCallback, error) { + pinned := normalizePinnedHostKey(pinnedHostKey) + if pinned != "" { + return func(_ string, _ net.Addr, key ssh.PublicKey) error { + actual := sshFingerprintSHA256(key) + if actual == pinned { + return nil + } + return fmt.Errorf("host key pinning failed: server key %s does not match pinned %s", actual, pinned) + }, nil + } + if insecure { return ssh.InsecureIgnoreHostKey(), nil } @@ -31,3 +55,60 @@ func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, return knownhosts.New(knownHostsPath) } + +// sshFingerprintSHA256 returns the OpenSSH-style SHA-256 fingerprint +// of an SSH public key, e.g. "SHA256:abc123...". This matches the +// output of `ssh-keygen -lf `. +func sshFingerprintSHA256(key ssh.PublicKey) string { + hash := sha256.Sum256(key.Marshal()) + return "SHA256:" + base64.StdEncoding.EncodeToString(hash[:]) +} + +// normalizePinnedHostKey accepts either the OpenSSH fingerprint form +// ("SHA256:") or a raw lowercase-hex SHA-256 of the key wire +// format, and returns the canonical OpenSSH fingerprint form. An empty +// input returns an empty string. +func normalizePinnedHostKey(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + if strings.HasPrefix(s, "SHA256:") { + return s + } + // Raw hex form — convert to base64 to match the OpenSSH fingerprint. + if decoded := tryHexDecode(s); decoded != nil { + return "SHA256:" + base64.StdEncoding.EncodeToString(decoded) + } + return s +} + +// tryHexDecode decodes a lowercase-or-uppercase hex string. Returns nil +// on any error or odd length. +func tryHexDecode(s string) []byte { + if len(s)%2 != 0 { + return nil + } + out := make([]byte, len(s)/2) + for i := 0; i < len(s); i += 2 { + hi, ok1 := hexNibble(s[i]) + lo, ok2 := hexNibble(s[i+1]) + if !ok1 || !ok2 { + return nil + } + out[i/2] = hi<<4 | lo + } + return out +} + +func hexNibble(c byte) (byte, bool) { + switch { + case c >= '0' && c <= '9': + return c - '0', true + case c >= 'a' && c <= 'f': + return c - 'a' + 10, true + case c >= 'A' && c <= 'F': + return c - 'A' + 10, true + } + return 0, false +} diff --git a/internal/protocol/sftp/hostkey_test.go b/internal/protocol/sftp/hostkey_test.go index 76fdb7c..2a1133c 100644 --- a/internal/protocol/sftp/hostkey_test.go +++ b/internal/protocol/sftp/hostkey_test.go @@ -4,11 +4,14 @@ package sftp import ( + "strings" "testing" + + "golang.org/x/crypto/ssh" ) func TestHostKeyCallbackInsecure(t *testing.T) { - cb, err := hostKeyCallback("", true) + cb, err := hostKeyCallback("", true, "") if err != nil { t.Fatalf("hostKeyCallback: %v", err) } @@ -18,8 +21,61 @@ func TestHostKeyCallbackInsecure(t *testing.T) { } func TestHostKeyCallbackMissingFile(t *testing.T) { - _, err := hostKeyCallback("/nonexistent/known_hosts", false) + _, err := hostKeyCallback("/nonexistent/known_hosts", false, "") if err == nil { t.Fatal("expected error for missing known_hosts") } } + +func TestHostKeyCallbackPinned(t *testing.T) { + // Pinning takes precedence over insecure and known_hosts: the + // callback is returned without touching the filesystem. + cb, err := hostKeyCallback("/nonexistent/known_hosts", false, "SHA256:abcdefghijklmnopqrstuvwxyz0123456789+/=") + if err != nil { + t.Fatalf("hostKeyCallback with pin: %v", err) + } + if cb == nil { + t.Fatal("expected callback for pinned host key") + } +} + +func TestNormalizePinnedHostKey(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty", "", ""}, + {"whitespace only", " ", ""}, + {"openssh form preserved", "SHA256:abc123", "SHA256:abc123"}, + {"hex converted to base64", "00", "SHA256:AA=="}, + {"invalid hex falls through", "nothex", "nothex"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := normalizePinnedHostKey(c.in); got != c.want { + t.Errorf("normalizePinnedHostKey(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +func TestSSHFingerprintSHA256(t *testing.T) { + // Use a fixed test key so the fingerprint is deterministic. + // Generated with ssh-keygen -t ed25519 -f /tmp/test_key -N "" + // and parsed here. We use ssh.ParseAuthorizedKey on a known-good + // public key line. + pubLine := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+HHi0MtRj4VPl8mdP8gniGZDRb0SZTdI2TPxRyCm1H test@example.com" + key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubLine)) + if err != nil { + t.Fatalf("ParseAuthorizedKey: %v", err) + } + fp := sshFingerprintSHA256(key) + if !strings.HasPrefix(fp, "SHA256:") { + t.Errorf("fingerprint should start with SHA256:, got %q", fp) + } + // The fingerprint must be deterministic for the same key. + if fp != sshFingerprintSHA256(key) { + t.Error("fingerprint should be deterministic") + } +} diff --git a/internal/protocol/sftp/sftp.go b/internal/protocol/sftp/sftp.go index 756251a..b641d3e 100644 --- a/internal/protocol/sftp/sftp.go +++ b/internal/protocol/sftp/sftp.go @@ -113,7 +113,7 @@ type client struct { maxPacket uint32 } -func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool) (*client, error) { +func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool, pinnedHostKey string) (*client, error) { host := u.Hostname() port := u.Port() if port == "" { @@ -127,7 +127,7 @@ func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool) addr := net.JoinHostPort(host, port) - hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure) + hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure, pinnedHostKey) if err != nil { return nil, err } @@ -639,7 +639,7 @@ func download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadRes return downloadRecursiveSequential(ctx, req) } - c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure) + c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey) if err != nil { return nil, err } @@ -816,7 +816,7 @@ func newSFTPClientPool(ctx context.Context, req *core.DownloadRequest, size int) all := make([]*client, 0, size) avail := make(chan *client, size) for i := 0; i < size; i++ { - c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure) + c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey) if err != nil { for _, existing := range all { existing.close() @@ -854,7 +854,7 @@ func (p *sftpClientPool) close() { // downloadRecursiveParallel and is enabled when req.RecursiveParallel // is greater than one. func downloadRecursiveSequential(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) { - c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure) + c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey) if err != nil { return nil, err } diff --git a/internal/protocol/sftp/sftp_test.go b/internal/protocol/sftp/sftp_test.go index 5fe2a0a..f8da161 100644 --- a/internal/protocol/sftp/sftp_test.go +++ b/internal/protocol/sftp/sftp_test.go @@ -490,7 +490,7 @@ func TestIsEOF(t *testing.T) { func TestDialTimeout(t *testing.T) { u, _ := url.Parse("sftp://user@127.0.0.1:1/") - _, err := dial(context.Background(), u, "", true) + _, err := dial(context.Background(), u, "", true, "") if err == nil { t.Fatal("expected error") } diff --git a/internal/protocol/webdav/protocol.go b/internal/protocol/webdav/protocol.go index d1d4728..7798b89 100644 --- a/internal/protocol/webdav/protocol.go +++ b/internal/protocol/webdav/protocol.go @@ -246,8 +246,21 @@ func (p *Protocol) getHTTPTransport() *http.Transport { return p.httpTransport } + // Build the TLS config from the caller-provided transport.TLSConfig + // (which carries CA certs, mTLS, pinned-cert hash, keylog, etc.) + // rather than a fresh tls.Config that would silently drop pinning. + var tlsClientCfg *tls.Config + if p.tlsConfig != nil { + if cfg, err := p.tlsConfig.ToTLSConfig(); err == nil { + tlsClientCfg = cfg + } + } + if tlsClientCfg == nil { + tlsClientCfg = &tls.Config{InsecureSkipVerify: p.insecureSkip} //nolint:gosec + } + tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: p.insecureSkip}, //nolint:gosec + TLSClientConfig: tlsClientCfg, DisableCompression: true, Proxy: http.ProxyFromEnvironment, MaxIdleConns: 100,