feat: extend certificate pinning to all TLS protocols and add SSH
host-key pinning
This commit is contained in:
+1
-1
@@ -16,7 +16,7 @@ certificate pinning and metadata debugging to all TLS protocols.
|
||||
|
||||
- [x] **Graceful shutdown for all protocols** — persist `.goget.meta` on
|
||||
SIGINT for FTP single-file and SFTP parallel recursive paths.
|
||||
- [ ] **`--pinned-cert` for all TLS protocols** — extend certificate
|
||||
- [x] **`--pinned-cert` for all TLS protocols** — extend certificate
|
||||
pinning to FTPS, Gemini, and WebDAVS (currently HTTP/HTTPS only).
|
||||
Also add SSH host-key pinning for SFTP as `--pinned-host-key`.
|
||||
- [x] **Unified worker pool** — a single protocol-agnostic worker pool
|
||||
|
||||
@@ -18,6 +18,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt
|
||||
for SFTP single-file downloads, ensuring progress is saved for resume.
|
||||
|
||||
**Security**
|
||||
|
||||
- **Certificate pinning across all TLS protocols** — `--pinned-cert` now
|
||||
applies to FTPS, Gemini, and WebDAVS in addition to HTTP/HTTPS. The leaf
|
||||
certificate SHA-256 hash is verified via `tls.Config.VerifyPeerCertificate`
|
||||
on every TLS-bearing protocol.
|
||||
- **SSH host-key pinning for SFTP** — new `--pinned-host-key` flag pins the
|
||||
SSH host key by its OpenSSH fingerprint (`SHA256:<base64>`, as produced by
|
||||
`ssh-keygen -lf`). Raw lowercase-hex SHA-256 of the key wire format is also
|
||||
accepted. Pinning takes precedence over `--ssh-insecure` and `known_hosts`.
|
||||
- **WebDAV TLS config fix** — `webdav.Protocol.getHTTPTransport` now builds
|
||||
its `tls.Config` from the caller-provided `transport.TLSConfig` instead of
|
||||
a fresh config that silently dropped CA certs, mTLS, keylog, and pinned-cert
|
||||
settings.
|
||||
|
||||
**Recursive downloading & mirroring**
|
||||
|
||||
- `--adjust-extension` extended to non-HTTP protocols — applies `.html`
|
||||
|
||||
@@ -606,6 +606,8 @@ func (a *app) execute() int {
|
||||
Ctx: ctx,
|
||||
SSHKnownHosts: *f.KnownHosts,
|
||||
SSHInsecure: *f.SSHInsecure,
|
||||
PinnedCertHash: *f.PinnedCert,
|
||||
PinnedHostKey: *f.PinnedHostKey,
|
||||
}
|
||||
|
||||
// Configure TLS for WebDAV if supported.
|
||||
|
||||
+3
-1
@@ -150,6 +150,7 @@ type Flags struct {
|
||||
Referer *string
|
||||
RetryDelay *time.Duration
|
||||
PinnedCert *string
|
||||
PinnedHostKey *string
|
||||
AllowPrivate *bool
|
||||
Range *string
|
||||
ConnectTimeout *time.Duration
|
||||
@@ -289,7 +290,8 @@ func defineFlags(fs *flag.FlagSet, cfg *config.Config) *Flags {
|
||||
Compressed: fs.Bool("compressed", false, "Send Accept-Encoding header for auto-decompression"),
|
||||
Referer: fs.String("referer", "", "Send Referer header"),
|
||||
RetryDelay: fs.Duration("retry-delay", 0, "Delay between retry attempts"),
|
||||
PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning"),
|
||||
PinnedCert: fs.String("pinned-cert", "", "SHA-256 hash of server certificate for pinning (TLS protocols: HTTPS, FTPS, Gemini, WebDAVS)"),
|
||||
PinnedHostKey: fs.String("pinned-host-key", "", "SSH host key fingerprint for pinning (SFTP). Accepts 'SHA256:<base64>' (ssh-keygen -lf) or raw hex SHA-256."),
|
||||
AllowPrivate: fs.Bool("allow-private", false, "Allow connections to private/internal IPs (disables SSRF protection)"),
|
||||
Range: fs.String("range", "", "Download only a range of bytes (e.g., 0-1000)"),
|
||||
ConnectTimeout: fs.Duration("connect-timeout", 0, "Timeout for connection establishment"),
|
||||
|
||||
@@ -49,7 +49,8 @@ goget --url <URL> [OPTIONS]
|
||||
| `--dns-servers` | `<IPs>` | Custom DNS servers, comma-separated (e.g. `1.1.1.1,8.8.8.8`) |
|
||||
| `--bind-interface` | `<IFACE>` | Bind outgoing connections to a specific network interface (Linux only) |
|
||||
| `--allow-private` | — | Allow connections to private/internal IPs (disabled by default for SSRF protection) |
|
||||
| `--pinned-cert` | `<SHA256>` | Certificate pinning — abort if server cert SHA-256 doesn't match |
|
||||
| `--pinned-cert` | `<SHA256>` | Certificate pinning (TLS protocols: HTTPS, FTPS, Gemini, WebDAVS) — abort if leaf cert SHA-256 doesn't match |
|
||||
| `--pinned-host-key` | `<FINGERPRINT>` | SSH host-key pinning (SFTP) — accepts `SHA256:<base64>` (`ssh-keygen -lf`) or raw hex SHA-256 |
|
||||
|
||||
## HTTP / Network Flags
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ Custom FTP implementation supporting active and passive modes.
|
||||
- **Explicit TLS (FTPS)** — AUTH TLS command on port 21
|
||||
- **Resume** — REST command for partial transfers
|
||||
- **Directory listing** — For recursive operations
|
||||
- **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match on FTPS connections
|
||||
|
||||
### Limitations
|
||||
|
||||
@@ -95,6 +96,9 @@ Custom FTP implementation supporting active and passive modes.
|
||||
|
||||
# Anonymous (default)
|
||||
--url ftp://anonymous@ftp.example.com/file.zip
|
||||
|
||||
# Certificate pinning (FTPS only)
|
||||
--url ftps://ftp.example.com/file.zip --pinned-cert a1b2c3d4e5f6...
|
||||
```
|
||||
|
||||
## SFTP (`sftp://`)
|
||||
@@ -108,6 +112,7 @@ SFTP over SSH protocol.
|
||||
- **Known hosts verification** — Via `~/.ssh/known_hosts`
|
||||
- **Resume** — Partial transfer support; parallel recursive downloads available via `--recursive-parallel N` with connection pooling
|
||||
- **`known_hosts` verification** — Reads `~/.ssh/known_hosts`; does not write to it. First-connection accept via `--ssh-insecure`
|
||||
- **Host-key pinning** — `--pinned-host-key` enforces an exact SSH host-key match (takes precedence over `--ssh-insecure` and `known_hosts`)
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -123,6 +128,9 @@ SFTP over SSH protocol.
|
||||
|
||||
# With password
|
||||
--url sftp://user@host:/path/to/file.zip --password "secret"
|
||||
|
||||
# Host-key pinning (OpenSSH fingerprint from ssh-keygen -lf)
|
||||
--url sftp://user@host:/path/to/file.zip --pinned-host-key SHA256:AAAAC3NzaC1lZDI1NTE5AAAAIE...
|
||||
```
|
||||
|
||||
## Local File (`file://`)
|
||||
@@ -202,6 +210,7 @@ Implements the Gemini protocol (a lightweight alternative to HTTP/HTTPS).
|
||||
- **Redirect following** — Respects Gemini redirect status codes (3x), up to 10 redirects
|
||||
- **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled
|
||||
- **Text and binary downloads** — 2x success responses stream content directly
|
||||
- **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match
|
||||
|
||||
### Limitations
|
||||
|
||||
@@ -217,6 +226,9 @@ goget --url gemini://gemini.example.com/file.zip
|
||||
|
||||
# Browse a capsule
|
||||
goget --url gemini://gemini.example.com/
|
||||
|
||||
# Pin the server certificate
|
||||
goget --url gemini://gemini.example.com/ --pinned-cert a1b2c3d4e5f6...
|
||||
```
|
||||
|
||||
## Protocol Resolution Flow
|
||||
|
||||
+19
-2
@@ -12,13 +12,30 @@ goget implements multiple security features to protect both the client and serve
|
||||
|
||||
### Certificate Pinning
|
||||
|
||||
Pin a specific certificate by its SHA-256 hash:
|
||||
Pin a specific certificate by its SHA-256 hash. This works across every
|
||||
TLS-bearing protocol goget supports: HTTPS, FTPS, Gemini, and WebDAVS.
|
||||
|
||||
```bash
|
||||
goget --pinned-cert "a1b2c3d4e5f6..." --url https://example.com
|
||||
goget --pinned-cert "a1b2c3d4e5f6..." --url ftps://ftp.example.com/file
|
||||
goget --pinned-cert "a1b2c3d4e5f6..." --url gemini://geminiprotocol.net
|
||||
goget --pinned-cert "a1b2c3d4e5f6..." --url webdavs://dav.example.com/file
|
||||
```
|
||||
|
||||
If the server certificate's SHA-256 doesn't match, the connection is aborted.
|
||||
If the leaf certificate's SHA-256 doesn't match, the connection is aborted.
|
||||
|
||||
### SSH Host-Key Pinning (SFTP)
|
||||
|
||||
Pin the SSH host key for SFTP by its OpenSSH fingerprint (the same value
|
||||
printed by `ssh-keygen -lf ~/.ssh/known_hosts`). Pinning takes precedence
|
||||
over `--ssh-insecure` and the `known_hosts` file.
|
||||
|
||||
```bash
|
||||
goget --pinned-host-key "SHA256:AAAAC3NzaC1lZDI1NTE5AAAAIE+HHi0MtRj4VPl8mdP8gniGZDRb0SZTdI2TPxRyCm1H" \
|
||||
--url sftp://example.com/file
|
||||
```
|
||||
|
||||
The raw lowercase-hex SHA-256 of the key wire format is also accepted.
|
||||
|
||||
### Client Certificates (mTLS)
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ NETWORK
|
||||
--dns-servers <IPs> Custom DNS servers (comma-separated)
|
||||
--bind-interface <IFACE> Bind to specific network interface
|
||||
--allow-private Allow connections to private/internal IPs
|
||||
--pinned-cert <SHA256> Certificate pinning (SHA-256 hash)
|
||||
--pinned-cert <SHA256> Certificate pinning (TLS: HTTPS, FTPS, Gemini, WebDAVS)
|
||||
--pinned-host-key <FP> SSH host-key pinning (SFTP): SHA256:<base64> or hex
|
||||
|
||||
CURL-COMPATIBLE
|
||||
--header "K: V" Custom HTTP header (repeatable)
|
||||
|
||||
@@ -87,6 +87,17 @@ type DownloadRequest struct {
|
||||
Ctx context.Context
|
||||
SSHKnownHosts string // path to known_hosts file (SFTP)
|
||||
SSHInsecure bool // skip SSH host key verification (SFTP)
|
||||
|
||||
// PinnedCertHash pins a TLS leaf certificate by its SHA-256 hash
|
||||
// (lowercase hex). Applied to every TLS-bearing protocol: HTTPS,
|
||||
// FTPS, Gemini, WebDAVS. Empty disables pinning.
|
||||
PinnedCertHash string
|
||||
|
||||
// PinnedHostKey pins an SSH host key. Accepted formats:
|
||||
// "SHA256:<base64>" — OpenSSH fingerprint (ssh-keygen -lf)
|
||||
// "<hex>" — raw SHA-256 hex of the key wire format
|
||||
// Applied to SFTP only. Empty disables pinning.
|
||||
PinnedHostKey string
|
||||
}
|
||||
|
||||
// DownloadResult represents a download result
|
||||
|
||||
@@ -6,7 +6,10 @@ package ftp
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -35,6 +38,11 @@ type Client struct {
|
||||
useTLS bool
|
||||
passive bool
|
||||
timeout time.Duration
|
||||
|
||||
// pinnedCertHash, when non-empty, requires the server's leaf
|
||||
// certificate to match this SHA-256 hash (lowercase hex). Only
|
||||
// meaningful for FTPS.
|
||||
pinnedCertHash string
|
||||
}
|
||||
|
||||
func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
|
||||
@@ -86,6 +94,13 @@ func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// SetPinnedCert configures certificate pinning for FTPS connections.
|
||||
// The expectedHash is a SHA-256 hash of the leaf certificate, expressed
|
||||
// as lowercase hex. Calling this on a plain-FTP client is a no-op.
|
||||
func (c *Client) SetPinnedCert(expectedHash string) {
|
||||
c.pinnedCertHash = strings.ToLower(strings.TrimSpace(expectedHash))
|
||||
}
|
||||
|
||||
func (c *Client) Connect() error {
|
||||
addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port))
|
||||
conn, err := net.DialTimeout("tcp", addr, c.timeout)
|
||||
@@ -130,6 +145,23 @@ func (c *Client) Connect() error {
|
||||
}
|
||||
|
||||
// Wrap connection with TLS
|
||||
if c.pinnedCertHash != "" {
|
||||
expected := c.pinnedCertHash
|
||||
c.tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
for _, rawCert := range rawCerts {
|
||||
cert, err := x509.ParseCertificate(rawCert)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
hash := sha256.Sum256(cert.Raw)
|
||||
if hex.EncodeToString(hash[:]) == expected {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected)
|
||||
}
|
||||
}
|
||||
|
||||
tlsConn := tls.Client(c.conn, c.tlsConfig)
|
||||
if err := tlsConn.Handshake(); err != nil {
|
||||
return fmt.Errorf("tls handshake failed: %w", err)
|
||||
@@ -647,7 +679,7 @@ type clientPool struct {
|
||||
|
||||
// newClientPool opens `size` independent FTP connections to the same
|
||||
// server. Connections are kept open until close() is called.
|
||||
func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) {
|
||||
func newClientPool(rawURL string, timeout time.Duration, size int, pinnedCertHash string) (*clientPool, error) {
|
||||
if size < 1 {
|
||||
size = 1
|
||||
}
|
||||
@@ -661,6 +693,9 @@ func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool,
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if pinnedCertHash != "" {
|
||||
c.SetPinnedCert(pinnedCertHash)
|
||||
}
|
||||
if err := c.Connect(); err != nil {
|
||||
for _, existing := range all {
|
||||
_ = existing.Close()
|
||||
@@ -774,11 +809,11 @@ func ftpTaskHandler(cpool *clientPool, progressFunc func(file string, current, t
|
||||
// on the server side. The returned error is the first error encountered
|
||||
// by any worker; other workers continue in the background until they
|
||||
// hit ctx cancellation or finish their tasks.
|
||||
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error {
|
||||
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, pinnedCertHash string, progressFunc func(file string, current, total int64)) error {
|
||||
if parallel < 1 {
|
||||
parallel = 1
|
||||
}
|
||||
cpool, err := newClientPool(rawURL, timeout, parallel)
|
||||
cpool, err := newClientPool(rawURL, timeout, parallel, pinnedCertHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -380,3 +380,21 @@ func TestNewClientTLSConfig(t *testing.T) {
|
||||
t.Errorf("tlsConfig.InsecureSkipVerify should be false by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPinnedCert(t *testing.T) {
|
||||
client, err := NewClient("ftps://secure.example.com/file.txt", 30*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() returned error: %v", err)
|
||||
}
|
||||
|
||||
// Pinning is off by default.
|
||||
if client.pinnedCertHash != "" {
|
||||
t.Errorf("pinnedCertHash should default to empty, got %q", client.pinnedCertHash)
|
||||
}
|
||||
|
||||
// SetPinnedCert normalises to lowercase and trims whitespace.
|
||||
client.SetPinnedCert(" ABCDEF0123456789 ")
|
||||
if client.pinnedCertHash != "abcdef0123456789" {
|
||||
t.Errorf("pinnedCertHash = %q, want %q", client.pinnedCertHash, "abcdef0123456789")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,6 +79,9 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co
|
||||
if err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("failed to create FTP client: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
if req.PinnedCertHash != "" {
|
||||
client.SetPinnedCert(req.PinnedCertHash)
|
||||
}
|
||||
|
||||
if err := client.Connect(); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("failed to connect: %v", err), nil, core.SafeURL(req.URL))
|
||||
@@ -169,7 +172,7 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co
|
||||
// opens N control connections to avoid stepping on CWD state and
|
||||
// to keep the existing client-side protocol serialisation.
|
||||
if req.RecursiveParallel > 1 {
|
||||
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, progressCallback); err != nil {
|
||||
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, req.PinnedCertHash, progressCallback); err != nil {
|
||||
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -6,7 +6,10 @@ package gemini
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -63,6 +66,27 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
|
||||
InsecureSkipVerify: p.TLSInsecure,
|
||||
}
|
||||
|
||||
// Certificate pinning: require the leaf certificate to match the
|
||||
// caller-provided SHA-256 hash. The check runs after the standard
|
||||
// verification chain so a pinned self-signed cert still needs
|
||||
// InsecureSkipVerify (or a matching CA) to clear the normal path.
|
||||
if req != nil && req.PinnedCertHash != "" {
|
||||
expected := strings.ToLower(strings.TrimSpace(req.PinnedCertHash))
|
||||
tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
for _, rawCert := range rawCerts {
|
||||
cert, err := x509.ParseCertificate(rawCert)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
hash := sha256.Sum256(cert.Raw)
|
||||
if hex.EncodeToString(hash[:]) == expected {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected)
|
||||
}
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{Timeout: 30 * time.Second}
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsCfg)
|
||||
if err != nil {
|
||||
@@ -123,11 +147,12 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
|
||||
}
|
||||
// Follow redirect with incremented depth
|
||||
redirectReq := &core.DownloadRequest{
|
||||
URL: resolved,
|
||||
Output: req.Output,
|
||||
Verbose: req.Verbose,
|
||||
Writer: req.Writer,
|
||||
Ctx: ctx,
|
||||
URL: resolved,
|
||||
Output: req.Output,
|
||||
Verbose: req.Verbose,
|
||||
Writer: req.Writer,
|
||||
PinnedCertHash: req.PinnedCertHash,
|
||||
Ctx: ctx,
|
||||
}
|
||||
return p.downloadWithRedirects(ctx, redirectReq, redirectCount+1)
|
||||
|
||||
|
||||
@@ -132,3 +132,41 @@ func TestGeminiPermanentFailure(t *testing.T) {
|
||||
t.Error("expected permanent failure error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeminiPinnedCertMismatch exercises the certificate-pinning path.
|
||||
// The server presents a self-signed cert whose hash does NOT match the
|
||||
// pinned value, so the download must fail with a pinning error.
|
||||
func TestGeminiPinnedCertMismatch(t *testing.T) {
|
||||
cfg := testTLSConfig()
|
||||
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
go func() {
|
||||
conn, _ := listener.Accept()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
r := bufio.NewReader(conn)
|
||||
r.ReadString('\n')
|
||||
conn.Write([]byte("20 text/gemini\r\nhello\n"))
|
||||
}()
|
||||
|
||||
proto := NewProtocol()
|
||||
proto.TLSInsecure = true // accept self-signed, then let pinning reject
|
||||
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
|
||||
_, err = proto.Download(context.Background(), &core.DownloadRequest{
|
||||
URL: u,
|
||||
PinnedCertHash: "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected pinning failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pinning failed") {
|
||||
t.Errorf("expected pinning error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,15 +4,39 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, error) {
|
||||
// hostKeyCallback builds the ssh.HostKeyCallback used for dialing.
|
||||
//
|
||||
// Precedence:
|
||||
//
|
||||
// 1. pinnedHostKey — when set, only a host whose key matches the
|
||||
// given fingerprint is accepted. known_hosts and --ssh-insecure
|
||||
// are both ignored because pinning is a stricter guarantee.
|
||||
// 2. insecure --ssh-insecure skips verification entirely.
|
||||
// 3. known_hosts file (default ~/.ssh/known_hosts).
|
||||
func hostKeyCallback(knownHostsPath string, insecure bool, pinnedHostKey string) (ssh.HostKeyCallback, error) {
|
||||
pinned := normalizePinnedHostKey(pinnedHostKey)
|
||||
if pinned != "" {
|
||||
return func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||
actual := sshFingerprintSHA256(key)
|
||||
if actual == pinned {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("host key pinning failed: server key %s does not match pinned %s", actual, pinned)
|
||||
}, nil
|
||||
}
|
||||
|
||||
if insecure {
|
||||
return ssh.InsecureIgnoreHostKey(), nil
|
||||
}
|
||||
@@ -31,3 +55,60 @@ func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback,
|
||||
|
||||
return knownhosts.New(knownHostsPath)
|
||||
}
|
||||
|
||||
// sshFingerprintSHA256 returns the OpenSSH-style SHA-256 fingerprint
|
||||
// of an SSH public key, e.g. "SHA256:abc123...". This matches the
|
||||
// output of `ssh-keygen -lf <keyfile>`.
|
||||
func sshFingerprintSHA256(key ssh.PublicKey) string {
|
||||
hash := sha256.Sum256(key.Marshal())
|
||||
return "SHA256:" + base64.StdEncoding.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
// normalizePinnedHostKey accepts either the OpenSSH fingerprint form
|
||||
// ("SHA256:<base64>") or a raw lowercase-hex SHA-256 of the key wire
|
||||
// format, and returns the canonical OpenSSH fingerprint form. An empty
|
||||
// input returns an empty string.
|
||||
func normalizePinnedHostKey(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(s, "SHA256:") {
|
||||
return s
|
||||
}
|
||||
// Raw hex form — convert to base64 to match the OpenSSH fingerprint.
|
||||
if decoded := tryHexDecode(s); decoded != nil {
|
||||
return "SHA256:" + base64.StdEncoding.EncodeToString(decoded)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// tryHexDecode decodes a lowercase-or-uppercase hex string. Returns nil
|
||||
// on any error or odd length.
|
||||
func tryHexDecode(s string) []byte {
|
||||
if len(s)%2 != 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(s)/2)
|
||||
for i := 0; i < len(s); i += 2 {
|
||||
hi, ok1 := hexNibble(s[i])
|
||||
lo, ok2 := hexNibble(s[i+1])
|
||||
if !ok1 || !ok2 {
|
||||
return nil
|
||||
}
|
||||
out[i/2] = hi<<4 | lo
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hexNibble(c byte) (byte, bool) {
|
||||
switch {
|
||||
case c >= '0' && c <= '9':
|
||||
return c - '0', true
|
||||
case c >= 'a' && c <= 'f':
|
||||
return c - 'a' + 10, true
|
||||
case c >= 'A' && c <= 'F':
|
||||
return c - 'A' + 10, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
@@ -4,11 +4,14 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func TestHostKeyCallbackInsecure(t *testing.T) {
|
||||
cb, err := hostKeyCallback("", true)
|
||||
cb, err := hostKeyCallback("", true, "")
|
||||
if err != nil {
|
||||
t.Fatalf("hostKeyCallback: %v", err)
|
||||
}
|
||||
@@ -18,8 +21,61 @@ func TestHostKeyCallbackInsecure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHostKeyCallbackMissingFile(t *testing.T) {
|
||||
_, err := hostKeyCallback("/nonexistent/known_hosts", false)
|
||||
_, err := hostKeyCallback("/nonexistent/known_hosts", false, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing known_hosts")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostKeyCallbackPinned(t *testing.T) {
|
||||
// Pinning takes precedence over insecure and known_hosts: the
|
||||
// callback is returned without touching the filesystem.
|
||||
cb, err := hostKeyCallback("/nonexistent/known_hosts", false, "SHA256:abcdefghijklmnopqrstuvwxyz0123456789+/=")
|
||||
if err != nil {
|
||||
t.Fatalf("hostKeyCallback with pin: %v", err)
|
||||
}
|
||||
if cb == nil {
|
||||
t.Fatal("expected callback for pinned host key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePinnedHostKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"empty", "", ""},
|
||||
{"whitespace only", " ", ""},
|
||||
{"openssh form preserved", "SHA256:abc123", "SHA256:abc123"},
|
||||
{"hex converted to base64", "00", "SHA256:AA=="},
|
||||
{"invalid hex falls through", "nothex", "nothex"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := normalizePinnedHostKey(c.in); got != c.want {
|
||||
t.Errorf("normalizePinnedHostKey(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHFingerprintSHA256(t *testing.T) {
|
||||
// Use a fixed test key so the fingerprint is deterministic.
|
||||
// Generated with ssh-keygen -t ed25519 -f /tmp/test_key -N ""
|
||||
// and parsed here. We use ssh.ParseAuthorizedKey on a known-good
|
||||
// public key line.
|
||||
pubLine := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+HHi0MtRj4VPl8mdP8gniGZDRb0SZTdI2TPxRyCm1H test@example.com"
|
||||
key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubLine))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseAuthorizedKey: %v", err)
|
||||
}
|
||||
fp := sshFingerprintSHA256(key)
|
||||
if !strings.HasPrefix(fp, "SHA256:") {
|
||||
t.Errorf("fingerprint should start with SHA256:, got %q", fp)
|
||||
}
|
||||
// The fingerprint must be deterministic for the same key.
|
||||
if fp != sshFingerprintSHA256(key) {
|
||||
t.Error("fingerprint should be deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,7 +113,7 @@ type client struct {
|
||||
maxPacket uint32
|
||||
}
|
||||
|
||||
func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool) (*client, error) {
|
||||
func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool, pinnedHostKey string) (*client, error) {
|
||||
host := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
@@ -127,7 +127,7 @@ func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool)
|
||||
|
||||
addr := net.JoinHostPort(host, port)
|
||||
|
||||
hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure)
|
||||
hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure, pinnedHostKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -639,7 +639,7 @@ func download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadRes
|
||||
return downloadRecursiveSequential(ctx, req)
|
||||
}
|
||||
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -816,7 +816,7 @@ func newSFTPClientPool(ctx context.Context, req *core.DownloadRequest, size int)
|
||||
all := make([]*client, 0, size)
|
||||
avail := make(chan *client, size)
|
||||
for i := 0; i < size; i++ {
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
|
||||
if err != nil {
|
||||
for _, existing := range all {
|
||||
existing.close()
|
||||
@@ -854,7 +854,7 @@ func (p *sftpClientPool) close() {
|
||||
// downloadRecursiveParallel and is enabled when req.RecursiveParallel
|
||||
// is greater than one.
|
||||
func downloadRecursiveSequential(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
|
||||
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -490,7 +490,7 @@ func TestIsEOF(t *testing.T) {
|
||||
|
||||
func TestDialTimeout(t *testing.T) {
|
||||
u, _ := url.Parse("sftp://user@127.0.0.1:1/")
|
||||
_, err := dial(context.Background(), u, "", true)
|
||||
_, err := dial(context.Background(), u, "", true, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
|
||||
@@ -246,8 +246,21 @@ func (p *Protocol) getHTTPTransport() *http.Transport {
|
||||
return p.httpTransport
|
||||
}
|
||||
|
||||
// Build the TLS config from the caller-provided transport.TLSConfig
|
||||
// (which carries CA certs, mTLS, pinned-cert hash, keylog, etc.)
|
||||
// rather than a fresh tls.Config that would silently drop pinning.
|
||||
var tlsClientCfg *tls.Config
|
||||
if p.tlsConfig != nil {
|
||||
if cfg, err := p.tlsConfig.ToTLSConfig(); err == nil {
|
||||
tlsClientCfg = cfg
|
||||
}
|
||||
}
|
||||
if tlsClientCfg == nil {
|
||||
tlsClientCfg = &tls.Config{InsecureSkipVerify: p.insecureSkip} //nolint:gosec
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: p.insecureSkip}, //nolint:gosec
|
||||
TLSClientConfig: tlsClientCfg,
|
||||
DisableCompression: true,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
MaxIdleConns: 100,
|
||||
|
||||
Reference in New Issue
Block a user