feat: extend certificate pinning to all TLS protocols and add SSH

host-key pinning
This commit is contained in:
2026-06-29 19:23:10 +02:00
parent e7ed3a334e
commit fc65ea8340
19 changed files with 355 additions and 25 deletions
+2 -1
View File
@@ -37,7 +37,8 @@ NETWORK
--dns-servers <IPs> Custom DNS servers (comma-separated)
--bind-interface <IFACE> Bind to specific network interface
--allow-private Allow connections to private/internal IPs
--pinned-cert <SHA256> Certificate pinning (SHA-256 hash)
--pinned-cert <SHA256> Certificate pinning (TLS: HTTPS, FTPS, Gemini, WebDAVS)
--pinned-host-key <FP> SSH host-key pinning (SFTP): SHA256:<base64> or hex
CURL-COMPATIBLE
--header "K: V" Custom HTTP header (repeatable)
+11
View File
@@ -87,6 +87,17 @@ type DownloadRequest struct {
Ctx context.Context
SSHKnownHosts string // path to known_hosts file (SFTP)
SSHInsecure bool // skip SSH host key verification (SFTP)
// PinnedCertHash pins a TLS leaf certificate by its SHA-256 hash
// (lowercase hex). Applied to every TLS-bearing protocol: HTTPS,
// FTPS, Gemini, WebDAVS. Empty disables pinning.
PinnedCertHash string
// PinnedHostKey pins an SSH host key. Accepted formats:
// "SHA256:<base64>" — OpenSSH fingerprint (ssh-keygen -lf)
// "<hex>" — raw SHA-256 hex of the key wire format
// Applied to SFTP only. Empty disables pinning.
PinnedHostKey string
}
// DownloadResult represents a download result
+38 -3
View File
@@ -6,7 +6,10 @@ package ftp
import (
"bufio"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"fmt"
"io"
"net"
@@ -35,6 +38,11 @@ type Client struct {
useTLS bool
passive bool
timeout time.Duration
// pinnedCertHash, when non-empty, requires the server's leaf
// certificate to match this SHA-256 hash (lowercase hex). Only
// meaningful for FTPS.
pinnedCertHash string
}
func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
@@ -86,6 +94,13 @@ func NewClient(rawURL string, timeout time.Duration) (*Client, error) {
return c, nil
}
// SetPinnedCert configures certificate pinning for FTPS connections.
// The expectedHash is a SHA-256 hash of the leaf certificate, expressed
// as lowercase hex. Calling this on a plain-FTP client is a no-op.
func (c *Client) SetPinnedCert(expectedHash string) {
c.pinnedCertHash = strings.ToLower(strings.TrimSpace(expectedHash))
}
func (c *Client) Connect() error {
addr := net.JoinHostPort(c.host, fmt.Sprintf("%d", c.port))
conn, err := net.DialTimeout("tcp", addr, c.timeout)
@@ -130,6 +145,23 @@ func (c *Client) Connect() error {
}
// Wrap connection with TLS
if c.pinnedCertHash != "" {
expected := c.pinnedCertHash
c.tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
for _, rawCert := range rawCerts {
cert, err := x509.ParseCertificate(rawCert)
if err != nil {
continue
}
hash := sha256.Sum256(cert.Raw)
if hex.EncodeToString(hash[:]) == expected {
return nil
}
}
return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected)
}
}
tlsConn := tls.Client(c.conn, c.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
return fmt.Errorf("tls handshake failed: %w", err)
@@ -647,7 +679,7 @@ type clientPool struct {
// newClientPool opens `size` independent FTP connections to the same
// server. Connections are kept open until close() is called.
func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool, error) {
func newClientPool(rawURL string, timeout time.Duration, size int, pinnedCertHash string) (*clientPool, error) {
if size < 1 {
size = 1
}
@@ -661,6 +693,9 @@ func newClientPool(rawURL string, timeout time.Duration, size int) (*clientPool,
}
return nil, err
}
if pinnedCertHash != "" {
c.SetPinnedCert(pinnedCertHash)
}
if err := c.Connect(); err != nil {
for _, existing := range all {
_ = existing.Close()
@@ -774,11 +809,11 @@ func ftpTaskHandler(cpool *clientPool, progressFunc func(file string, current, t
// on the server side. The returned error is the first error encountered
// by any worker; other workers continue in the background until they
// hit ctx cancellation or finish their tasks.
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, progressFunc func(file string, current, total int64)) error {
func DownloadDirectoryConcurrent(ctx context.Context, rawURL string, timeout time.Duration, remotePath, localPath string, parallel int, pinnedCertHash string, progressFunc func(file string, current, total int64)) error {
if parallel < 1 {
parallel = 1
}
cpool, err := newClientPool(rawURL, timeout, parallel)
cpool, err := newClientPool(rawURL, timeout, parallel, pinnedCertHash)
if err != nil {
return err
}
+18
View File
@@ -380,3 +380,21 @@ func TestNewClientTLSConfig(t *testing.T) {
t.Errorf("tlsConfig.InsecureSkipVerify should be false by default")
}
}
func TestSetPinnedCert(t *testing.T) {
client, err := NewClient("ftps://secure.example.com/file.txt", 30*time.Second)
if err != nil {
t.Fatalf("NewClient() returned error: %v", err)
}
// Pinning is off by default.
if client.pinnedCertHash != "" {
t.Errorf("pinnedCertHash should default to empty, got %q", client.pinnedCertHash)
}
// SetPinnedCert normalises to lowercase and trims whitespace.
client.SetPinnedCert(" ABCDEF0123456789 ")
if client.pinnedCertHash != "abcdef0123456789" {
t.Errorf("pinnedCertHash = %q, want %q", client.pinnedCertHash, "abcdef0123456789")
}
}
+4 -1
View File
@@ -79,6 +79,9 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co
if err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("failed to create FTP client: %v", err), nil, core.SafeURL(req.URL))
}
if req.PinnedCertHash != "" {
client.SetPinnedCert(req.PinnedCertHash)
}
if err := client.Connect(); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("failed to connect: %v", err), nil, core.SafeURL(req.URL))
@@ -169,7 +172,7 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co
// opens N control connections to avoid stepping on CWD state and
// to keep the existing client-side protocol serialisation.
if req.RecursiveParallel > 1 {
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, progressCallback); err != nil {
if err := DownloadDirectoryConcurrent(ctx, req.URL.String(), timeout, remotePath, outputDir, req.RecursiveParallel, req.PinnedCertHash, progressCallback); err != nil {
return nil, core.NewProtocolError(fmt.Sprintf("directory download failed: %v", err), nil, core.SafeURL(req.URL))
}
} else {
+30 -5
View File
@@ -6,7 +6,10 @@ package gemini
import (
"bufio"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"fmt"
"io"
"net"
@@ -63,6 +66,27 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
InsecureSkipVerify: p.TLSInsecure,
}
// Certificate pinning: require the leaf certificate to match the
// caller-provided SHA-256 hash. The check runs after the standard
// verification chain so a pinned self-signed cert still needs
// InsecureSkipVerify (or a matching CA) to clear the normal path.
if req != nil && req.PinnedCertHash != "" {
expected := strings.ToLower(strings.TrimSpace(req.PinnedCertHash))
tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
for _, rawCert := range rawCerts {
cert, err := x509.ParseCertificate(rawCert)
if err != nil {
continue
}
hash := sha256.Sum256(cert.Raw)
if hex.EncodeToString(hash[:]) == expected {
return nil
}
}
return fmt.Errorf("certificate pinning failed: no certificate matches sha-256 %s", expected)
}
}
dialer := &net.Dialer{Timeout: 30 * time.Second}
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsCfg)
if err != nil {
@@ -123,11 +147,12 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
}
// Follow redirect with incremented depth
redirectReq := &core.DownloadRequest{
URL: resolved,
Output: req.Output,
Verbose: req.Verbose,
Writer: req.Writer,
Ctx: ctx,
URL: resolved,
Output: req.Output,
Verbose: req.Verbose,
Writer: req.Writer,
PinnedCertHash: req.PinnedCertHash,
Ctx: ctx,
}
return p.downloadWithRedirects(ctx, redirectReq, redirectCount+1)
+38
View File
@@ -132,3 +132,41 @@ func TestGeminiPermanentFailure(t *testing.T) {
t.Error("expected permanent failure error")
}
}
// TestGeminiPinnedCertMismatch exercises the certificate-pinning path.
// The server presents a self-signed cert whose hash does NOT match the
// pinned value, so the download must fail with a pinning error.
func TestGeminiPinnedCertMismatch(t *testing.T) {
cfg := testTLSConfig()
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("20 text/gemini\r\nhello\n"))
}()
proto := NewProtocol()
proto.TLSInsecure = true // accept self-signed, then let pinning reject
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
_, err = proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
PinnedCertHash: "0000000000000000000000000000000000000000000000000000000000000000",
})
if err == nil {
t.Fatal("expected pinning failure")
}
if !strings.Contains(err.Error(), "pinning failed") {
t.Errorf("expected pinning error, got: %v", err)
}
}
+82 -1
View File
@@ -4,15 +4,39 @@
package sftp
import (
"crypto/sha256"
"encoding/base64"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback, error) {
// hostKeyCallback builds the ssh.HostKeyCallback used for dialing.
//
// Precedence:
//
// 1. pinnedHostKey — when set, only a host whose key matches the
// given fingerprint is accepted. known_hosts and --ssh-insecure
// are both ignored because pinning is a stricter guarantee.
// 2. insecure --ssh-insecure skips verification entirely.
// 3. known_hosts file (default ~/.ssh/known_hosts).
func hostKeyCallback(knownHostsPath string, insecure bool, pinnedHostKey string) (ssh.HostKeyCallback, error) {
pinned := normalizePinnedHostKey(pinnedHostKey)
if pinned != "" {
return func(_ string, _ net.Addr, key ssh.PublicKey) error {
actual := sshFingerprintSHA256(key)
if actual == pinned {
return nil
}
return fmt.Errorf("host key pinning failed: server key %s does not match pinned %s", actual, pinned)
}, nil
}
if insecure {
return ssh.InsecureIgnoreHostKey(), nil
}
@@ -31,3 +55,60 @@ func hostKeyCallback(knownHostsPath string, insecure bool) (ssh.HostKeyCallback,
return knownhosts.New(knownHostsPath)
}
// sshFingerprintSHA256 returns the OpenSSH-style SHA-256 fingerprint
// of an SSH public key, e.g. "SHA256:abc123...". This matches the
// output of `ssh-keygen -lf <keyfile>`.
func sshFingerprintSHA256(key ssh.PublicKey) string {
hash := sha256.Sum256(key.Marshal())
return "SHA256:" + base64.StdEncoding.EncodeToString(hash[:])
}
// normalizePinnedHostKey accepts either the OpenSSH fingerprint form
// ("SHA256:<base64>") or a raw lowercase-hex SHA-256 of the key wire
// format, and returns the canonical OpenSSH fingerprint form. An empty
// input returns an empty string.
func normalizePinnedHostKey(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
if strings.HasPrefix(s, "SHA256:") {
return s
}
// Raw hex form — convert to base64 to match the OpenSSH fingerprint.
if decoded := tryHexDecode(s); decoded != nil {
return "SHA256:" + base64.StdEncoding.EncodeToString(decoded)
}
return s
}
// tryHexDecode decodes a lowercase-or-uppercase hex string. Returns nil
// on any error or odd length.
func tryHexDecode(s string) []byte {
if len(s)%2 != 0 {
return nil
}
out := make([]byte, len(s)/2)
for i := 0; i < len(s); i += 2 {
hi, ok1 := hexNibble(s[i])
lo, ok2 := hexNibble(s[i+1])
if !ok1 || !ok2 {
return nil
}
out[i/2] = hi<<4 | lo
}
return out
}
func hexNibble(c byte) (byte, bool) {
switch {
case c >= '0' && c <= '9':
return c - '0', true
case c >= 'a' && c <= 'f':
return c - 'a' + 10, true
case c >= 'A' && c <= 'F':
return c - 'A' + 10, true
}
return 0, false
}
+58 -2
View File
@@ -4,11 +4,14 @@
package sftp
import (
"strings"
"testing"
"golang.org/x/crypto/ssh"
)
func TestHostKeyCallbackInsecure(t *testing.T) {
cb, err := hostKeyCallback("", true)
cb, err := hostKeyCallback("", true, "")
if err != nil {
t.Fatalf("hostKeyCallback: %v", err)
}
@@ -18,8 +21,61 @@ func TestHostKeyCallbackInsecure(t *testing.T) {
}
func TestHostKeyCallbackMissingFile(t *testing.T) {
_, err := hostKeyCallback("/nonexistent/known_hosts", false)
_, err := hostKeyCallback("/nonexistent/known_hosts", false, "")
if err == nil {
t.Fatal("expected error for missing known_hosts")
}
}
func TestHostKeyCallbackPinned(t *testing.T) {
// Pinning takes precedence over insecure and known_hosts: the
// callback is returned without touching the filesystem.
cb, err := hostKeyCallback("/nonexistent/known_hosts", false, "SHA256:abcdefghijklmnopqrstuvwxyz0123456789+/=")
if err != nil {
t.Fatalf("hostKeyCallback with pin: %v", err)
}
if cb == nil {
t.Fatal("expected callback for pinned host key")
}
}
func TestNormalizePinnedHostKey(t *testing.T) {
cases := []struct {
name string
in string
want string
}{
{"empty", "", ""},
{"whitespace only", " ", ""},
{"openssh form preserved", "SHA256:abc123", "SHA256:abc123"},
{"hex converted to base64", "00", "SHA256:AA=="},
{"invalid hex falls through", "nothex", "nothex"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := normalizePinnedHostKey(c.in); got != c.want {
t.Errorf("normalizePinnedHostKey(%q) = %q, want %q", c.in, got, c.want)
}
})
}
}
func TestSSHFingerprintSHA256(t *testing.T) {
// Use a fixed test key so the fingerprint is deterministic.
// Generated with ssh-keygen -t ed25519 -f /tmp/test_key -N ""
// and parsed here. We use ssh.ParseAuthorizedKey on a known-good
// public key line.
pubLine := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+HHi0MtRj4VPl8mdP8gniGZDRb0SZTdI2TPxRyCm1H test@example.com"
key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubLine))
if err != nil {
t.Fatalf("ParseAuthorizedKey: %v", err)
}
fp := sshFingerprintSHA256(key)
if !strings.HasPrefix(fp, "SHA256:") {
t.Errorf("fingerprint should start with SHA256:, got %q", fp)
}
// The fingerprint must be deterministic for the same key.
if fp != sshFingerprintSHA256(key) {
t.Error("fingerprint should be deterministic")
}
}
+5 -5
View File
@@ -113,7 +113,7 @@ type client struct {
maxPacket uint32
}
func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool) (*client, error) {
func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool, pinnedHostKey string) (*client, error) {
host := u.Hostname()
port := u.Port()
if port == "" {
@@ -127,7 +127,7 @@ func dial(ctx context.Context, u *url.URL, knownHostsPath string, insecure bool)
addr := net.JoinHostPort(host, port)
hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure)
hostKeyCB, err := hostKeyCallback(knownHostsPath, insecure, pinnedHostKey)
if err != nil {
return nil, err
}
@@ -639,7 +639,7 @@ func download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadRes
return downloadRecursiveSequential(ctx, req)
}
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
if err != nil {
return nil, err
}
@@ -816,7 +816,7 @@ func newSFTPClientPool(ctx context.Context, req *core.DownloadRequest, size int)
all := make([]*client, 0, size)
avail := make(chan *client, size)
for i := 0; i < size; i++ {
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
if err != nil {
for _, existing := range all {
existing.close()
@@ -854,7 +854,7 @@ func (p *sftpClientPool) close() {
// downloadRecursiveParallel and is enabled when req.RecursiveParallel
// is greater than one.
func downloadRecursiveSequential(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure)
c, err := dial(ctx, req.URL, req.SSHKnownHosts, req.SSHInsecure, req.PinnedHostKey)
if err != nil {
return nil, err
}
+1 -1
View File
@@ -490,7 +490,7 @@ func TestIsEOF(t *testing.T) {
func TestDialTimeout(t *testing.T) {
u, _ := url.Parse("sftp://user@127.0.0.1:1/")
_, err := dial(context.Background(), u, "", true)
_, err := dial(context.Background(), u, "", true, "")
if err == nil {
t.Fatal("expected error")
}
+14 -1
View File
@@ -246,8 +246,21 @@ func (p *Protocol) getHTTPTransport() *http.Transport {
return p.httpTransport
}
// Build the TLS config from the caller-provided transport.TLSConfig
// (which carries CA certs, mTLS, pinned-cert hash, keylog, etc.)
// rather than a fresh tls.Config that would silently drop pinning.
var tlsClientCfg *tls.Config
if p.tlsConfig != nil {
if cfg, err := p.tlsConfig.ToTLSConfig(); err == nil {
tlsClientCfg = cfg
}
}
if tlsClientCfg == nil {
tlsClientCfg = &tls.Config{InsecureSkipVerify: p.insecureSkip} //nolint:gosec
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: p.insecureSkip}, //nolint:gosec
TLSClientConfig: tlsClientCfg,
DisableCompression: true,
Proxy: http.ProxyFromEnvironment,
MaxIdleConns: 100,