feat: initial goget release — modern IPv6-first download utility

internal/hsts/fuzz_test.go

@@ -0,0 +1,52 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package hsts
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+// FuzzHSTSParseHeader tests HSTS header parsing with fuzzed input.
+func FuzzHSTSParseHeader(f *testing.F) {
+	f.Add("max-age=31536000")
+	f.Add("max-age=31536000; includeSubDomains")
+	f.Add("max-age=0")
+	f.Add("")
+	f.Add("invalid")
+	f.Add("max-age=abc")
+	f.Add("max-age=31536000; includeSubDomains; preload")
+	f.Add("MAX-AGE=31536000")
+
+	f.Fuzz(func(t *testing.T, header string) {
+		_, _ = parseHeader(header)
+	})
+}
+
+// FuzzHSTSLoadSave tests loading and saving HSTS cache with fuzzed data.
+func FuzzHSTSLoadSave(f *testing.F) {
+	f.Add([]byte(`{}`))
+	f.Add([]byte(`{"example.com":{"maxAge":31536000,"includeSubDomains":true,"expires":"2026-01-01T00:00:00Z"}}`))
+	f.Add([]byte(`{invalid`))
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		cache := &Cache{
+			entries: make(map[string]*Entry),
+			path:    t.TempDir() + "/fuzz-hsts.json",
+		}
+
+		// Try loading fuzzed JSON
+		_ = json.Unmarshal(data, &cache.entries)
+
+		// Save to file must not panic
+		tmp := t.TempDir() + "/fuzz-hsts"
+		_ = os.WriteFile(tmp, data, 0644)
+		err := cache.Load(tmp)
+		if err == nil {
+			_ = cache.Save()
+		}
+		os.Remove(tmp)
+	})
+}

internal/hsts/hsts.go

@@ -0,0 +1,226 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+// Package hsts implements HTTP Strict Transport Security (RFC 6797) caching.
+package hsts
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/publicsuffix"
+)
+
+// DefaultCachePath returns the default HSTS cache file path.
+func DefaultCachePath() string {
+	if path := os.Getenv("GOGET_HSTS"); path != "" {
+		return path
+	}
+	if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
+		return filepath.Join(xdg, "goget", "hsts")
+	}
+	if home, err := os.UserHomeDir(); err == nil {
+		return filepath.Join(home, ".config", "goget", "hsts")
+	}
+	return ""
+}
+
+// Entry represents a single HSTS rule for a host.
+type Entry struct {
+	Host              string    `json:"host"`
+	IncludeSubdomains bool      `json:"include_subdomains"`
+	Expires           time.Time `json:"expires"`
+}
+
+// Cache stores HSTS rules persistently.
+type Cache struct {
+	entries map[string]*Entry
+	mu      sync.RWMutex
+	path    string
+}
+
+// NewCache creates an empty HSTS cache.
+func NewCache() *Cache {
+	return &Cache{
+		entries: make(map[string]*Entry),
+	}
+}
+
+// Load reads HSTS entries from disk.
+func (c *Cache) Load(path string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.path = path
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to read hsts cache: %w", err)
+	}
+
+	var entries []*Entry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return fmt.Errorf("failed to parse hsts cache: %w", err)
+	}
+
+	now := time.Now()
+	for _, e := range entries {
+		if e.Expires.After(now) {
+			c.entries[e.Host] = e
+		}
+	}
+	return nil
+}
+
+// Save writes HSTS entries to disk.
+func (c *Cache) Save() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.path == "" {
+		return nil
+	}
+
+	now := time.Now()
+	var active []*Entry
+	for _, e := range c.entries {
+		if e.Expires.After(now) {
+			active = append(active, e)
+		}
+	}
+
+	if len(active) == 0 {
+		_ = os.Remove(c.path)
+		return nil
+	}
+
+	data, err := json.MarshalIndent(active, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal hsts cache: %w", err)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(c.path), 0755); err != nil {
+		return fmt.Errorf("failed to create hsts directory: %w", err)
+	}
+
+	if err := os.WriteFile(c.path, data, 0644); err != nil {
+		return fmt.Errorf("failed to write hsts cache: %w", err)
+	}
+	return nil
+}
+
+// Apply upgrades http:// to https:// if the host is in the HSTS cache.
+// Returns true if the URL was modified.
+func (c *Cache) Apply(u *url.URL) bool {
+	if u.Scheme != "http" {
+		return false
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	host := strings.ToLower(u.Hostname())
+	if c.matchLocked(host) != nil {
+		u.Scheme = "https"
+		return true
+	}
+	return false
+}
+
+// Update parses Strict-Transport-Security from an HTTPS response and stores it.
+// Per RFC 6797, HSTS headers on non-HTTPS responses are ignored.
+func (c *Cache) Update(resp *http.Response) {
+	if resp == nil || resp.Request == nil || resp.Request.URL.Scheme != "https" {
+		return
+	}
+
+	header := resp.Header.Get("Strict-Transport-Security")
+	if header == "" {
+		return
+	}
+
+	maxAge, includeSubdomains := parseHeader(header)
+	if maxAge <= 0 {
+		return
+	}
+
+	host := strings.ToLower(resp.Request.URL.Hostname())
+	entry := &Entry{
+		Host:              host,
+		IncludeSubdomains: includeSubdomains,
+		Expires:           time.Now().Add(time.Duration(maxAge) * time.Second),
+	}
+
+	c.mu.Lock()
+	c.entries[host] = entry
+	c.mu.Unlock()
+}
+
+// matchLocked finds an entry for the given host (must hold read lock).
+// When includeSubdomains is set on an entry, its rule applies to all
+// subdomains, but only down to the registrable domain boundary
+// (publicsuffix.EffectiveTLDPlusOne). Walking past the registrable
+// domain would compare against a public suffix (e.g. "co.uk"), which
+// RFC 6797 does not authorise and would let a HSTS entry mistakenly
+// stored at the suffix level cover unrelated registrable domains.
+func (c *Cache) matchLocked(host string) *Entry {
+	if e, ok := c.entries[host]; ok {
+		if e.Expires.After(time.Now()) {
+			return e
+		}
+	}
+	// Walk up parent domains, stopping once we step outside the
+	// registrable domain (eTLD+1). We check the public-suffix
+	// boundary *before* entries: a HSTS entry stored at a public
+	// suffix (e.g. "co.uk") must not cover unrelated registrable
+	// domains like "example.co.uk".
+	for {
+		dot := strings.Index(host, ".")
+		if dot < 0 {
+			break
+		}
+		host = host[dot+1:]
+		// If the parent is a public suffix or a single label
+		// (EffectiveTLDPlusOne returns an error), there is no further
+		// in-scope parent to consider.
+		if _, err := publicsuffix.EffectiveTLDPlusOne(host); err != nil {
+			break
+		}
+		if e, ok := c.entries[host]; ok && e.IncludeSubdomains && e.Expires.After(time.Now()) {
+			return e
+		}
+	}
+	return nil
+}
+
+// parseHeader parses the Strict-Transport-Security header value.
+// Returns max-age in seconds and whether includeSubDomains is set.
+func parseHeader(value string) (int64, bool) {
+	var maxAge int64 = -1
+	var includeSubdomains bool
+
+	for _, part := range strings.Split(value, ";") {
+		part = strings.TrimSpace(part)
+		if strings.HasPrefix(strings.ToLower(part), "max-age=") {
+			v := strings.TrimPrefix(part, "max-age=")
+			v = strings.TrimPrefix(v, "max-Age=")
+			v = strings.TrimPrefix(v, "MAX-AGE=")
+			if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+				maxAge = n
+			}
+		} else if strings.EqualFold(part, "includesubdomains") {
+			includeSubdomains = true
+		}
+	}
+	return maxAge, includeSubdomains
+}

internal/hsts/hsts_test.go

@@ -0,0 +1,376 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package hsts
+
+import (
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestParseHeader(t *testing.T) {
+	tests := []struct {
+		input                 string
+		wantMaxAge            int64
+		wantIncludeSubdomains bool
+	}{
+		{"max-age=31536000", 31536000, false},
+		{"max-age=0", 0, false},
+		{"max-age=31536000; includeSubDomains", 31536000, true},
+		{"includeSubDomains; max-age=31536000", 31536000, true},
+		{"max-age=60; includesubdomains", 60, true},
+		{"MAX-AGE=120", 120, false},
+		{"", -1, false},
+		{"max-age=bad", -1, false},
+		{"preload; max-age=100", 100, false},
+	}
+
+	for _, tt := range tests {
+		age, inc := parseHeader(tt.input)
+		if age != tt.wantMaxAge || inc != tt.wantIncludeSubdomains {
+			t.Errorf("parseHeader(%q) = (%d, %v), want (%d, %v)",
+				tt.input, age, inc, tt.wantMaxAge, tt.wantIncludeSubdomains)
+		}
+	}
+}
+
+func TestCacheApply(t *testing.T) {
+	c := NewCache()
+	c.entries["example.com"] = &Entry{
+		Host:              "example.com",
+		IncludeSubdomains: false,
+		Expires:           time.Now().Add(time.Hour),
+	}
+	c.entries["sub.example.com"] = &Entry{
+		Host:              "sub.example.com",
+		IncludeSubdomains: true,
+		Expires:           time.Now().Add(time.Hour),
+	}
+
+	tests := []struct {
+		urlStr     string
+		modified   bool
+		wantScheme string
+	}{
+		{"http://example.com/path", true, "https"},
+		{"https://example.com/path", false, "https"},
+		{"http://other.com/path", false, "http"},
+		{"http://sub.example.com/path", true, "https"},
+		{"http://deep.sub.example.com/path", true, "https"},
+		{"ftp://example.com/path", false, "ftp"},
+	}
+
+	for _, tt := range tests {
+		u, _ := url.Parse(tt.urlStr)
+		got := c.Apply(u)
+		if got != tt.modified {
+			t.Errorf("Apply(%q) modified=%v, want %v", tt.urlStr, got, tt.modified)
+		}
+		if u.Scheme != tt.wantScheme {
+			t.Errorf("Apply(%q) scheme=%q, want %q", tt.urlStr, u.Scheme, tt.wantScheme)
+		}
+	}
+}
+
+func TestCacheUpdate(t *testing.T) {
+	c := NewCache()
+	u, _ := url.Parse("https://example.com/")
+
+	resp := &http.Response{
+		Header:  http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
+		Request: &http.Request{URL: u},
+	}
+	c.Update(resp)
+
+	if len(c.entries) != 1 {
+		t.Fatalf("expected 1 entry, got %d", len(c.entries))
+	}
+	e := c.entries["example.com"]
+	if e == nil {
+		t.Fatal("expected entry for example.com")
+	}
+	if e.IncludeSubdomains {
+		t.Error("expected IncludeSubdomains=false")
+	}
+	if e.Expires.Before(time.Now().Add(59*time.Minute)) || e.Expires.After(time.Now().Add(61*time.Minute)) {
+		t.Errorf("unexpected expires: %v", e.Expires)
+	}
+}
+
+func TestCacheUpdateIgnoresHTTP(t *testing.T) {
+	c := NewCache()
+	u, _ := url.Parse("http://example.com/")
+
+	resp := &http.Response{
+		Header:  http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
+		Request: &http.Request{URL: u},
+	}
+	c.Update(resp)
+
+	if len(c.entries) != 0 {
+		t.Errorf("expected 0 entries for HTTP response, got %d", len(c.entries))
+	}
+}
+
+func TestCacheLoadSave(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hsts")
+
+	c := NewCache()
+	c.entries["example.com"] = &Entry{
+		Host:    "example.com",
+		Expires: time.Now().Add(time.Hour),
+	}
+	c.entries["expired.com"] = &Entry{
+		Host:    "expired.com",
+		Expires: time.Now().Add(-time.Hour),
+	}
+	c.path = path
+
+	if err := c.Save(); err != nil {
+		t.Fatalf("save failed: %v", err)
+	}
+
+	c2 := NewCache()
+	if err := c2.Load(path); err != nil {
+		t.Fatalf("load failed: %v", err)
+	}
+
+	if len(c2.entries) != 1 {
+		t.Errorf("expected 1 active entry after load, got %d", len(c2.entries))
+	}
+	if c2.entries["example.com"] == nil {
+		t.Error("expected example.com entry")
+	}
+	if c2.entries["expired.com"] != nil {
+		t.Error("did not expect expired.com entry")
+	}
+}
+
+func TestCacheSaveRemovesEmpty(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hsts")
+
+	// Create a dummy file
+	if err := os.WriteFile(path, []byte("[]"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	c := NewCache()
+	c.path = path
+	if err := c.Save(); err != nil {
+		t.Fatalf("save failed: %v", err)
+	}
+
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Error("expected cache file to be removed when empty")
+	}
+}
+
+func TestCacheSubdomainMatch(t *testing.T) {
+	c := NewCache()
+	c.entries["example.com"] = &Entry{
+		Host:              "example.com",
+		IncludeSubdomains: true,
+		Expires:           time.Now().Add(time.Hour),
+	}
+
+	u, _ := url.Parse("http://a.b.c.example.com/")
+	if !c.Apply(u) {
+		t.Error("expected deep subdomain to match")
+	}
+	if u.Scheme != "https" {
+		t.Errorf("expected https, got %s", u.Scheme)
+	}
+}
+
+func TestCacheExpiredEntry(t *testing.T) {
+	c := NewCache()
+	c.entries["example.com"] = &Entry{
+		Host:    "example.com",
+		Expires: time.Now().Add(-time.Hour),
+	}
+
+	u, _ := url.Parse("http://example.com/")
+	if c.Apply(u) {
+		t.Error("expected expired entry to not match")
+	}
+}
+
+func TestCacheUpdateOverwrite(t *testing.T) {
+	c := NewCache()
+	u, _ := url.Parse("https://example.com/")
+
+	resp1 := &http.Response{
+		Header:  http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
+		Request: &http.Request{URL: u},
+	}
+	c.Update(resp1)
+
+	resp2 := &http.Response{
+		Header:  http.Header{"Strict-Transport-Security": []string{"max-age=7200; includeSubDomains"}},
+		Request: &http.Request{URL: u},
+	}
+	c.Update(resp2)
+
+	e := c.entries["example.com"]
+	if e == nil {
+		t.Fatal("expected entry")
+	}
+	if !e.IncludeSubdomains {
+		t.Error("expected IncludeSubdomains=true after overwrite")
+	}
+	if e.Expires.Before(time.Now().Add(119*time.Minute)) || e.Expires.After(time.Now().Add(121*time.Minute)) {
+		t.Errorf("unexpected expires after overwrite: %v", e.Expires)
+	}
+}
+
+func TestCacheConcurrent(t *testing.T) {
+	c := NewCache()
+	u, _ := url.Parse("https://example.com/")
+
+	// Concurrent reads and writes should not race
+	for i := 0; i < 100; i++ {
+		go func() {
+			url, _ := url.Parse("http://example.com/")
+			c.Apply(url)
+		}()
+		go func() {
+			resp := &http.Response{
+				Header:  http.Header{"Strict-Transport-Security": []string{"max-age=3600"}},
+				Request: &http.Request{URL: u},
+			}
+			c.Update(resp)
+		}()
+	}
+	time.Sleep(50 * time.Millisecond)
+}
+
+func TestDefaultPath(t *testing.T) {
+	home := t.TempDir()
+	os.Setenv("HOME", home)
+	os.Unsetenv("XDG_CONFIG_HOME")
+
+	path := DefaultCachePath()
+	wantSuffix := filepath.Join(".config", "goget", "hsts")
+	if !strings.HasSuffix(path, wantSuffix) {
+		t.Errorf("default path %q does not end with %q", path, wantSuffix)
+	}
+}
+
+// TestMatchPublicSuffixBoundary is a regression guard for the BACKLOG
+// entry "HSTS matches beyond public suffix boundary". The previous
+// implementation walked parent domains one dot at a time without
+// stopping at the registrable domain (eTLD+1), so a HSTS entry
+// accidentally stored at a public suffix (e.g. "co.uk") would be
+// applied to the unrelated registrable domain "example.co.uk",
+// violating RFC 6797.
+func TestMatchPublicSuffixBoundary(t *testing.T) {
+	// addEntry is a local test helper that stores a HSTS entry
+	// directly in the private map, bypassing the HTTPS-only Update path.
+	addEntry := func(c *Cache, host string, includeSubdomains bool) {
+		c.entries[host] = &Entry{
+			Host:              host,
+			IncludeSubdomains: includeSubdomains,
+			Expires:           time.Now().Add(time.Hour),
+		}
+	}
+
+	tests := []struct {
+		name             string
+		policyHost       string
+		policySubdomains bool
+		requestHost      string
+		expectUpgrade    bool
+	}{
+		// Legitimate: HSTS on the registrable domain applies to its
+		// subdomains.
+		{
+			name:             "subdomain of eTLD+1 is upgraded",
+			policyHost:       "example.co.uk",
+			policySubdomains: true,
+			requestHost:      "www.example.co.uk",
+			expectUpgrade:    true,
+		},
+		{
+			name:             "registrable domain itself is upgraded",
+			policyHost:       "example.co.uk",
+			policySubdomains: true,
+			requestHost:      "example.co.uk",
+			expectUpgrade:    true,
+		},
+
+		// Pathological but possible: an HSTS entry was stored on a
+		// public suffix. The matchLocked walk must NOT let it cover
+		// the unrelated registrable domain.
+		{
+			name:             "public suffix entry does not upgrade unrelated eTLD+1",
+			policyHost:       "co.uk",
+			policySubdomains: true,
+			requestHost:      "example.co.uk",
+			expectUpgrade:    false,
+		},
+		{
+			name:             "public suffix entry does not upgrade further sibling",
+			policyHost:       "uk",
+			policySubdomains: true,
+			requestHost:      "example.co.uk",
+			expectUpgrade:    false,
+		},
+
+		// Same idea at the .com boundary.
+		{
+			name:             "TLD entry does not upgrade unrelated .com domain",
+			policyHost:       "com",
+			policySubdomains: true,
+			requestHost:      "example.com",
+			expectUpgrade:    false,
+		},
+		{
+			name:             "TLD entry does not upgrade further sibling",
+			policyHost:       "com",
+			policySubdomains: true,
+			requestHost:      "anotherexample.com",
+			expectUpgrade:    false,
+		},
+
+		// Subdomains are not set — no upgrade even on the registrable
+		// domain.
+		{
+			name:             "no upgrade without includeSubdomains",
+			policyHost:       "example.com",
+			policySubdomains: false,
+			requestHost:      "www.example.com",
+			expectUpgrade:    false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			cache := NewCache()
+			addEntry(cache, tc.policyHost, tc.policySubdomains)
+
+			u, err := url.Parse("http://" + tc.requestHost + "/")
+			if err != nil {
+				t.Fatalf("parse: %v", err)
+			}
+
+			upgraded := cache.Apply(u)
+			if upgraded != tc.expectUpgrade {
+				t.Errorf("Apply(%q) returned %v, want %v (final scheme=%q)",
+					tc.requestHost, upgraded, tc.expectUpgrade, u.Scheme)
+			}
+			if tc.expectUpgrade && u.Scheme != "https" {
+				t.Errorf("expected https after upgrade, got %q", u.Scheme)
+			}
+			if !tc.expectUpgrade && u.Scheme != "http" {
+				t.Errorf("expected http (no upgrade), got %q", u.Scheme)
+			}
+		})
+	}
+}