feat: initial goget release — modern IPv6-first download utility

2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
@@ -0,0 +1,29 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package output
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// CreateTempFile creates a temporary file in the specified directory for atomic writes.
+func CreateTempFile(dir, pattern string) (*os.File, error) {
+	return os.CreateTemp(dir, pattern)
+}
+
+// AtomicReplace atomically renames a temporary file to its final destination.
+func AtomicReplace(tempPath, finalPath string) error {
+	return os.Rename(tempPath, finalPath)
+}
+
+// CleanupTempFile removes a temporary file at the given path.
+func CleanupTempFile(path string) error {
+	return os.Remove(path)
+}
+
+// GetTempDir returns the directory portion of the given path for temporary file placement.
+func GetTempDir(path string) string {
+	return filepath.Dir(path)
+}
@@ -0,0 +1,154 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package output
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+)
+
+// ResumeMetaFileSuffix is the file suffix used for resume metadata files.
+const ResumeMetaFileSuffix = ".goget.meta"
+
+// ResumeMetadata holds metadata for resuming interrupted downloads.
+type ResumeMetadata struct {
+	URL          string        `json:"url"`
+	ETag         string        `json:"etag,omitempty"`
+	LastModified string        `json:"last_modified,omitempty"`
+	Downloaded   int64         `json:"downloaded"`
+	Total        int64         `json:"total,omitempty"`
+	LastWrite    time.Time     `json:"last_write"`
+	Version      string        `json:"version"`
+	Chunks       map[int]int64 `json:"chunks,omitempty"` // per-chunk progress for parallel downloads
+}
+
+// NewResumeMetadata creates a new ResumeMetadata instance with the given parameters.
+func NewResumeMetadata(url, etag, lastModified string, downloaded, total int64) *ResumeMetadata {
+	return &ResumeMetadata{
+		URL:          url,
+		ETag:         etag,
+		LastModified: lastModified,
+		Downloaded:   downloaded,
+		Total:        total,
+		LastWrite:    time.Now(),
+		Version:      "1.0",
+	}
+}
+
+// Save writes the metadata to a JSON file alongside the downloaded file.
+func (rm *ResumeMetadata) Save(outputPath string) error {
+	metaPath := outputPath + ResumeMetaFileSuffix
+	data, err := json.MarshalIndent(rm, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal meta: %w", err)
+	}
+	return os.WriteFile(metaPath, data, 0600)
+}
+
+// LoadResumeMetadata reads resume metadata from the associated JSON file.
+func LoadResumeMetadata(outputPath string) (*ResumeMetadata, error) {
+	metaPath := outputPath + ResumeMetaFileSuffix
+	data, err := os.ReadFile(metaPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to read meta: %w", err)
+	}
+
+	var rm ResumeMetadata
+	if err := json.Unmarshal(data, &rm); err != nil {
+		return nil, fmt.Errorf("failed to parse metadata: %w", err)
+	}
+
+	return &rm, nil
+}
+
+// DeleteResumeMetadata removes the metadata file associated with the given output path.
+func DeleteResumeMetadata(outputPath string) error {
+	metaPath := outputPath + ResumeMetaFileSuffix
+	if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// CanResume checks whether a download can be resumed by validating the metadata file.
+// It returns true only if valid metadata exists and matches the requested URL.
+func CanResume(outputPath string, reqURL string) (bool, *ResumeMetadata, error) {
+	// Check if output file exists
+	info, err := os.Stat(outputPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil, nil
+		}
+		return false, nil, err
+	}
+
+	// Load metadata
+	meta, err := LoadResumeMetadata(outputPath)
+	if err != nil {
+		return false, nil, err
+	}
+
+	// Without valid metadata, resume is not safe (file may be from another download or corrupted)
+	if meta == nil {
+		return false, nil, nil
+	}
+
+	// Verify URL matches
+	if meta.URL != reqURL {
+		return false, nil, fmt.Errorf("url mismatch: metadata for %s, requested %s", meta.URL, reqURL)
+	}
+
+	// Verify file size matches
+	if info.Size() != meta.Downloaded {
+		return false, nil, fmt.Errorf("size mismatch: file has %d bytes, metadata expects %d", info.Size(), meta.Downloaded)
+	}
+
+	return true, meta, nil
+}
+
+// GetResumeInfo creates a ResumeInfo from the metadata for use in download resumption.
+func GetResumeInfo(meta *ResumeMetadata, outputPath string) (*core.ResumeInfo, error) {
+	info, err := os.Stat(outputPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &core.ResumeInfo{
+		DownloadedBytes: meta.Downloaded,
+		ETag:            meta.ETag,
+		LastModified:    meta.LastModified,
+		URL:             meta.URL,
+		LastWrite:       info.ModTime(),
+	}, nil
+}
+
+// FileExists checks whether a file exists at the given path.
+func FileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// GetFileSize returns the size of the file at the given path.
+func GetFileSize(path string) (int64, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	return info.Size(), nil
+}
+
+// CleanupResumeFiles removes the downloaded file and its associated metadata file.
+func CleanupResumeFiles(outputPath string) error {
+	if err := os.Remove(outputPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return DeleteResumeMetadata(outputPath)
+}
@@ -0,0 +1,303 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package output
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// WriterConfig holds configuration for the Writer.
+type WriterConfig struct {
+	// Output is the destination file path (use "-" for stdout).
+	Output string
+
+	// BufferSize is the size of the write buffer.
+	BufferSize int
+
+	// Atomic enables atomic write via temp file + rename.
+	Atomic bool
+
+	// Resume enables support for resuming interrupted downloads.
+	Resume bool
+
+	// ProgressCallback is called on each write with the current progress.
+	ProgressCallback func(current, total int64, speed float64)
+
+	// Verbose enables verbose logging.
+	Verbose bool
+
+	// CreateDirs creates missing parent directories of Output via os.MkdirAll
+	// before opening the file. Ignored when Output is "-" (stdout). The
+	// equivalent of curl --create-dirs / wget --directory-prefix.
+	CreateDirs bool
+}
+
+// DefaultWriterConfig returns a WriterConfig with sensible defaults.
+func DefaultWriterConfig() *WriterConfig {
+	return &WriterConfig{
+		BufferSize: 32 * 1024, // 32KB
+		Atomic:     true,
+		Resume:     false,
+		Verbose:    false,
+	}
+}
+
+// Writer is a progress-aware file writer supporting atomic writes and resume.
+type Writer struct {
+	config    *WriterConfig
+	file      *os.File
+	tempFile  *os.File
+	written   int64
+	total     int64
+	startTime time.Time
+	mu        sync.Mutex
+	callback  func(current, total int64, speed float64)
+	ctx       context.Context
+	cancel    context.CancelFunc
+}
+
+// NewWriter creates a new writer
+func NewWriter(cfg *WriterConfig) (*Writer, error) {
+	if cfg == nil {
+		cfg = DefaultWriterConfig()
+	}
+
+	w := &Writer{
+		config:    cfg,
+		written:   0,
+		startTime: time.Now(),
+		callback:  cfg.ProgressCallback,
+	}
+
+	// Create context for cancellation.
+	w.ctx, w.cancel = context.WithCancel(context.Background())
+
+	// Determine output path
+	outputPath := cfg.Output
+	if outputPath == "" {
+		outputPath = "-" // stdout
+	}
+
+	// Create parent directories on demand (curl --create-dirs / wget
+	// --directory-prefix). Skip for stdout where there is no path.
+	if cfg.CreateDirs && outputPath != "-" {
+		dir := filepath.Dir(outputPath)
+		if dir != "" && dir != "." {
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return nil, fmt.Errorf("failed to create output directory: %w", err)
+			}
+		}
+	}
+
+	// Atomic write - create temp file
+	if cfg.Atomic && outputPath != "-" {
+		dir := filepath.Dir(outputPath)
+		name := filepath.Base(outputPath)
+		tempName := fmt.Sprintf(".%s.tmp.%d", name, time.Now().UnixNano())
+		tempPath := filepath.Join(dir, tempName)
+
+		tempFile, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create temp file: %w", err)
+		}
+
+		w.tempFile = tempFile
+		w.file = tempFile
+	} else if outputPath == "-" {
+		w.file = os.Stdout
+	} else {
+		// Check if file exists for resume.
+		if cfg.Resume {
+			if info, err := os.Stat(outputPath); err == nil {
+				w.written = info.Size()
+				// Open in append mode
+				file, err := os.OpenFile(outputPath, os.O_APPEND|os.O_WRONLY, 0644)
+				if err != nil {
+					return nil, fmt.Errorf("failed to open file for resume: %w", err)
+				}
+				w.file = file
+			} else {
+				// File doesn't exist, create new
+				file, err := os.Create(outputPath)
+				if err != nil {
+					return nil, fmt.Errorf("failed to create file: %w", err)
+				}
+				w.file = file
+			}
+		} else {
+			// Check if file exists
+			if _, err := os.Stat(outputPath); err == nil {
+				return nil, fmt.Errorf("file already exists: %s", outputPath)
+			}
+
+			file, err := os.Create(outputPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create file: %w", err)
+			}
+			w.file = file
+		}
+	}
+
+	return w, nil
+}
+
+// Write implements the io.Writer interface.
+func (w *Writer) Write(p []byte) (int, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	n, err := w.file.Write(p)
+	if err != nil {
+		return n, err
+	}
+
+	w.written += int64(n)
+
+	// Report progress
+	if w.callback != nil {
+		duration := time.Since(w.startTime)
+		speed := float64(w.written) / duration.Seconds()
+		w.callback(w.written, w.total, speed)
+	}
+
+	return n, nil
+}
+
+// SetTotal sets the expected total size for progress tracking.
+func (w *Writer) SetTotal(total int64) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.total = total
+}
+
+// SetProgressCallback sets the callback function for progress reporting.
+func (w *Writer) SetProgressCallback(cb func(current, total int64, speed float64)) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.callback = cb
+}
+
+// Written returns the number of bytes written so far.
+func (w *Writer) Written() int64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.written
+}
+
+// Total returns the expected total size.
+func (w *Writer) Total() int64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.total
+}
+
+// Close finalizes the write and performs the atomic rename if applicable.
+func (w *Writer) Close() error {
+	w.cancel()
+
+	if w.file == nil {
+		return nil
+	}
+
+	// Don't close stdout — it's shared across the process
+	if w.file != os.Stdout {
+		if err := w.file.Close(); err != nil {
+			return err
+		}
+	}
+
+	// Atomic rename if temp file was used
+	if w.tempFile != nil && w.config.Output != "-" {
+		tempPath := w.tempFile.Name()
+		if err := os.Rename(tempPath, w.config.Output); err != nil {
+			if errors.Is(err, syscall.EXDEV) {
+				// Cross-filesystem rename is not supported by the OS.
+				// Fall back to a copy + remove so the data still reach
+				// the destination directory instead of being lost.
+				if fallbackErr := copyFileAndRemove(tempPath, w.config.Output); fallbackErr != nil {
+					return fmt.Errorf("atomic rename failed across filesystems and fallback copy failed: rename=%v copy=%w", err, fallbackErr)
+				}
+				return nil
+			}
+			// Any other error (permission denied, target is a directory,
+			// target busy on Windows, etc.) — keep the temp file so the
+			// user can recover the partial download manually.
+			return fmt.Errorf("failed to rename temp file %q to %q: %w (temp file kept for recovery)", tempPath, w.config.Output, err)
+		}
+	}
+
+	return nil
+}
+
+// copyFileAndRemove copies src to dst (truncating any existing file) and
+// removes src on success. Used as a fallback for os.Rename when the temp
+// file and the destination are on different filesystems.
+func copyFileAndRemove(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	if err := out.Close(); err != nil {
+		return err
+	}
+
+	return os.Remove(src)
+}
+
+// Cancel aborts the write and cleans up any temporary files.
+func (w *Writer) Cancel() {
+	w.cancel()
+
+	if w.tempFile != nil {
+		// Remove temp file on cancel
+		tempPath := w.tempFile.Name()
+		w.tempFile.Close()
+		os.Remove(tempPath)
+	}
+}
+
+// GetSpeed returns the average write speed in bytes per second.
+func (w *Writer) GetSpeed() float64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	duration := time.Since(w.startTime)
+	if duration.Seconds() == 0 {
+		return 0
+	}
+	return float64(w.written) / duration.Seconds()
+}
+
+// GetDuration returns the elapsed time since the writer was created.
+func (w *Writer) GetDuration() time.Duration {
+	return time.Since(w.startTime)
+}
+
+// GetProgress returns the progress as a percentage of the total size.
+func (w *Writer) GetProgress() float64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.total == 0 {
+		return 0
+	}
+	return float64(w.written) / float64(w.total) * 100
+}
@@ -0,0 +1,232 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package output
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestCloseAtomicSuccess verifies the happy path: temp file is renamed to
+// the final destination and no temp file remains.
+func TestCloseAtomicSuccess(t *testing.T) {
+	dir := tempDir(t)
+	outputPath := filepath.Join(dir, "output.txt")
+
+	w, err := NewWriter(&WriterConfig{
+		Output: outputPath,
+		Atomic: true,
+	})
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+
+	payload := []byte("hello world")
+	if _, err := w.Write(payload); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+
+	got, err := os.ReadFile(outputPath)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("payload = %q, want %q", got, payload)
+	}
+
+	// No leftover temp files in the directory.
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("ReadDir: %v", err)
+	}
+	for _, e := range entries {
+		if strings.Contains(e.Name(), ".tmp.") {
+			t.Errorf("temp file %q should have been renamed away", e.Name())
+		}
+	}
+}
+
+// TestCloseKeepsTempOnRenameError is a regression guard for the BACKLOG
+// entry "Atomic write data loss on cross-filesystem rename". The previous
+// implementation called os.Remove(tempPath) on any rename error, deleting
+// the only copy of the downloaded data. After the fix, the temp file must
+// remain on disk so the user can recover the partial download manually.
+//
+// We force a rename error by pre-creating the destination as a directory —
+// os.Rename will refuse to overwrite a non-empty directory with a file.
+func TestCloseKeepsTempOnRenameError(t *testing.T) {
+	dir := tempDir(t)
+	outputPath := filepath.Join(dir, "output.txt")
+
+	// Block the rename by making the target path a directory.
+	if err := os.Mkdir(outputPath, 0755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	w, err := NewWriter(&WriterConfig{
+		Output: outputPath,
+		Atomic: true,
+	})
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+
+	payload := []byte("recoverable download data")
+	if _, err := w.Write(payload); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+
+	tempPath := w.tempFile.Name()
+
+	if err := w.Close(); err == nil {
+		t.Fatal("expected Close() to return an error when target is a directory")
+	}
+
+	// The temp file MUST still exist — the data must be recoverable.
+	if _, err := os.Stat(tempPath); err != nil {
+		t.Errorf("temp file %q should still exist after Close error, got: %v", tempPath, err)
+	}
+
+	// And it should still hold the full payload.
+	got, err := os.ReadFile(tempPath)
+	if err != nil {
+		t.Fatalf("ReadFile(temp): %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("temp file payload = %q, want %q", got, payload)
+	}
+}
+
+// TestCopyFileAndRemove covers the EXDEV fallback helper directly. We do
+// not have an easy way to mount two different filesystems in a unit test,
+// so we just verify the copy + remove behaviour on the same filesystem.
+func TestCopyFileAndRemove(t *testing.T) {
+	dir := tempDir(t)
+	src := filepath.Join(dir, "src.tmp")
+	dst := filepath.Join(dir, "dst.bin")
+
+	payload := []byte("cross-fs fallback payload")
+	if err := os.WriteFile(src, payload, 0644); err != nil {
+		t.Fatalf("WriteFile(src): %v", err)
+	}
+
+	if err := copyFileAndRemove(src, dst); err != nil {
+		t.Fatalf("copyFileAndRemove: %v", err)
+	}
+
+	got, err := os.ReadFile(dst)
+	if err != nil {
+		t.Fatalf("ReadFile(dst): %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("dst = %q, want %q", got, payload)
+	}
+
+	if _, err := os.Stat(src); !os.IsNotExist(err) {
+		t.Errorf("src should be removed, got stat err = %v", err)
+	}
+}
+
+// TestCopyFileAndRemoveOverwritesExisting verifies the fallback truncates
+// any existing file at the destination (matching os.Rename semantics).
+func TestCopyFileAndRemoveOverwritesExisting(t *testing.T) {
+	dir := tempDir(t)
+	src := filepath.Join(dir, "src.tmp")
+	dst := filepath.Join(dir, "dst.bin")
+
+	if err := os.WriteFile(src, []byte("new"), 0644); err != nil {
+		t.Fatalf("WriteFile(src): %v", err)
+	}
+	if err := os.WriteFile(dst, []byte("OLD CONTENT"), 0644); err != nil {
+		t.Fatalf("WriteFile(dst): %v", err)
+	}
+
+	if err := copyFileAndRemove(src, dst); err != nil {
+		t.Fatalf("copyFileAndRemove: %v", err)
+	}
+
+	got, err := os.ReadFile(dst)
+	if err != nil {
+		t.Fatalf("ReadFile(dst): %v", err)
+	}
+	if string(got) != "new" {
+		t.Errorf("dst = %q, want %q", got, "new")
+	}
+}
+
+// TestNewWriterCreateDirsMissingParent verifies the curl --create-dirs
+// equivalent: when CreateDirs is true and the parent directory of Output
+// does not exist, NewWriter must create the full chain via MkdirAll and
+// successfully write the file. This is the happy path for "goget
+// --output ./data/archives/file.zip" against a fresh tree.
+func TestNewWriterCreateDirsMissingParent(t *testing.T) {
+	dir := tempDir(t)
+	outputPath := filepath.Join(dir, "deeply", "nested", "subdir", "file.dat")
+
+	w, err := NewWriter(&WriterConfig{
+		Output:     outputPath,
+		Atomic:     true,
+		CreateDirs: true,
+	})
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+
+	payload := []byte("created in a fresh dir tree")
+	if _, err := w.Write(payload); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+
+	got, err := os.ReadFile(outputPath)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("payload = %q, want %q", got, payload)
+	}
+}
+
+// TestNewWriterCreateDirsDisabledLeavesError confirms that without
+// CreateDirs the writer still surfaces the underlying "parent dir missing"
+// error — CreateDirs is opt-in, not a silent behaviour change for existing
+// users.
+func TestNewWriterCreateDirsDisabledLeavesError(t *testing.T) {
+	dir := tempDir(t)
+	outputPath := filepath.Join(dir, "missing", "file.dat")
+
+	_, err := NewWriter(&WriterConfig{
+		Output:     outputPath,
+		Atomic:     true,
+		CreateDirs: false,
+	})
+	if err == nil {
+		t.Fatal("expected NewWriter to fail when parent dir is missing and CreateDirs=false")
+	}
+}
+
+// TestNewWriterCreateDirsIgnoredForStdout makes sure the - / stdout path is
+// untouched by the new flag: nothing to create, no spurious error.
+func TestNewWriterCreateDirsIgnoredForStdout(t *testing.T) {
+	w, err := NewWriter(&WriterConfig{
+		Output:     "-",
+		Atomic:     false,
+		CreateDirs: true,
+	})
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+	if w.file != os.Stdout {
+		t.Errorf("expected writer to point at stdout, got %v", w.file)
+	}
+	w.Close()
+}