feat: add resume support to Gemini protocol

This commit is contained in:
2026-06-29 20:06:08 +02:00
parent fc65ea8340
commit 9ef62520c7
6 changed files with 387 additions and 13 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ certificate pinning and metadata debugging to all TLS protocols.
### 🟡 High Priority ### 🟡 High Priority
- [ ] **Gemini resume & metadata** — persist `.goget.meta` during Gemini - [x] **Gemini resume & metadata** — persist `.goget.meta` during Gemini
downloads; `--resume` support. Currently Gemini has `Features: []` downloads; `--resume` support. Currently Gemini has `Features: []`
it doesn't advertise resume, and the download loop doesn't save it doesn't advertise resume, and the download loop doesn't save
progress. progress.
+3
View File
@@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
concurrency primitive. concurrency primitive.
- **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt - **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt
for SFTP single-file downloads, ensuring progress is saved for resume. for SFTP single-file downloads, ensuring progress is saved for resume.
- **Gemini resume & metadata** — Gemini downloads now persist `.goget.meta`
sidecar on interrupt and delete it on successful completion. `--resume`
continues interrupted Gemini transfers from the saved byte offset.
**Security** **Security**
+1 -1
View File
@@ -40,7 +40,7 @@ It is designed to be:
- **Parallel chunked downloads** — automatically splits files over 100 MB into - **Parallel chunked downloads** — automatically splits files over 100 MB into
concurrent chunks using byte-range requests. concurrent chunks using byte-range requests.
- **Resume interrupted transfers** — metadata-driven resume with `.goget.meta` - **Resume interrupted transfers** — metadata-driven resume with `.goget.meta`
tracking across HTTP, FTP, SFTP, and WebDAV. tracking across HTTP, FTP, SFTP, WebDAV, and Gemini.
- **Recursive downloading & mirroring** — follow links, filter by pattern or - **Recursive downloading & mirroring** — follow links, filter by pattern or
MIME type, limit depth, stay within domain. Full site mirrors with link MIME type, limit depth, stay within domain. Full site mirrors with link
conversion for offline viewing and `robots.txt` compliance. conversion for offline viewing and `robots.txt` compliance.
+1 -1
View File
@@ -211,10 +211,10 @@ Implements the Gemini protocol (a lightweight alternative to HTTP/HTTPS).
- **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled - **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled
- **Text and binary downloads** — 2x success responses stream content directly - **Text and binary downloads** — 2x success responses stream content directly
- **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match - **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match
- **Resume** — `.goget.meta` sidecar tracks partial progress; `--resume` continues interrupted downloads
### Limitations ### Limitations
- No resume
- No parallel downloads - No parallel downloads
- No upload (Gemini is download-only) - No upload (Gemini is download-only)
+116 -5
View File
@@ -15,11 +15,13 @@ import (
"net" "net"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"codeberg.org/petrbalvin/goget/internal/core" "codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
"codeberg.org/petrbalvin/goget/internal/protocol" "codeberg.org/petrbalvin/goget/internal/protocol"
) )
@@ -39,7 +41,7 @@ func NewProtocol() *Protocol {
Name: "GEMINI", Name: "GEMINI",
Scheme: "gemini", Scheme: "gemini",
DefaultPort: 1965, DefaultPort: 1965,
Features: []string{}, Features: []string{"resume"},
}), }),
} }
} }
@@ -96,6 +98,14 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
conn.SetDeadline(time.Now().Add(30 * time.Second)) conn.SetDeadline(time.Now().Add(30 * time.Second))
// Close the connection when the context is cancelled so any
// blocking Read returns immediately instead of waiting for the
// 30-second deadline.
go func() {
<-ctx.Done()
conn.Close()
}()
// Send request: URL + CRLF // Send request: URL + CRLF
requestURL := req.URL.String() requestURL := req.URL.String()
if _, err := fmt.Fprintf(conn, "%s\r\n", requestURL); err != nil { if _, err := fmt.Fprintf(conn, "%s\r\n", requestURL); err != nil {
@@ -178,43 +188,129 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
} }
// Category 2x: success — continue to read body // Category 2x: success — continue to read body
var totalRead int64 var totalRead int64 // total bytes received from server (including skipped)
var writer io.Writer var writer io.Writer
var file *os.File
resumed := false
var startOffset int64 // bytes to skip on resume
// Set up output destination. When req.Output is set and no external
// Writer was provided, we open the file directly. This covers both
// fresh downloads (create) and resume (append). When req.Writer is
// set (caller manages the file), we use it as-is.
if req.Output != "" && req.Output != "-" {
if req.Resume {
canResume, resumeMeta, rerr := output.CanResume(req.Output, req.URL.String())
if rerr != nil {
if req.Verbose {
fmt.Fprintf(os.Stderr, "[gemini] Resume check failed: %v\n", rerr)
}
output.CleanupResumeFiles(req.Output)
} else if canResume && resumeMeta != nil {
file, rerr = os.OpenFile(req.Output, os.O_APPEND|os.O_WRONLY, 0644)
if rerr != nil {
return nil, core.NewFileError("failed to open file for resume", rerr)
}
defer file.Close()
writer = file
startOffset = resumeMeta.Downloaded
resumed = true
if req.Verbose {
fmt.Fprintf(os.Stderr, "[gemini] Resuming from byte %d\n", startOffset)
}
}
}
if writer == nil && req.Writer == nil {
// Fresh download — create the output file.
dir := filepath.Dir(req.Output)
if dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, core.NewFileError("failed to create output directory", err)
}
}
file, err = os.Create(req.Output)
if err != nil {
return nil, core.NewFileError("failed to create output file", err)
}
defer file.Close()
writer = file
}
}
if writer == nil {
if req.Writer != nil { if req.Writer != nil {
writer = req.Writer writer = req.Writer
} else { } else {
writer = io.Discard writer = io.Discard
} }
}
// Gemini does not support range requests, so resume re-downloads
// the full content and skips the bytes already saved to disk.
var skipped int64
buf := make([]byte, 32*1024) buf := make([]byte, 32*1024)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
saveGeminiResume(req, totalRead)
return nil, ctx.Err() return nil, ctx.Err()
default: default:
} }
n, err := reader.Read(buf) n, err := reader.Read(buf)
if n > 0 { if n > 0 {
if _, werr := writer.Write(buf[:n]); werr != nil { totalRead += int64(n)
chunk := buf[:n]
// On resume, discard bytes we already have on disk.
if skipped < startOffset {
remain := startOffset - skipped
if int64(n) <= remain {
skipped += int64(n)
chunk = nil
} else {
skipped += remain
chunk = chunk[int(remain):]
}
}
if len(chunk) > 0 {
if _, werr := writer.Write(chunk); werr != nil {
saveGeminiResume(req, totalRead)
return nil, core.NewFileError("failed to write gemini data", werr) return nil, core.NewFileError("failed to write gemini data", werr)
} }
totalRead += int64(n) }
if req.ProgressCallback != nil { if req.ProgressCallback != nil {
speed := float64(totalRead) / time.Since(startTime).Seconds() speed := float64(totalRead) / time.Since(startTime).Seconds()
req.ProgressCallback(totalRead, -1, speed) req.ProgressCallback(totalRead, -1, speed)
} }
} }
if err == io.EOF { if err == io.EOF {
// The connection may have been closed by our context
// cancellation goroutine. If so, the EOF is not a genuine
// end-of-data but a side-effect of the cancel.
if ctx.Err() != nil {
saveGeminiResume(req, totalRead)
return nil, ctx.Err()
}
break break
} }
if err != nil { if err != nil {
saveGeminiResume(req, totalRead)
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, core.NewNetworkError("failed to read gemini response body", err, core.SafeURL(req.URL)) return nil, core.NewNetworkError("failed to read gemini response body", err, core.SafeURL(req.URL))
} }
} }
// Download complete — remove resume metadata sidecar.
if req.Output != "" && req.Output != "-" {
output.DeleteResumeMetadata(req.Output)
}
duration := time.Since(startTime) duration := time.Since(startTime)
speed := float64(totalRead) / duration.Seconds() speed := float64(0)
if duration.Seconds() > 0 {
speed = float64(totalRead) / duration.Seconds()
}
return &core.DownloadResult{ return &core.DownloadResult{
BytesDownloaded: totalRead, BytesDownloaded: totalRead,
@@ -224,5 +320,20 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download
IPVersion: 4, IPVersion: 4,
Speed: speed, Speed: speed,
OutputPath: req.Output, OutputPath: req.Output,
Resumed: resumed,
}, nil }, nil
} }
// saveGeminiResume persists partial Gemini download progress to the
// standard `.goget.meta` sidecar. No-op for stdout writes or zero-byte
// transfers. Gemini does not expose Content-Length in the response
// header, so Total is recorded as 0 (size unknown).
func saveGeminiResume(req *core.DownloadRequest, downloaded int64) {
if req == nil || req.Output == "" || req.Output == "-" || downloaded <= 0 {
return
}
meta := output.NewResumeMetadata(req.URL.String(), "", "", downloaded, 0)
if err := meta.Save(req.Output); err != nil && req.Verbose {
fmt.Fprintf(os.Stderr, "[gemini] Warning: failed to save resume metadata: %v\n", err)
}
}
+260
View File
@@ -15,12 +15,14 @@ import (
"math/big" "math/big"
"net" "net"
"net/url" "net/url"
"os"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
"codeberg.org/petrbalvin/goget/internal/core" "codeberg.org/petrbalvin/goget/internal/core"
"codeberg.org/petrbalvin/goget/internal/output"
) )
type stringWriter struct{ sb *strings.Builder } type stringWriter struct{ sb *strings.Builder }
@@ -170,3 +172,261 @@ func TestGeminiPinnedCertMismatch(t *testing.T) {
t.Errorf("expected pinning error, got: %v", err) t.Errorf("expected pinning error, got: %v", err)
} }
} }
// TestGeminiResumeSaveMetadata verifies that partial download progress is
// persisted to a .goget.meta sidecar when the context is cancelled.
func TestGeminiResumeSaveMetadata(t *testing.T) {
cfg := testTLSConfig()
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
// Send data in small pieces with delays so the download is still in
// progress when the context is cancelled.
body := make([]byte, 2000)
for i := range body {
body[i] = 'A'
}
go func() {
conn, _ := listener.Accept()
if conn == nil {
return
}
defer conn.Close()
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("20 application/octet-stream\r\n"))
// Send 500 bytes per chunk with 100ms delays.
for off := 0; off < len(body); off += 500 {
end := off + 500
if end > len(body) {
end = len(body)
}
conn.Write(body[off:end])
time.Sleep(100 * time.Millisecond)
}
}()
tmp := t.TempDir()
outPath := tmp + "/partial.bin"
proto := NewProtocol()
proto.TLSInsecure = true
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/")
ctx, cancel := context.WithCancel(context.Background())
errCh := make(chan error, 1)
go func() {
_, err := proto.Download(ctx, &core.DownloadRequest{
URL: u,
Output: outPath,
})
errCh <- err
}()
// Cancel mid-download — the slow server keeps the connection open
// long enough for the cancellation to fire between reads.
time.Sleep(150 * time.Millisecond)
cancel()
err = <-errCh
if err == nil {
t.Fatal("expected error from cancelled context")
}
// Verify resume metadata was saved.
metaPath := outPath + output.ResumeMetaFileSuffix
if _, err := os.Stat(metaPath); os.IsNotExist(err) {
t.Fatal("expected .goget.meta sidecar to exist")
}
meta, err := output.LoadResumeMetadata(outPath)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata, got nil")
}
if meta.Downloaded <= 0 {
t.Errorf("Downloaded = %d, want > 0", meta.Downloaded)
}
if meta.URL != u.String() {
t.Errorf("URL = %q, want %q", meta.URL, u.String())
}
}
// TestGeminiResumeDownload verifies end-to-end resume: a first download is
// interrupted, and the second download picks up from the saved offset.
func TestGeminiResumeDownload(t *testing.T) {
cfg := testTLSConfig()
listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
port := listener.Addr().(*net.TCPAddr).Port
fullBody := make([]byte, 1000)
for i := range fullBody {
fullBody[i] = byte('A' + i%26)
}
cancelReady := make(chan struct{}) // client signals it has cancelled
connReady := make(chan struct{}) // server signals connection is done
go func() {
for i := 0; i < 2; i++ {
conn, lerr := listener.Accept()
if lerr != nil {
return
}
r := bufio.NewReader(conn)
r.ReadString('\n')
conn.Write([]byte("20 application/octet-stream\r\n"))
if i == 0 {
// First connection: send 500 bytes, then wait
// for the client to cancel before closing.
conn.Write(fullBody[:500])
<-cancelReady
conn.Close()
connReady <- struct{}{}
} else {
// Second connection: send the full body.
conn.Write(fullBody)
conn.Close()
}
}
}()
tmp := t.TempDir()
outPath := tmp + "/resume.bin"
proto := NewProtocol()
proto.TLSInsecure = true
u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/data")
// First attempt: the server sends 500 bytes and blocks. We cancel
// the context, then the server closes the connection.
ctx1, cancel1 := context.WithCancel(context.Background())
errCh := make(chan error, 1)
go func() {
_, dlErr := proto.Download(ctx1, &core.DownloadRequest{
URL: u,
Output: outPath,
})
errCh <- dlErr
}()
time.Sleep(100 * time.Millisecond)
cancel1()
cancelReady <- struct{}{} // let server close connection
<-connReady
dlErr := <-errCh
if dlErr == nil {
t.Fatal("expected error from cancelled context")
}
// Verify partial file and metadata exist.
info, statErr := os.Stat(outPath)
if statErr != nil {
t.Fatalf("partial file missing: %v", statErr)
}
if info.Size() == 0 {
t.Fatal("partial file is empty")
}
meta, metaErr := output.LoadResumeMetadata(outPath)
if metaErr != nil || meta == nil {
t.Fatalf("expected resume metadata: err=%v meta=%v", metaErr, meta)
}
// Second attempt: resume.
result, resErr := proto.Download(context.Background(), &core.DownloadRequest{
URL: u,
Output: outPath,
Resume: true,
})
if resErr != nil {
t.Fatalf("resume download failed: %v", resErr)
}
if !result.Resumed {
t.Error("expected Resumed=true")
}
// The file should now contain the full body.
data, readErr := os.ReadFile(outPath)
if readErr != nil {
t.Fatalf("read output: %v", readErr)
}
if string(data) != string(fullBody) {
t.Errorf("file content length = %d, want %d", len(data), len(fullBody))
}
// Resume metadata should be deleted after successful completion.
if output.FileExists(outPath + output.ResumeMetaFileSuffix) {
t.Error("expected .goget.meta to be deleted after successful resume")
}
}
// TestSaveGeminiResumeMetadata verifies the helper function directly.
func TestSaveGeminiResumeMetadata(t *testing.T) {
tmp := t.TempDir()
out := tmp + "/file.bin"
u, err := url.Parse("gemini://example.com/file.bin")
if err != nil {
t.Fatalf("parse URL: %v", err)
}
req := &core.DownloadRequest{
URL: u,
Output: out,
}
saveGeminiResume(req, 4096)
meta, err := output.LoadResumeMetadata(out)
if err != nil {
t.Fatalf("LoadResumeMetadata: %v", err)
}
if meta == nil {
t.Fatal("expected resume metadata, got nil")
}
if meta.Downloaded != 4096 {
t.Errorf("Downloaded = %d, want 4096", meta.Downloaded)
}
if meta.URL != "gemini://example.com/file.bin" {
t.Errorf("URL = %q, want %q", meta.URL, "gemini://example.com/file.bin")
}
// Gemini doesn't expose Content-Length, so Total should be 0.
if meta.Total != 0 {
t.Errorf("Total = %d, want 0", meta.Total)
}
}
// TestSaveGeminiResumeNoop verifies that the helper is a no-op for
// stdout writes, empty output, and zero-byte transfers.
func TestSaveGeminiResumeNoop(t *testing.T) {
u, _ := url.Parse("gemini://example.com/")
t.Run("stdout", func(t *testing.T) {
saveGeminiResume(&core.DownloadRequest{URL: u, Output: "-"}, 100)
// Should not panic or create any file.
})
t.Run("empty output", func(t *testing.T) {
saveGeminiResume(&core.DownloadRequest{URL: u, Output: ""}, 100)
})
t.Run("zero bytes", func(t *testing.T) {
tmp := t.TempDir()
saveGeminiResume(&core.DownloadRequest{URL: u, Output: tmp + "/f"}, 0)
if output.FileExists(tmp + "/f" + output.ResumeMetaFileSuffix) {
t.Error("metadata created for zero-byte transfer")
}
})
t.Run("nil request", func(t *testing.T) {
saveGeminiResume(nil, 100)
// Should not panic.
})
}