diff --git a/BACKLOG.md b/BACKLOG.md index ac55d84..818b16a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -25,7 +25,7 @@ certificate pinning and metadata debugging to all TLS protocols. ### 🟡 High Priority -- [ ] **Gemini resume & metadata** — persist `.goget.meta` during Gemini +- [x] **Gemini resume & metadata** — persist `.goget.meta` during Gemini downloads; `--resume` support. Currently Gemini has `Features: []` — it doesn't advertise resume, and the download loop doesn't save progress. diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e6747..e24842d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 concurrency primitive. - **Graceful shutdown on SIGINT** — persist `.goget.meta` sidecar on interrupt for SFTP single-file downloads, ensuring progress is saved for resume. +- **Gemini resume & metadata** — Gemini downloads now persist `.goget.meta` + sidecar on interrupt and delete it on successful completion. `--resume` + continues interrupted Gemini transfers from the saved byte offset. **Security** diff --git a/README.md b/README.md index b191c74..ac156d3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ It is designed to be: - **Parallel chunked downloads** — automatically splits files over 100 MB into concurrent chunks using byte-range requests. - **Resume interrupted transfers** — metadata-driven resume with `.goget.meta` - tracking across HTTP, FTP, SFTP, and WebDAV. + tracking across HTTP, FTP, SFTP, WebDAV, and Gemini. - **Recursive downloading & mirroring** — follow links, filter by pattern or MIME type, limit depth, stay within domain. Full site mirrors with link conversion for offline viewing and `robots.txt` compliance. diff --git a/docs/protocols.md b/docs/protocols.md index 39b23c0..3f0a107 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -211,10 +211,10 @@ Implements the Gemini protocol (a lightweight alternative to HTTP/HTTPS). - **Error handling** — Input prompts (1x) return an error with the prompt text; temporary (4x) and permanent (5x) failures are handled - **Text and binary downloads** — 2x success responses stream content directly - **Certificate pinning** — `--pinned-cert` enforces a SHA-256 leaf-cert match +- **Resume** — `.goget.meta` sidecar tracks partial progress; `--resume` continues interrupted downloads ### Limitations -- No resume - No parallel downloads - No upload (Gemini is download-only) diff --git a/internal/protocol/gemini/gemini.go b/internal/protocol/gemini/gemini.go index bb040bd..1a2d23a 100644 --- a/internal/protocol/gemini/gemini.go +++ b/internal/protocol/gemini/gemini.go @@ -15,11 +15,13 @@ import ( "net" "net/url" "os" + "path/filepath" "strconv" "strings" "time" "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/output" "codeberg.org/petrbalvin/goget/internal/protocol" ) @@ -39,7 +41,7 @@ func NewProtocol() *Protocol { Name: "GEMINI", Scheme: "gemini", DefaultPort: 1965, - Features: []string{}, + Features: []string{"resume"}, }), } } @@ -96,6 +98,14 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download conn.SetDeadline(time.Now().Add(30 * time.Second)) + // Close the connection when the context is cancelled so any + // blocking Read returns immediately instead of waiting for the + // 30-second deadline. + go func() { + <-ctx.Done() + conn.Close() + }() + // Send request: URL + CRLF requestURL := req.URL.String() if _, err := fmt.Fprintf(conn, "%s\r\n", requestURL); err != nil { @@ -178,43 +188,129 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download } // Category 2x: success — continue to read body - var totalRead int64 + var totalRead int64 // total bytes received from server (including skipped) var writer io.Writer + var file *os.File + resumed := false + var startOffset int64 // bytes to skip on resume - if req.Writer != nil { - writer = req.Writer - } else { - writer = io.Discard + // Set up output destination. When req.Output is set and no external + // Writer was provided, we open the file directly. This covers both + // fresh downloads (create) and resume (append). When req.Writer is + // set (caller manages the file), we use it as-is. + if req.Output != "" && req.Output != "-" { + if req.Resume { + canResume, resumeMeta, rerr := output.CanResume(req.Output, req.URL.String()) + if rerr != nil { + if req.Verbose { + fmt.Fprintf(os.Stderr, "[gemini] Resume check failed: %v\n", rerr) + } + output.CleanupResumeFiles(req.Output) + } else if canResume && resumeMeta != nil { + file, rerr = os.OpenFile(req.Output, os.O_APPEND|os.O_WRONLY, 0644) + if rerr != nil { + return nil, core.NewFileError("failed to open file for resume", rerr) + } + defer file.Close() + writer = file + startOffset = resumeMeta.Downloaded + resumed = true + if req.Verbose { + fmt.Fprintf(os.Stderr, "[gemini] Resuming from byte %d\n", startOffset) + } + } + } + if writer == nil && req.Writer == nil { + // Fresh download — create the output file. + dir := filepath.Dir(req.Output) + if dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, core.NewFileError("failed to create output directory", err) + } + } + file, err = os.Create(req.Output) + if err != nil { + return nil, core.NewFileError("failed to create output file", err) + } + defer file.Close() + writer = file + } } + if writer == nil { + if req.Writer != nil { + writer = req.Writer + } else { + writer = io.Discard + } + } + + // Gemini does not support range requests, so resume re-downloads + // the full content and skips the bytes already saved to disk. + var skipped int64 buf := make([]byte, 32*1024) for { select { case <-ctx.Done(): + saveGeminiResume(req, totalRead) return nil, ctx.Err() default: } n, err := reader.Read(buf) if n > 0 { - if _, werr := writer.Write(buf[:n]); werr != nil { - return nil, core.NewFileError("failed to write gemini data", werr) - } totalRead += int64(n) + chunk := buf[:n] + // On resume, discard bytes we already have on disk. + if skipped < startOffset { + remain := startOffset - skipped + if int64(n) <= remain { + skipped += int64(n) + chunk = nil + } else { + skipped += remain + chunk = chunk[int(remain):] + } + } + if len(chunk) > 0 { + if _, werr := writer.Write(chunk); werr != nil { + saveGeminiResume(req, totalRead) + return nil, core.NewFileError("failed to write gemini data", werr) + } + } if req.ProgressCallback != nil { speed := float64(totalRead) / time.Since(startTime).Seconds() req.ProgressCallback(totalRead, -1, speed) } } if err == io.EOF { + // The connection may have been closed by our context + // cancellation goroutine. If so, the EOF is not a genuine + // end-of-data but a side-effect of the cancel. + if ctx.Err() != nil { + saveGeminiResume(req, totalRead) + return nil, ctx.Err() + } break } if err != nil { + saveGeminiResume(req, totalRead) + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, core.NewNetworkError("failed to read gemini response body", err, core.SafeURL(req.URL)) } } + // Download complete — remove resume metadata sidecar. + if req.Output != "" && req.Output != "-" { + output.DeleteResumeMetadata(req.Output) + } + duration := time.Since(startTime) - speed := float64(totalRead) / duration.Seconds() + speed := float64(0) + if duration.Seconds() > 0 { + speed = float64(totalRead) / duration.Seconds() + } return &core.DownloadResult{ BytesDownloaded: totalRead, @@ -224,5 +320,20 @@ func (p *Protocol) downloadWithRedirects(ctx context.Context, req *core.Download IPVersion: 4, Speed: speed, OutputPath: req.Output, + Resumed: resumed, }, nil } + +// saveGeminiResume persists partial Gemini download progress to the +// standard `.goget.meta` sidecar. No-op for stdout writes or zero-byte +// transfers. Gemini does not expose Content-Length in the response +// header, so Total is recorded as 0 (size unknown). +func saveGeminiResume(req *core.DownloadRequest, downloaded int64) { + if req == nil || req.Output == "" || req.Output == "-" || downloaded <= 0 { + return + } + meta := output.NewResumeMetadata(req.URL.String(), "", "", downloaded, 0) + if err := meta.Save(req.Output); err != nil && req.Verbose { + fmt.Fprintf(os.Stderr, "[gemini] Warning: failed to save resume metadata: %v\n", err) + } +} diff --git a/internal/protocol/gemini/gemini_test.go b/internal/protocol/gemini/gemini_test.go index 0d445e4..43362e0 100644 --- a/internal/protocol/gemini/gemini_test.go +++ b/internal/protocol/gemini/gemini_test.go @@ -15,12 +15,14 @@ import ( "math/big" "net" "net/url" + "os" "strconv" "strings" "testing" "time" "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/output" ) type stringWriter struct{ sb *strings.Builder } @@ -170,3 +172,261 @@ func TestGeminiPinnedCertMismatch(t *testing.T) { t.Errorf("expected pinning error, got: %v", err) } } + +// TestGeminiResumeSaveMetadata verifies that partial download progress is +// persisted to a .goget.meta sidecar when the context is cancelled. +func TestGeminiResumeSaveMetadata(t *testing.T) { + cfg := testTLSConfig() + listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + // Send data in small pieces with delays so the download is still in + // progress when the context is cancelled. + body := make([]byte, 2000) + for i := range body { + body[i] = 'A' + } + go func() { + conn, _ := listener.Accept() + if conn == nil { + return + } + defer conn.Close() + r := bufio.NewReader(conn) + r.ReadString('\n') + conn.Write([]byte("20 application/octet-stream\r\n")) + // Send 500 bytes per chunk with 100ms delays. + for off := 0; off < len(body); off += 500 { + end := off + 500 + if end > len(body) { + end = len(body) + } + conn.Write(body[off:end]) + time.Sleep(100 * time.Millisecond) + } + }() + + tmp := t.TempDir() + outPath := tmp + "/partial.bin" + + proto := NewProtocol() + proto.TLSInsecure = true + u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/") + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, err := proto.Download(ctx, &core.DownloadRequest{ + URL: u, + Output: outPath, + }) + errCh <- err + }() + + // Cancel mid-download — the slow server keeps the connection open + // long enough for the cancellation to fire between reads. + time.Sleep(150 * time.Millisecond) + cancel() + + err = <-errCh + if err == nil { + t.Fatal("expected error from cancelled context") + } + + // Verify resume metadata was saved. + metaPath := outPath + output.ResumeMetaFileSuffix + if _, err := os.Stat(metaPath); os.IsNotExist(err) { + t.Fatal("expected .goget.meta sidecar to exist") + } + + meta, err := output.LoadResumeMetadata(outPath) + if err != nil { + t.Fatalf("LoadResumeMetadata: %v", err) + } + if meta == nil { + t.Fatal("expected resume metadata, got nil") + } + if meta.Downloaded <= 0 { + t.Errorf("Downloaded = %d, want > 0", meta.Downloaded) + } + if meta.URL != u.String() { + t.Errorf("URL = %q, want %q", meta.URL, u.String()) + } +} + +// TestGeminiResumeDownload verifies end-to-end resume: a first download is +// interrupted, and the second download picks up from the saved offset. +func TestGeminiResumeDownload(t *testing.T) { + cfg := testTLSConfig() + listener, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + fullBody := make([]byte, 1000) + for i := range fullBody { + fullBody[i] = byte('A' + i%26) + } + cancelReady := make(chan struct{}) // client signals it has cancelled + connReady := make(chan struct{}) // server signals connection is done + go func() { + for i := 0; i < 2; i++ { + conn, lerr := listener.Accept() + if lerr != nil { + return + } + r := bufio.NewReader(conn) + r.ReadString('\n') + conn.Write([]byte("20 application/octet-stream\r\n")) + if i == 0 { + // First connection: send 500 bytes, then wait + // for the client to cancel before closing. + conn.Write(fullBody[:500]) + <-cancelReady + conn.Close() + connReady <- struct{}{} + } else { + // Second connection: send the full body. + conn.Write(fullBody) + conn.Close() + } + } + }() + + tmp := t.TempDir() + outPath := tmp + "/resume.bin" + + proto := NewProtocol() + proto.TLSInsecure = true + u, _ := url.Parse("gemini://127.0.0.1:" + strconv.Itoa(port) + "/data") + + // First attempt: the server sends 500 bytes and blocks. We cancel + // the context, then the server closes the connection. + ctx1, cancel1 := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, dlErr := proto.Download(ctx1, &core.DownloadRequest{ + URL: u, + Output: outPath, + }) + errCh <- dlErr + }() + time.Sleep(100 * time.Millisecond) + cancel1() + cancelReady <- struct{}{} // let server close connection + <-connReady + dlErr := <-errCh + if dlErr == nil { + t.Fatal("expected error from cancelled context") + } + + // Verify partial file and metadata exist. + info, statErr := os.Stat(outPath) + if statErr != nil { + t.Fatalf("partial file missing: %v", statErr) + } + if info.Size() == 0 { + t.Fatal("partial file is empty") + } + + meta, metaErr := output.LoadResumeMetadata(outPath) + if metaErr != nil || meta == nil { + t.Fatalf("expected resume metadata: err=%v meta=%v", metaErr, meta) + } + + // Second attempt: resume. + result, resErr := proto.Download(context.Background(), &core.DownloadRequest{ + URL: u, + Output: outPath, + Resume: true, + }) + if resErr != nil { + t.Fatalf("resume download failed: %v", resErr) + } + if !result.Resumed { + t.Error("expected Resumed=true") + } + + // The file should now contain the full body. + data, readErr := os.ReadFile(outPath) + if readErr != nil { + t.Fatalf("read output: %v", readErr) + } + if string(data) != string(fullBody) { + t.Errorf("file content length = %d, want %d", len(data), len(fullBody)) + } + + // Resume metadata should be deleted after successful completion. + if output.FileExists(outPath + output.ResumeMetaFileSuffix) { + t.Error("expected .goget.meta to be deleted after successful resume") + } +} + +// TestSaveGeminiResumeMetadata verifies the helper function directly. +func TestSaveGeminiResumeMetadata(t *testing.T) { + tmp := t.TempDir() + out := tmp + "/file.bin" + + u, err := url.Parse("gemini://example.com/file.bin") + if err != nil { + t.Fatalf("parse URL: %v", err) + } + req := &core.DownloadRequest{ + URL: u, + Output: out, + } + + saveGeminiResume(req, 4096) + + meta, err := output.LoadResumeMetadata(out) + if err != nil { + t.Fatalf("LoadResumeMetadata: %v", err) + } + if meta == nil { + t.Fatal("expected resume metadata, got nil") + } + if meta.Downloaded != 4096 { + t.Errorf("Downloaded = %d, want 4096", meta.Downloaded) + } + if meta.URL != "gemini://example.com/file.bin" { + t.Errorf("URL = %q, want %q", meta.URL, "gemini://example.com/file.bin") + } + // Gemini doesn't expose Content-Length, so Total should be 0. + if meta.Total != 0 { + t.Errorf("Total = %d, want 0", meta.Total) + } +} + +// TestSaveGeminiResumeNoop verifies that the helper is a no-op for +// stdout writes, empty output, and zero-byte transfers. +func TestSaveGeminiResumeNoop(t *testing.T) { + u, _ := url.Parse("gemini://example.com/") + + t.Run("stdout", func(t *testing.T) { + saveGeminiResume(&core.DownloadRequest{URL: u, Output: "-"}, 100) + // Should not panic or create any file. + }) + + t.Run("empty output", func(t *testing.T) { + saveGeminiResume(&core.DownloadRequest{URL: u, Output: ""}, 100) + }) + + t.Run("zero bytes", func(t *testing.T) { + tmp := t.TempDir() + saveGeminiResume(&core.DownloadRequest{URL: u, Output: tmp + "/f"}, 0) + if output.FileExists(tmp + "/f" + output.ResumeMetaFileSuffix) { + t.Error("metadata created for zero-byte transfer") + } + }) + + t.Run("nil request", func(t *testing.T) { + saveGeminiResume(nil, 100) + // Should not panic. + }) +}