diff --git a/BACKLOG.md b/BACKLOG.md index 818b16a..7c4bf01 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -29,7 +29,7 @@ certificate pinning and metadata debugging to all TLS protocols. downloads; `--resume` support. Currently Gemini has `Features: []` — it doesn't advertise resume, and the download loop doesn't save progress. -- [ ] **Gopher resume & metadata** — same as Gemini. Gopher `typeHTML` +- [x] **Gopher resume & metadata** — same as Gemini. Gopher `typeHTML` and `typeTextFile` responses should save `.goget.meta` for resume. ### 🟢 Medium Priority diff --git a/CHANGELOG.md b/CHANGELOG.md index edfaa52..884bca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Gemini resume & metadata** — Gemini downloads now persist `.goget.meta` sidecar on interrupt and delete it on successful completion. `--resume` continues interrupted Gemini transfers from the saved byte offset. +- **Gopher resume & metadata** — Gopher `typeHTML` and `typeTextFile` + responses now persist `.goget.meta` sidecar on interrupt and delete it on + successful completion. `--resume` continues interrupted Gopher text + transfers, skipping already-written output bytes (post-transform length + because Gopher line metadata is stripped before writing). **Security** diff --git a/internal/protocol/gopher/gopher.go b/internal/protocol/gopher/gopher.go index 84a65f4..e79f5c4 100644 --- a/internal/protocol/gopher/gopher.go +++ b/internal/protocol/gopher/gopher.go @@ -9,10 +9,13 @@ import ( "fmt" "io" "net" + "os" + "path/filepath" "strings" "time" "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/output" "codeberg.org/petrbalvin/goget/internal/protocol" ) @@ -43,7 +46,7 @@ func NewProtocol() *Protocol { Name: "GOPHER", Scheme: "gopher", DefaultPort: 70, - Features: []string{}, + Features: []string{"resume"}, }), } } @@ -72,6 +75,14 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co defer conn.Close() conn.SetDeadline(time.Now().Add(30 * time.Second)) + // Close the connection when the context is cancelled so any + // blocking Read returns immediately instead of waiting for the + // 30-second deadline. + go func() { + <-ctx.Done() + conn.Close() + }() + // Send selector + CRLF if _, err := fmt.Fprintf(conn, "%s\r\n", selector); err != nil { return nil, core.NewNetworkError("failed to send gopher selector", err, core.SafeURL(req.URL)) @@ -138,20 +149,81 @@ func isTextGopherType(t byte) bool { } // downloadText handles text responses where each line has a type prefix. +// Supports resume for typeHTML and typeTextFile responses: when interrupted, +// writes a `.goget.meta` sidecar; on resume with valid metadata, skips +// already-written output bytes by length. Gopher does not expose +// Content-Length, so the resume metadata records only the bytes +// actually delivered to the writer. func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time, firstType byte) (*core.DownloadResult, error) { var writer io.Writer - if req.Writer != nil { - writer = req.Writer - } else { - writer = io.Discard + var file *os.File + resumed := false + var startOffset int64 + + // Set up output destination. When req.Output is set and no external + // Writer was provided, we open the file directly. This covers both + // fresh downloads (create) and resume (append). When req.Writer is + // set (caller manages the file), we use it as-is. + if req.Output != "" && req.Output != "-" { + if req.Resume { + canResume, resumeMeta, rerr := output.CanResume(req.Output, req.URL.String()) + if rerr != nil { + if req.Verbose { + fmt.Fprintf(os.Stderr, "[gopher] Resume check failed: %v\n", rerr) + } + output.CleanupResumeFiles(req.Output) + } else if canResume && resumeMeta != nil { + file, rerr = os.OpenFile(req.Output, os.O_APPEND|os.O_WRONLY, 0644) + if rerr != nil { + return nil, core.NewFileError("failed to open file for resume", rerr) + } + defer file.Close() + writer = file + startOffset = resumeMeta.Downloaded + resumed = true + if req.Verbose { + fmt.Fprintf(os.Stderr, "[gopher] Resuming from byte %d\n", startOffset) + } + } + } + if writer == nil && req.Writer == nil { + // Fresh download — create the output file. + dir := filepath.Dir(req.Output) + if dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, core.NewFileError("failed to create output directory", err) + } + } + var ferr error + file, ferr = os.Create(req.Output) + if ferr != nil { + return nil, core.NewFileError("failed to create output file", ferr) + } + defer file.Close() + writer = file + } + } + if writer == nil { + if req.Writer != nil { + writer = req.Writer + } else { + writer = io.Discard + } } - var totalRead int64 + // bytesWritten tracks output payload actually delivered to the writer + // (after the per-line metadata strip). On resume we skip ahead by + // startOffset bytes of this post-transform stream so the file on + // disk ends up as the concatenation of the previous run and the + // new one — same convention as Gemini. + var bytesWritten int64 + var skipped int64 lineCount := 0 for { select { case <-ctx.Done(): + saveGopherResume(req, bytesWritten) return nil, ctx.Err() default: } @@ -161,6 +233,7 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req * if err == io.EOF { break } + saveGopherResume(req, bytesWritten) return nil, core.NewNetworkError("failed to read gopher line", err, core.SafeURL(req.URL)) } @@ -172,12 +245,8 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req * // Strip item-type prefix if present (first char = type, second char is usually tab/space) displayLine := line if len(line) > 1 && line[1] != '.' { - // Full menu line: type + display string + tab + selector + tab + host + tab + port - if true { - // For text downloads, strip the type and metadata, show only display string + \n - clean := stripGopherMeta(line) - displayLine = clean + "\n" - } + clean := stripGopherMeta(line) + displayLine = clean + "\n" } // For informational lines (type i), don't strip anything — just pass through @@ -185,34 +254,75 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req * displayLine = line } - n, err := writer.Write([]byte(displayLine)) - if err != nil { - return nil, core.NewFileError("failed to write gopher text", err) + // On resume, drop the first startOffset bytes of the + // post-transform output so the file appends cleanly. + payload := []byte(displayLine) + if skipped < startOffset { + remain := startOffset - skipped + if int64(len(payload)) <= remain { + skipped += int64(len(payload)) + payload = nil + } else { + skipped += remain + payload = payload[remain:] + } + } + + if len(payload) > 0 { + n, werr := writer.Write(payload) + if werr != nil { + saveGopherResume(req, bytesWritten) + return nil, core.NewFileError("failed to write gopher text", werr) + } + bytesWritten += int64(n) } - totalRead += int64(n) lineCount++ if req.ProgressCallback != nil { - speed := float64(totalRead) / time.Since(startTime).Seconds() - req.ProgressCallback(totalRead, -1, speed) + speed := float64(bytesWritten) / time.Since(startTime).Seconds() + req.ProgressCallback(bytesWritten, -1, speed) } } + // Download complete — remove resume metadata sidecar. + if req.Output != "" && req.Output != "-" { + output.DeleteResumeMetadata(req.Output) + } + duration := time.Since(startTime) - speed := float64(totalRead) / duration.Seconds() + speed := float64(0) + if duration.Seconds() > 0 { + speed = float64(bytesWritten) / duration.Seconds() + } return &core.DownloadResult{ - BytesDownloaded: totalRead, - TotalSize: totalRead, + BytesDownloaded: bytesWritten, + TotalSize: bytesWritten, Duration: duration, Protocol: "GOPHER", IPVersion: 4, Speed: speed, OutputPath: req.Output, ContentType: gopherContentType(firstType), + Resumed: resumed, }, nil } +// saveGopherResume persists partial Gopher text download progress to the +// standard `.goget.meta` sidecar. No-op for stdout writes, nil requests, +// or zero-byte transfers. Gopher does not expose Content-Length, so the +// total field stays 0; the resume offset is taken from the number of +// bytes already delivered to the writer. +func saveGopherResume(req *core.DownloadRequest, downloaded int64) { + if req == nil || req.Output == "" || req.Output == "-" || downloaded <= 0 { + return + } + meta := output.NewResumeMetadata(req.URL.String(), "", "", downloaded, 0) + if err := meta.Save(req.Output); err != nil && req.Verbose { + fmt.Fprintf(os.Stderr, "[gopher] Warning: failed to save resume metadata: %v\n", err) + } +} + // gopherContentType returns the MIME content type for a gopher item type. func gopherContentType(t byte) string { switch t { diff --git a/internal/protocol/gopher/gopher_test.go b/internal/protocol/gopher/gopher_test.go index 7075825..4e7dc3c 100644 --- a/internal/protocol/gopher/gopher_test.go +++ b/internal/protocol/gopher/gopher_test.go @@ -8,11 +8,14 @@ import ( "context" "net" "net/url" + "os" "strconv" "strings" "testing" + "time" "codeberg.org/petrbalvin/goget/internal/core" + "codeberg.org/petrbalvin/goget/internal/output" ) func TestFormatGopherMenu(t *testing.T) { @@ -108,3 +111,309 @@ func TestGopherDownloadMock(t *testing.T) { t.Errorf("output missing text: %q", buf.String()) } } + +// TestGopherResumeSaveMetadata verifies that interrupting a Gopher text +// download persists a .goget.meta sidecar describing the partial state. +func TestGopherResumeSaveMetadata(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + go func() { + conn, _ := listener.Accept() + if conn == nil { + return + } + defer conn.Close() + reader := bufio.NewReader(conn) + reader.ReadString('\n') + // Send the first display line, sleep so the client has a chance + // to land in the read loop and receive the cancellation, then + // send the second line and the terminator. + conn.Write([]byte("hFirst\t/first\thost\t70\r\n")) + time.Sleep(150 * time.Millisecond) + conn.Write([]byte("hSecond\t/second\thost\t70\r\n")) + conn.Write([]byte(".\r\n")) + }() + + tmp := t.TempDir() + outPath := tmp + "/page.html" + + proto := NewProtocol() + u, _ := url.Parse("gopher://127.0.0.1:" + strconv.Itoa(port) + "/page") + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, dlErr := proto.Download(ctx, &core.DownloadRequest{ + URL: u, + Output: outPath, + }) + errCh <- dlErr + }() + + // Cancel mid-download. The slow server keeps the connection open + // long enough for the cancellation to fire between reads. + time.Sleep(50 * time.Millisecond) + cancel() + + dlErr := <-errCh + if dlErr == nil { + t.Fatal("expected error from cancelled context") + } + + metaPath := outPath + output.ResumeMetaFileSuffix + if _, statErr := os.Stat(metaPath); statErr != nil { + t.Fatalf("expected .goget.meta sidecar to exist: %v", statErr) + } + + meta, metaErr := output.LoadResumeMetadata(outPath) + if metaErr != nil { + t.Fatalf("LoadResumeMetadata: %v", metaErr) + } + if meta == nil { + t.Fatal("expected resume metadata, got nil") + } + if meta.Downloaded <= 0 { + t.Errorf("Downloaded = %d, want > 0", meta.Downloaded) + } + if meta.URL != u.String() { + t.Errorf("URL = %q, want %q", meta.URL, u.String()) + } + // Gopher does not expose Content-Length, so Total stays 0. + if meta.Total != 0 { + t.Errorf("Total = %d, want 0", meta.Total) + } + // Sanity-check: the saved Downloaded should match what's on disk. + info, statErr := os.Stat(outPath) + if statErr != nil { + t.Fatalf("partial file missing: %v", statErr) + } + if info.Size() != meta.Downloaded { + t.Errorf("file size = %d, metadata.Downloaded = %d (must match)", + info.Size(), meta.Downloaded) + } +} + +// TestGopherResumeDownload verifies end-to-end resume: an interrupted +// download completes on the second attempt with --resume=true, and the +// metadata sidecar is removed after the successful completion. +func TestGopherResumeDownload(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + cancelReady := make(chan struct{}) + connReady := make(chan struct{}) + go func() { + for i := 0; i < 2; i++ { + conn, lerr := listener.Accept() + if lerr != nil { + return + } + r := bufio.NewReader(conn) + r.ReadString('\n') + if i == 0 { + // First attempt: send the first line, then wait + // for the client to cancel before closing. + conn.Write([]byte("0Line one\t/foo\thost\t70\r\n")) + <-cancelReady + conn.Close() + connReady <- struct{}{} + } else { + // Second attempt: send the full body. + conn.Write([]byte("0Line one\t/foo\thost\t70\r\n")) + conn.Write([]byte("0Line two\t/bar\thost\t70\r\n")) + conn.Write([]byte(".\r\n")) + conn.Close() + } + } + }() + + tmp := t.TempDir() + outPath := tmp + "/resume.txt" + + proto := NewProtocol() + u, _ := url.Parse("gopher://127.0.0.1:" + strconv.Itoa(port) + "/foo") + + // First attempt: send only the first line, then cancel. + ctx1, cancel1 := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, dlErr := proto.Download(ctx1, &core.DownloadRequest{ + URL: u, + Output: outPath, + }) + errCh <- dlErr + }() + time.Sleep(100 * time.Millisecond) + cancel1() + cancelReady <- struct{}{} + <-connReady + dlErr := <-errCh + if dlErr == nil { + t.Fatal("expected error from cancelled context") + } + + // Sanity-check: partial file + metadata sidecar present. + info, statErr := os.Stat(outPath) + if statErr != nil { + t.Fatalf("partial file missing: %v", statErr) + } + if info.Size() == 0 { + t.Fatal("partial file is empty") + } + meta, metaErr := output.LoadResumeMetadata(outPath) + if metaErr != nil || meta == nil { + t.Fatalf("expected resume metadata: err=%v meta=%v", metaErr, meta) + } + + // Second attempt: resume should pick up where we left off. + result, resErr := proto.Download(context.Background(), &core.DownloadRequest{ + URL: u, + Output: outPath, + Resume: true, + }) + if resErr != nil { + t.Fatalf("resume download failed: %v", resErr) + } + if !result.Resumed { + t.Error("expected Resumed=true") + } + + // The file should contain the full body, with the resume logic + // skipping already-written bytes via the post-transform offset. + data, readErr := os.ReadFile(outPath) + if readErr != nil { + t.Fatalf("read output: %v", readErr) + } + want := "Line one\nLine two\n" + if string(data) != want { + t.Errorf("file content = %q, want %q", string(data), want) + } + + // Metadata sidecar should be deleted after a successful completion. + if output.FileExists(outPath + output.ResumeMetaFileSuffix) { + t.Error("expected .goget.meta to be deleted after successful resume") + } +} + +// TestSaveGopherResumeMetadata exercises the helper directly. +func TestSaveGopherResumeMetadata(t *testing.T) { + tmp := t.TempDir() + out := tmp + "/file.txt" + + u, err := url.Parse("gopher://example.com/foo.txt") + if err != nil { + t.Fatalf("parse URL: %v", err) + } + req := &core.DownloadRequest{ + URL: u, + Output: out, + } + + saveGopherResume(req, 4096) + + meta, err := output.LoadResumeMetadata(out) + if err != nil { + t.Fatalf("LoadResumeMetadata: %v", err) + } + if meta == nil { + t.Fatal("expected resume metadata, got nil") + } + if meta.Downloaded != 4096 { + t.Errorf("Downloaded = %d, want 4096", meta.Downloaded) + } + if meta.URL != "gopher://example.com/foo.txt" { + t.Errorf("URL = %q, want %q", meta.URL, "gopher://example.com/foo.txt") + } + if meta.Total != 0 { + t.Errorf("Total = %d, want 0", meta.Total) + } +} + +// TestSaveGopherResumeNoop verifies that the helper is a no-op for +// stdout writes, empty output paths, zero-byte transfers, and nil +// requests. +func TestSaveGopherResumeNoop(t *testing.T) { + u, _ := url.Parse("gopher://example.com/") + + t.Run("stdout", func(t *testing.T) { + saveGopherResume(&core.DownloadRequest{URL: u, Output: "-"}, 100) + }) + + t.Run("empty output", func(t *testing.T) { + saveGopherResume(&core.DownloadRequest{URL: u, Output: ""}, 100) + }) + + t.Run("zero bytes", func(t *testing.T) { + tmp := t.TempDir() + saveGopherResume(&core.DownloadRequest{URL: u, Output: tmp + "/f"}, 0) + if output.FileExists(tmp + "/f" + output.ResumeMetaFileSuffix) { + t.Error("metadata created for zero-byte transfer") + } + }) + + t.Run("nil request", func(t *testing.T) { + saveGopherResume(nil, 100) + }) +} + +// TestGopherResumeStaleMetadata exercises the path where the .goget.meta +// sidecar exists but the partial file doesn't (or has been truncated), +// forcing the resume code to clean up and start fresh. +func TestGopherResumeStaleMetadata(t *testing.T) { + tmp := t.TempDir() + outPath := tmp + "/stale.txt" + + // Pre-create an orphan .goget.meta for a URL we will not use. + orphanMeta := output.NewResumeMetadata("gopher://other.example/foo", "", "", 1234, 0) + if err := orphanMeta.Save(outPath); err != nil { + t.Fatalf("seed meta: %v", err) + } + // Do not pre-create the partial file; CanResume requires file size + // to match metadata Downloaded. + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + port := listener.Addr().(*net.TCPAddr).Port + + go func() { + conn, _ := listener.Accept() + if conn == nil { + return + } + defer conn.Close() + r := bufio.NewReader(conn) + r.ReadString('\n') + conn.Write([]byte("0Fresh\t/fresh\thost\t70\r\n.\r\n")) + }() + + proto := NewProtocol() + u, _ := url.Parse("gopher://127.0.0.1:" + strconv.Itoa(port) + "/fresh") + + res, err := proto.Download(context.Background(), &core.DownloadRequest{ + URL: u, + Output: outPath, + Resume: true, + }) + if err != nil { + t.Fatalf("download: %v", err) + } + if res.Resumed { + t.Error("expected Resumed=false after stale-metadata cleanup") + } + data, _ := os.ReadFile(outPath) + if string(data) != "Fresh\n" { + t.Errorf("content = %q, want %q", string(data), "Fresh\n") + } +}