feat(gopher): add resume and metadata support
This commit is contained in:
@@ -9,10 +9,13 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/petrbalvin/goget/internal/core"
|
||||
"codeberg.org/petrbalvin/goget/internal/output"
|
||||
"codeberg.org/petrbalvin/goget/internal/protocol"
|
||||
)
|
||||
|
||||
@@ -43,7 +46,7 @@ func NewProtocol() *Protocol {
|
||||
Name: "GOPHER",
|
||||
Scheme: "gopher",
|
||||
DefaultPort: 70,
|
||||
Features: []string{},
|
||||
Features: []string{"resume"},
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -72,6 +75,14 @@ func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*co
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
|
||||
// Close the connection when the context is cancelled so any
|
||||
// blocking Read returns immediately instead of waiting for the
|
||||
// 30-second deadline.
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
// Send selector + CRLF
|
||||
if _, err := fmt.Fprintf(conn, "%s\r\n", selector); err != nil {
|
||||
return nil, core.NewNetworkError("failed to send gopher selector", err, core.SafeURL(req.URL))
|
||||
@@ -138,20 +149,81 @@ func isTextGopherType(t byte) bool {
|
||||
}
|
||||
|
||||
// downloadText handles text responses where each line has a type prefix.
|
||||
// Supports resume for typeHTML and typeTextFile responses: when interrupted,
|
||||
// writes a `.goget.meta` sidecar; on resume with valid metadata, skips
|
||||
// already-written output bytes by length. Gopher does not expose
|
||||
// Content-Length, so the resume metadata records only the bytes
|
||||
// actually delivered to the writer.
|
||||
func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time, firstType byte) (*core.DownloadResult, error) {
|
||||
var writer io.Writer
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
var file *os.File
|
||||
resumed := false
|
||||
var startOffset int64
|
||||
|
||||
// Set up output destination. When req.Output is set and no external
|
||||
// Writer was provided, we open the file directly. This covers both
|
||||
// fresh downloads (create) and resume (append). When req.Writer is
|
||||
// set (caller manages the file), we use it as-is.
|
||||
if req.Output != "" && req.Output != "-" {
|
||||
if req.Resume {
|
||||
canResume, resumeMeta, rerr := output.CanResume(req.Output, req.URL.String())
|
||||
if rerr != nil {
|
||||
if req.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[gopher] Resume check failed: %v\n", rerr)
|
||||
}
|
||||
output.CleanupResumeFiles(req.Output)
|
||||
} else if canResume && resumeMeta != nil {
|
||||
file, rerr = os.OpenFile(req.Output, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if rerr != nil {
|
||||
return nil, core.NewFileError("failed to open file for resume", rerr)
|
||||
}
|
||||
defer file.Close()
|
||||
writer = file
|
||||
startOffset = resumeMeta.Downloaded
|
||||
resumed = true
|
||||
if req.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[gopher] Resuming from byte %d\n", startOffset)
|
||||
}
|
||||
}
|
||||
}
|
||||
if writer == nil && req.Writer == nil {
|
||||
// Fresh download — create the output file.
|
||||
dir := filepath.Dir(req.Output)
|
||||
if dir != "" && dir != "." {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, core.NewFileError("failed to create output directory", err)
|
||||
}
|
||||
}
|
||||
var ferr error
|
||||
file, ferr = os.Create(req.Output)
|
||||
if ferr != nil {
|
||||
return nil, core.NewFileError("failed to create output file", ferr)
|
||||
}
|
||||
defer file.Close()
|
||||
writer = file
|
||||
}
|
||||
}
|
||||
if writer == nil {
|
||||
if req.Writer != nil {
|
||||
writer = req.Writer
|
||||
} else {
|
||||
writer = io.Discard
|
||||
}
|
||||
}
|
||||
|
||||
var totalRead int64
|
||||
// bytesWritten tracks output payload actually delivered to the writer
|
||||
// (after the per-line metadata strip). On resume we skip ahead by
|
||||
// startOffset bytes of this post-transform stream so the file on
|
||||
// disk ends up as the concatenation of the previous run and the
|
||||
// new one — same convention as Gemini.
|
||||
var bytesWritten int64
|
||||
var skipped int64
|
||||
lineCount := 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
saveGopherResume(req, bytesWritten)
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
@@ -161,6 +233,7 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
saveGopherResume(req, bytesWritten)
|
||||
return nil, core.NewNetworkError("failed to read gopher line", err, core.SafeURL(req.URL))
|
||||
}
|
||||
|
||||
@@ -172,12 +245,8 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *
|
||||
// Strip item-type prefix if present (first char = type, second char is usually tab/space)
|
||||
displayLine := line
|
||||
if len(line) > 1 && line[1] != '.' {
|
||||
// Full menu line: type + display string + tab + selector + tab + host + tab + port
|
||||
if true {
|
||||
// For text downloads, strip the type and metadata, show only display string + \n
|
||||
clean := stripGopherMeta(line)
|
||||
displayLine = clean + "\n"
|
||||
}
|
||||
clean := stripGopherMeta(line)
|
||||
displayLine = clean + "\n"
|
||||
}
|
||||
|
||||
// For informational lines (type i), don't strip anything — just pass through
|
||||
@@ -185,34 +254,75 @@ func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *
|
||||
displayLine = line
|
||||
}
|
||||
|
||||
n, err := writer.Write([]byte(displayLine))
|
||||
if err != nil {
|
||||
return nil, core.NewFileError("failed to write gopher text", err)
|
||||
// On resume, drop the first startOffset bytes of the
|
||||
// post-transform output so the file appends cleanly.
|
||||
payload := []byte(displayLine)
|
||||
if skipped < startOffset {
|
||||
remain := startOffset - skipped
|
||||
if int64(len(payload)) <= remain {
|
||||
skipped += int64(len(payload))
|
||||
payload = nil
|
||||
} else {
|
||||
skipped += remain
|
||||
payload = payload[remain:]
|
||||
}
|
||||
}
|
||||
|
||||
if len(payload) > 0 {
|
||||
n, werr := writer.Write(payload)
|
||||
if werr != nil {
|
||||
saveGopherResume(req, bytesWritten)
|
||||
return nil, core.NewFileError("failed to write gopher text", werr)
|
||||
}
|
||||
bytesWritten += int64(n)
|
||||
}
|
||||
totalRead += int64(n)
|
||||
lineCount++
|
||||
|
||||
if req.ProgressCallback != nil {
|
||||
speed := float64(totalRead) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(totalRead, -1, speed)
|
||||
speed := float64(bytesWritten) / time.Since(startTime).Seconds()
|
||||
req.ProgressCallback(bytesWritten, -1, speed)
|
||||
}
|
||||
}
|
||||
|
||||
// Download complete — remove resume metadata sidecar.
|
||||
if req.Output != "" && req.Output != "-" {
|
||||
output.DeleteResumeMetadata(req.Output)
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
speed := float64(totalRead) / duration.Seconds()
|
||||
speed := float64(0)
|
||||
if duration.Seconds() > 0 {
|
||||
speed = float64(bytesWritten) / duration.Seconds()
|
||||
}
|
||||
|
||||
return &core.DownloadResult{
|
||||
BytesDownloaded: totalRead,
|
||||
TotalSize: totalRead,
|
||||
BytesDownloaded: bytesWritten,
|
||||
TotalSize: bytesWritten,
|
||||
Duration: duration,
|
||||
Protocol: "GOPHER",
|
||||
IPVersion: 4,
|
||||
Speed: speed,
|
||||
OutputPath: req.Output,
|
||||
ContentType: gopherContentType(firstType),
|
||||
Resumed: resumed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// saveGopherResume persists partial Gopher text download progress to the
|
||||
// standard `.goget.meta` sidecar. No-op for stdout writes, nil requests,
|
||||
// or zero-byte transfers. Gopher does not expose Content-Length, so the
|
||||
// total field stays 0; the resume offset is taken from the number of
|
||||
// bytes already delivered to the writer.
|
||||
func saveGopherResume(req *core.DownloadRequest, downloaded int64) {
|
||||
if req == nil || req.Output == "" || req.Output == "-" || downloaded <= 0 {
|
||||
return
|
||||
}
|
||||
meta := output.NewResumeMetadata(req.URL.String(), "", "", downloaded, 0)
|
||||
if err := meta.Save(req.Output); err != nil && req.Verbose {
|
||||
fmt.Fprintf(os.Stderr, "[gopher] Warning: failed to save resume metadata: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// gopherContentType returns the MIME content type for a gopher item type.
|
||||
func gopherContentType(t byte) string {
|
||||
switch t {
|
||||
|
||||
Reference in New Issue
Block a user