feat: initial goget release — modern IPv6-first download utility

2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
@@ -0,0 +1,384 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package gopher
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	"codeberg.org/petrbalvin/goget/internal/core"
+	"codeberg.org/petrbalvin/goget/internal/protocol"
+)
+
+// Gopher item types (RFC 1436).
+const (
+	typeTextFile    = '0'
+	typeDirectory   = '1'
+	typeError       = '3'
+	typeBinaryFile  = '9'
+	typeGIF         = 'g'
+	typeImage       = 'I'
+	typeHTML        = 'h'
+	typeInformation = 'i'
+	typeSound       = 's'
+	typeTelnet      = '8'
+	typeSearch      = '7' // index search
+)
+
+// Protocol implements gopher:// downloads (RFC 1436).
+type Protocol struct {
+	*protocol.BaseProtocol
+}
+
+// NewProtocol creates a new Gopher protocol handler.
+func NewProtocol() *Protocol {
+	return &Protocol{
+		BaseProtocol: protocol.NewBaseProtocol(protocol.ProtocolInfo{
+			Name:        "GOPHER",
+			Scheme:      "gopher",
+			DefaultPort: 70,
+			Features:    []string{},
+		}),
+	}
+}
+
+func (p *Protocol) Download(ctx context.Context, req *core.DownloadRequest) (*core.DownloadResult, error) {
+	startTime := time.Now()
+
+	host := req.URL.Hostname()
+	port := req.URL.Port()
+	if port == "" {
+		port = "70"
+	}
+	selector := req.URL.Path
+	if req.URL.RawQuery != "" {
+		selector += "?" + req.URL.RawQuery
+	}
+	if selector == "" {
+		selector = "/"
+	}
+
+	addr := net.JoinHostPort(host, port)
+	conn, err := net.DialTimeout("tcp", addr, 30*time.Second)
+	if err != nil {
+		return nil, core.NewNetworkError("failed to connect to gopher server", err, core.SafeURL(req.URL))
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(30 * time.Second))
+
+	// Send selector + CRLF
+	if _, err := fmt.Fprintf(conn, "%s\r\n", selector); err != nil {
+		return nil, core.NewNetworkError("failed to send gopher selector", err, core.SafeURL(req.URL))
+	}
+
+	// Peek at first byte to determine response type
+	reader := bufio.NewReader(conn)
+	firstByte, err := reader.Peek(1)
+	if err != nil {
+		return nil, core.NewNetworkError("failed to read gopher response", err, core.SafeURL(req.URL))
+	}
+
+	itemType := firstByte[0]
+
+	switch itemType {
+	case typeError:
+		// Type 3: error — read the error message
+		line, err := reader.ReadString('\n')
+		if err != nil && err != io.EOF {
+			return nil, core.NewNetworkError("failed to read gopher error", err, core.SafeURL(req.URL))
+		}
+		msg := strings.TrimSpace(line)
+		if len(msg) > 2 {
+			msg = msg[2:] // strip type + first char
+		}
+		return nil, core.NewProtocolError("gopher server error: "+msg, nil, core.SafeURL(req.URL))
+
+	case typeDirectory:
+		// Type 1: directory listing — format as readable text
+		return p.downloadDirectory(reader, req, startTime)
+
+	case typeTextFile, typeHTML, typeInformation:
+		// Text-based types — strip item-type prefix, handle .\r\n terminator
+		return p.downloadText(ctx, reader, req, startTime, itemType)
+
+	case typeSearch:
+		// Type 7: index search — server expects a query
+		return nil, core.NewProtocolError(
+			"gopher search server requires a query (append ?query to URL)", nil, core.SafeURL(req.URL))
+
+	case typeTelnet:
+		// Type 8: telnet session — interactive, cannot download
+		return nil, core.NewProtocolError(
+			"gopher telnet session type cannot be downloaded", nil, core.SafeURL(req.URL))
+
+	default:
+		// Binary types (9, g, I, s, etc.) — raw data with .\r\n termination
+		// But first check which ones have the type-prefix-per-line format
+		if isTextGopherType(itemType) {
+			return p.downloadText(ctx, reader, req, startTime, itemType)
+		}
+		return p.downloadBinary(ctx, reader, req, startTime)
+	}
+}
+
+// isTextGopherType returns true for types that use line-prefix format.
+func isTextGopherType(t byte) bool {
+	switch t {
+	case typeTextFile, typeDirectory, typeError, typeInformation, typeHTML:
+		return true
+	default:
+		return false
+	}
+}
+
+// downloadText handles text responses where each line has a type prefix.
+func (p *Protocol) downloadText(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time, firstType byte) (*core.DownloadResult, error) {
+	var writer io.Writer
+	if req.Writer != nil {
+		writer = req.Writer
+	} else {
+		writer = io.Discard
+	}
+
+	var totalRead int64
+	lineCount := 0
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, core.NewNetworkError("failed to read gopher line", err, core.SafeURL(req.URL))
+		}
+
+		// End-of-transmission: line containing only ".\r\n"
+		if line == ".\r\n" || line == ".\n" {
+			break
+		}
+
+		// Strip item-type prefix if present (first char = type, second char is usually tab/space)
+		displayLine := line
+		if len(line) > 1 && line[1] != '.' {
+			// Full menu line: type + display string + tab + selector + tab + host + tab + port
+			if true {
+				// For text downloads, strip the type and metadata, show only display string + \n
+				clean := stripGopherMeta(line)
+				displayLine = clean + "\n"
+			}
+		}
+
+		// For informational lines (type i), don't strip anything — just pass through
+		if firstType == typeInformation {
+			displayLine = line
+		}
+
+		n, err := writer.Write([]byte(displayLine))
+		if err != nil {
+			return nil, core.NewFileError("failed to write gopher text", err)
+		}
+		totalRead += int64(n)
+		lineCount++
+
+		if req.ProgressCallback != nil {
+			speed := float64(totalRead) / time.Since(startTime).Seconds()
+			req.ProgressCallback(totalRead, -1, speed)
+		}
+	}
+
+	duration := time.Since(startTime)
+	speed := float64(totalRead) / duration.Seconds()
+
+	return &core.DownloadResult{
+		BytesDownloaded: totalRead,
+		TotalSize:       totalRead,
+		Duration:        duration,
+		Protocol:        "GOPHER",
+		IPVersion:       4,
+		Speed:           speed,
+		OutputPath:      req.Output,
+	}, nil
+}
+
+// downloadDirectory formats a directory listing as readable text.
+func (p *Protocol) downloadDirectory(reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
+	var writer io.Writer
+	if req.Writer != nil {
+		writer = req.Writer
+	} else {
+		writer = io.Discard
+	}
+
+	var totalRead int64
+	var sb strings.Builder
+
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, core.NewNetworkError("failed to read gopher directory", err, core.SafeURL(req.URL))
+		}
+
+		if line == ".\r\n" || line == ".\n" {
+			break
+		}
+
+		// Parse gopher menu line: type + display string + tab + selector + tab + host + tab + port
+		formatted := formatGopherMenu(line)
+		sb.WriteString(formatted)
+		sb.WriteByte('\n')
+	}
+
+	output := sb.String()
+	n, err := writer.Write([]byte(output))
+	if err != nil {
+		return nil, core.NewFileError("failed to write gopher directory", err)
+	}
+	totalRead = int64(n)
+
+	duration := time.Since(startTime)
+	speed := float64(totalRead) / duration.Seconds()
+
+	return &core.DownloadResult{
+		BytesDownloaded: totalRead,
+		TotalSize:       totalRead,
+		Duration:        duration,
+		Protocol:        "GOPHER",
+		IPVersion:       4,
+		Speed:           speed,
+		OutputPath:      req.Output,
+	}, nil
+}
+
+// downloadBinary handles binary responses.
+func (p *Protocol) downloadBinary(ctx context.Context, reader *bufio.Reader, req *core.DownloadRequest, startTime time.Time) (*core.DownloadResult, error) {
+	var writer io.Writer
+	if req.Writer != nil {
+		writer = req.Writer
+	} else {
+		writer = io.Discard
+	}
+
+	var totalRead int64
+	buf := make([]byte, 32*1024)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		n, err := reader.Read(buf)
+		if n > 0 {
+			if _, werr := writer.Write(buf[:n]); werr != nil {
+				return nil, core.NewFileError("failed to write gopher binary data", werr)
+			}
+			totalRead += int64(n)
+			if req.ProgressCallback != nil {
+				speed := float64(totalRead) / time.Since(startTime).Seconds()
+				req.ProgressCallback(totalRead, -1, speed)
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, core.NewNetworkError("failed to read gopher binary", err, core.SafeURL(req.URL))
+		}
+	}
+
+	duration := time.Since(startTime)
+	speed := float64(totalRead) / duration.Seconds()
+
+	return &core.DownloadResult{
+		BytesDownloaded: totalRead,
+		TotalSize:       totalRead,
+		Duration:        duration,
+		Protocol:        "GOPHER",
+		IPVersion:       4,
+		Speed:           speed,
+		OutputPath:      req.Output,
+	}, nil
+}
+
+// stripGopherMeta removes the gopher metadata (tabs + selector + host + port) from a menu line.
+func stripGopherMeta(line string) string {
+	line = strings.TrimRight(line, "\r\n")
+	if len(line) < 2 {
+		return line
+	}
+	// Format: Xdisplay string\tselector\thost\tport
+	// Everything before the first tab is the type + display string
+	tabIdx := strings.IndexByte(line, '\t')
+	if tabIdx <= 1 {
+		return line // no metadata
+	}
+	return line[1:tabIdx] // strip type char, keep display text
+}
+
+// formatGopherMenu formats a gopher menu line for human display.
+func formatGopherMenu(line string) string {
+	line = strings.TrimRight(line, "\r\n")
+	if len(line) < 2 {
+		return line
+	}
+
+	itemType := line[0]
+	rest := line[1:]
+
+	// Split by tabs: display\0selector\0host\0port
+	parts := strings.Split(rest, "\t")
+	display := parts[0]
+	selector := ""
+	if len(parts) > 1 {
+		selector = parts[1]
+	}
+
+	icon := gopherIcon(itemType)
+
+	if selector != "" && selector != "(null)" {
+		return fmt.Sprintf("%s %s  [%s]", icon, display, selector)
+	}
+	return fmt.Sprintf("%s %s", icon, display)
+}
+
+// gopherIcon returns an icon for a gopher item type.
+func gopherIcon(t byte) string {
+	switch t {
+	case typeDirectory:
+		return "📁"
+	case typeTextFile:
+		return "📄"
+	case typeBinaryFile:
+		return "💾"
+	case typeGIF, typeImage:
+		return "🖼"
+	case typeHTML:
+		return "🌐"
+	case typeSound:
+		return "🔊"
+	case typeSearch:
+		return "🔍"
+	case typeTelnet:
+		return "🖥"
+	case typeInformation:
+		return "ℹ️"
+	default:
+		return "❓"
+	}
+}