feat: initial goget release — modern IPv6-first download utility

2026-06-26 21:21:58 +02:00
commit 53db81e1f7
147 changed files with 45931 additions and 0 deletions
@@ -0,0 +1,470 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package queue
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// QueueItem represents a single item in the queue
+type QueueItem struct {
+	ID          string     `json:"id"`
+	URL         string     `json:"url"`
+	Output      string     `json:"output"`
+	Priority    int        `json:"priority"` // 1-10, higher = more important
+	Status      string     `json:"status"`   // pending, downloading, completed, failed, paused
+	AddedAt     time.Time  `json:"added_at"`
+	StartedAt   *time.Time `json:"started_at,omitempty"`
+	CompletedAt *time.Time `json:"completed_at,omitempty"`
+	Size        int64      `json:"size,omitempty"`
+	Downloaded  int64      `json:"downloaded,omitempty"`
+	Error       string     `json:"error,omitempty"`
+	Retries     int        `json:"retries"`
+	MaxRetries  int        `json:"max_retries"`
+}
+
+// QueueConfig configures the queue
+type QueueConfig struct {
+	// MaxParallel is the maximum number of parallel downloads
+	MaxParallel int `json:"max_parallel"`
+
+	// QueueFile is the path to the queue file
+	QueueFile string `json:"queue_file"`
+
+	// AutoSave enables auto-saving on changes
+	AutoSave bool `json:"auto_save"`
+
+	// DefaultMaxRetries is the default maximum retry count for failed items
+	DefaultMaxRetries int `json:"default_max_retries"`
+}
+
+// DefaultQueueConfig returns the default configuration
+func DefaultQueueConfig() *QueueConfig {
+	return &QueueConfig{
+		MaxParallel:       3,
+		QueueFile:         getDefaultQueueFile(),
+		AutoSave:          true,
+		DefaultMaxRetries: 3,
+	}
+}
+
+// Queue represents download queue
+type Queue struct {
+	config *QueueConfig
+	items  []*QueueItem
+	mu     sync.RWMutex
+	dirty  bool
+}
+
+// NewQueue creates a new queue
+func NewQueue(cfg *QueueConfig) (*Queue, error) {
+	if cfg == nil {
+		cfg = DefaultQueueConfig()
+	}
+
+	q := &Queue{
+		config: cfg,
+		items:  make([]*QueueItem, 0),
+	}
+
+	// Try to load existing queue
+	if err := q.Load(); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("failed to load queue: %w", err)
+	}
+
+	return q, nil
+}
+
+// Add adds a new download to the queue
+func (q *Queue) Add(url, output string, priority int) *QueueItem {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if priority < 1 {
+		priority = 5
+	}
+	if priority > 10 {
+		priority = 10
+	}
+
+	item := &QueueItem{
+		ID:         generateID(),
+		URL:        url,
+		Output:     output,
+		Priority:   priority,
+		Status:     "pending",
+		AddedAt:    time.Now(),
+		MaxRetries: q.config.DefaultMaxRetries,
+	}
+
+	q.items = append(q.items, item)
+	q.dirty = true
+
+	if q.config.AutoSave {
+		q.saveUnsafe()
+	}
+
+	return item
+}
+
+// AddMultiple adds multiple downloads at once
+func (q *Queue) AddMultiple(items []QueueItem) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	for i := range items {
+		item := &items[i]
+		if item.ID == "" {
+			item.ID = generateID()
+		}
+		if item.Status == "" {
+			item.Status = "pending"
+		}
+		if item.AddedAt.IsZero() {
+			item.AddedAt = time.Now()
+		}
+		if item.MaxRetries == 0 {
+			item.MaxRetries = q.config.DefaultMaxRetries
+		}
+		q.items = append(q.items, item)
+	}
+
+	q.dirty = true
+	if q.config.AutoSave {
+		q.saveUnsafe()
+	}
+}
+
+// Remove removes an item from the queue
+func (q *Queue) Remove(id string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	for i, item := range q.items {
+		if item.ID == id {
+			q.items = append(q.items[:i], q.items[i+1:]...)
+			q.dirty = true
+			if q.config.AutoSave {
+				q.saveUnsafe()
+			}
+			return true
+		}
+	}
+	return false
+}
+
+// GetNext returns the next item to download (highest priority, oldest)
+func (q *Queue) GetNext() *QueueItem {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	var best *QueueItem
+
+	for _, item := range q.items {
+		if item.Status != "pending" {
+			continue
+		}
+		if best == nil || item.Priority > best.Priority ||
+			(item.Priority == best.Priority && item.AddedAt.Before(best.AddedAt)) {
+			best = item
+		}
+	}
+
+	return best
+}
+
+// GetPending returns all pending items
+func (q *Queue) GetPending() []*QueueItem {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	var pending []*QueueItem
+	for _, item := range q.items {
+		if item.Status == "pending" {
+			pending = append(pending, item)
+		}
+	}
+	return pending
+}
+
+// GetDownloading returns all downloading items
+func (q *Queue) GetDownloading() []*QueueItem {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	var downloading []*QueueItem
+	for _, item := range q.items {
+		if item.Status == "downloading" {
+			downloading = append(downloading, item)
+		}
+	}
+	return downloading
+}
+
+// GetCompleted returns all completed items
+func (q *Queue) GetCompleted() []*QueueItem {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	var completed []*QueueItem
+	for _, item := range q.items {
+		if item.Status == "completed" {
+			completed = append(completed, item)
+		}
+	}
+	return completed
+}
+
+// GetFailed returns all failed items
+func (q *Queue) GetFailed() []*QueueItem {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	var failed []*QueueItem
+	for _, item := range q.items {
+		if item.Status == "failed" {
+			failed = append(failed, item)
+		}
+	}
+	return failed
+}
+
+// UpdateStatus updates an item's status
+func (q *Queue) UpdateStatus(id, status string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	for _, item := range q.items {
+		if item.ID == id {
+			item.Status = status
+			now := time.Now()
+			if status == "downloading" && item.StartedAt == nil {
+				item.StartedAt = &now
+			}
+			if status == "completed" || status == "failed" {
+				item.CompletedAt = &now
+			}
+			q.dirty = true
+			if q.config.AutoSave {
+				q.saveUnsafe()
+			}
+			return true
+		}
+	}
+	return false
+}
+
+// UpdateProgress updates an item's progress
+func (q *Queue) UpdateProgress(id string, downloaded, size int64) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	for _, item := range q.items {
+		if item.ID == id {
+			item.Downloaded = downloaded
+			item.Size = size
+			q.dirty = true
+			if q.config.AutoSave {
+				q.saveUnsafe()
+			}
+			return true
+		}
+	}
+	return false
+}
+
+// Retry marks a failed item for retry
+func (q *Queue) Retry(id string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	for _, item := range q.items {
+		if item.ID == id {
+			if item.Retries < item.MaxRetries {
+				item.Retries++
+				item.Status = "pending"
+				item.Error = ""
+				item.StartedAt = nil
+				item.CompletedAt = nil
+				q.dirty = true
+				if q.config.AutoSave {
+					q.saveUnsafe()
+				}
+				return true
+			}
+			return false
+		}
+	}
+	return false
+}
+
+// ClearCompleted removes all completed items
+func (q *Queue) ClearCompleted() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	count := 0
+	var remaining []*QueueItem
+	for _, item := range q.items {
+		if item.Status == "completed" {
+			count++
+		} else {
+			remaining = append(remaining, item)
+		}
+	}
+
+	if count > 0 {
+		q.items = remaining
+		q.dirty = true
+		if q.config.AutoSave {
+			q.saveUnsafe()
+		}
+	}
+
+	return count
+}
+
+// GetAll returns all items
+func (q *Queue) GetAll() []*QueueItem {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	result := make([]*QueueItem, len(q.items))
+	copy(result, q.items)
+	return result
+}
+
+// Len returns the item count
+func (q *Queue) Len() int {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+	return len(q.items)
+}
+
+// Save saves the queue to a file
+func (q *Queue) Save() error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.saveUnsafe()
+}
+
+func (q *Queue) saveUnsafe() error {
+	if !q.dirty {
+		return nil
+	}
+
+	// Ensure directory exists
+	dir := filepath.Dir(q.config.QueueFile)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create queue directory: %w", err)
+	}
+
+	data, err := json.MarshalIndent(q.items, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal queue: %w", err)
+	}
+
+	// Write atomically
+	tmpFile := q.config.QueueFile + ".tmp"
+	if err := os.WriteFile(tmpFile, data, 0600); err != nil {
+		return fmt.Errorf("failed to write queue: %w", err)
+	}
+
+	if err := os.Rename(tmpFile, q.config.QueueFile); err != nil {
+		return fmt.Errorf("failed to rename queue file: %w", err)
+	}
+
+	q.dirty = false
+	return nil
+}
+
+// Load loads the queue from a file
+func (q *Queue) Load() error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	data, err := os.ReadFile(q.config.QueueFile)
+	if err != nil {
+		return err
+	}
+
+	var items []*QueueItem
+	if err := json.Unmarshal(data, &items); err != nil {
+		return fmt.Errorf("failed to unmarshal queue: %w", err)
+	}
+
+	// Reset downloading items to pending (they were interrupted)
+	for _, item := range items {
+		if item.Status == "downloading" {
+			item.Status = "pending"
+			item.StartedAt = nil
+		}
+	}
+
+	q.items = items
+	q.dirty = false
+	return nil
+}
+
+// GetStats returns queue statistics
+func (q *Queue) GetStats() QueueStats {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	stats := QueueStats{}
+	for _, item := range q.items {
+		switch item.Status {
+		case "pending":
+			stats.Pending++
+		case "downloading":
+			stats.Downloading++
+		case "completed":
+			stats.Completed++
+		case "failed":
+			stats.Failed++
+		case "paused":
+			stats.Paused++
+		}
+		stats.TotalSize += item.Size
+		stats.TotalDownloaded += item.Downloaded
+	}
+	stats.Total = len(q.items)
+	return stats
+}
+
+// QueueStats represents queue statistics
+type QueueStats struct {
+	Total           int   `json:"total"`
+	Pending         int   `json:"pending"`
+	Downloading     int   `json:"downloading"`
+	Completed       int   `json:"completed"`
+	Failed          int   `json:"failed"`
+	Paused          int   `json:"paused"`
+	TotalSize       int64 `json:"total_size"`
+	TotalDownloaded int64 `json:"total_downloaded"`
+}
+
+// generateID generates a unique ID with a random suffix to prevent collisions.
+func generateID() string {
+	b := make([]byte, 4)
+	_, _ = rand.Read(b)
+	return fmt.Sprintf("q_%d_%x", time.Now().UnixNano(), b)
+}
+
+// getDefaultQueueFile returns the default path to the queue file
+func getDefaultQueueFile() string {
+	if xdg := os.Getenv("XDG_STATE_HOME"); xdg != "" {
+		return filepath.Join(xdg, "goget", "queue.json")
+	}
+	if home, err := os.UserHomeDir(); err == nil {
+		return filepath.Join(home, ".local", "state", "goget", "queue.json")
+	}
+	return "goget.queue.json"
+}
@@ -0,0 +1,359 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package queue
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestNewQueue(t *testing.T) {
+	q, err := NewQueue(nil)
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+	if q == nil {
+		t.Fatal("Queue is nil")
+	}
+	if q.config.MaxParallel != 3 {
+		t.Errorf("Expected MaxParallel=3, got %d", q.config.MaxParallel)
+	}
+}
+
+func TestQueueAdd(t *testing.T) {
+	tmpDir := t.TempDir()
+	queueFile := filepath.Join(tmpDir, "queue.json")
+
+	q, err := NewQueue(&QueueConfig{QueueFile: queueFile, AutoSave: true})
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+	if item == nil {
+		t.Fatal("Added item is nil")
+	}
+	if item.URL != "https://example.com/file.zip" {
+		t.Errorf("Expected URL https://example.com/file.zip, got %s", item.URL)
+	}
+	if item.Priority != 5 {
+		t.Errorf("Expected priority 5, got %d", item.Priority)
+	}
+	if item.Status != "pending" {
+		t.Errorf("Expected status pending, got %s", item.Status)
+	}
+}
+
+func TestQueueAddPriority(t *testing.T) {
+	q, err := NewQueue(nil)
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	// Test priority clamping
+	item1 := q.Add("https://example.com/1.zip", "/tmp/1.zip", 0)
+	if item1.Priority != 5 {
+		t.Errorf("Expected priority 5 (default), got %d", item1.Priority)
+	}
+
+	item2 := q.Add("https://example.com/2.zip", "/tmp/2.zip", 15)
+	if item2.Priority != 10 {
+		t.Errorf("Expected priority 10 (max), got %d", item2.Priority)
+	}
+}
+
+func TestQueueRemove(t *testing.T) {
+	q, _ := NewQueue(nil)
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+	initialLen := q.Len()
+
+	if !q.Remove(item.ID) {
+		t.Error("Failed to remove item")
+	}
+	if q.Len() != initialLen-1 {
+		t.Errorf("Expected queue length %d, got %d", initialLen-1, q.Len())
+	}
+}
+
+func TestQueueGetNext(t *testing.T) {
+	q, _ := NewQueue(nil)
+
+	// Add items with different priorities
+	q.Add("https://example.com/low.zip", "/tmp/low.zip", 3)
+	q.Add("https://example.com/high.zip", "/tmp/high.zip", 10)
+	q.Add("https://example.com/med.zip", "/tmp/med.zip", 5)
+
+	next := q.GetNext()
+	if next == nil {
+		t.Fatal("GetNext returned nil")
+	}
+	// Should return highest priority (10)
+	if next.Priority < 8 {
+		t.Errorf("Expected high priority item (>=8), got %d", next.Priority)
+	}
+}
+
+func TestQueueUpdateStatus(t *testing.T) {
+	q, err := NewQueue(nil)
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+
+	if !q.UpdateStatus(item.ID, "downloading") {
+		t.Error("Failed to update status")
+	}
+
+	// Refresh item from queue
+	items := q.GetAll()
+	var updated *QueueItem
+	for _, it := range items {
+		if it.ID == item.ID {
+			updated = it
+			break
+		}
+	}
+
+	if updated == nil || updated.Status != "downloading" {
+		t.Errorf("Expected status downloading, got %s", updated.Status)
+	}
+}
+
+func TestQueueGetPending(t *testing.T) {
+	q, _ := NewQueue(nil)
+
+	q.Add("https://example.com/1.zip", "/tmp/1.zip", 5)
+	q.Add("https://example.com/2.zip", "/tmp/2.zip", 5)
+
+	items := q.GetAll()
+	if len(items) >= 2 {
+		q.UpdateStatus(items[0].ID, "completed")
+	}
+
+	pending := q.GetPending()
+	if len(pending) < 1 {
+		t.Errorf("Expected at least 1 pending item, got %d", len(pending))
+	}
+}
+
+func TestQueueGetStats(t *testing.T) {
+	q, _ := NewQueue(nil)
+	initialLen := q.Len()
+
+	q.Add("https://example.com/1.zip", "/tmp/1.zip", 5)
+	q.Add("https://example.com/2.zip", "/tmp/2.zip", 5)
+	q.Add("https://example.com/3.zip", "/tmp/3.zip", 5)
+
+	items := q.GetAll()
+	completedCount := 0
+	for _, item := range items {
+		if item.Status == "pending" && completedCount == 0 {
+			q.UpdateStatus(item.ID, "completed")
+			completedCount++
+		}
+	}
+
+	stats := q.GetStats()
+	if stats.Total < initialLen+3 {
+		t.Errorf("Expected at least %d total, got %d", initialLen+3, stats.Total)
+	}
+	if stats.Completed < 1 {
+		t.Errorf("Expected at least 1 completed, got %d", stats.Completed)
+	}
+}
+
+func TestQueueSaveLoad(t *testing.T) {
+	tmpDir := t.TempDir()
+	queueFile := filepath.Join(tmpDir, "queue.json")
+
+	// Create and save queue
+	q1, err := NewQueue(&QueueConfig{QueueFile: queueFile, AutoSave: false})
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	q1.Add("https://example.com/file.zip", "/tmp/file.zip", 7)
+	q1.Add("https://example.com/other.zip", "/tmp/other.zip", 3)
+
+	if err := q1.Save(); err != nil {
+		t.Fatalf("Failed to save queue: %v", err)
+	}
+
+	// Load queue
+	q2, err := NewQueue(&QueueConfig{QueueFile: queueFile})
+	if err != nil {
+		t.Fatalf("Failed to load queue: %v", err)
+	}
+
+	if q2.Len() != 2 {
+		t.Errorf("Expected 2 items after load, got %d", q2.Len())
+	}
+}
+
+func TestQueueClearCompleted(t *testing.T) {
+	q, _ := NewQueue(nil)
+	initialLen := q.Len()
+
+	q.Add("https://example.com/1.zip", "/tmp/1.zip", 5)
+	q.Add("https://example.com/2.zip", "/tmp/2.zip", 5)
+	q.Add("https://example.com/3.zip", "/tmp/3.zip", 5)
+
+	items := q.GetAll()
+	completedCount := 0
+	for _, item := range items {
+		if item.Status == "pending" && completedCount == 0 {
+			q.UpdateStatus(item.ID, "completed")
+			completedCount++
+		}
+	}
+
+	count := q.ClearCompleted()
+	if count < 1 {
+		t.Errorf("Expected to clear at least 1 item, cleared %d", count)
+	}
+	if q.Len() >= initialLen+3 {
+		t.Errorf("Expected less than %d items remaining, got %d", initialLen+3, q.Len())
+	}
+}
+
+func TestQueueRetry(t *testing.T) {
+	q, err := NewQueue(nil)
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+	q.UpdateStatus(item.ID, "failed")
+
+	// Should be able to retry
+	if !q.Retry(item.ID) {
+		t.Error("Failed to retry item")
+	}
+
+	// Check status is back to pending
+	items := q.GetAll()
+	for _, it := range items {
+		if it.ID == item.ID {
+			if it.Status != "pending" {
+				t.Errorf("Expected status pending after retry, got %s", it.Status)
+			}
+			break
+		}
+	}
+}
+
+func TestQueueRetryMaxRetries(t *testing.T) {
+	q, err := NewQueue(&QueueConfig{DefaultMaxRetries: 2})
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+	q.UpdateStatus(item.ID, "failed")
+
+	// First retry should work
+	if !q.Retry(item.ID) {
+		t.Error("First retry should succeed")
+	}
+	q.UpdateStatus(item.ID, "failed")
+
+	// Second retry should work
+	if !q.Retry(item.ID) {
+		t.Error("Second retry should succeed")
+	}
+	q.UpdateStatus(item.ID, "failed")
+
+	// Third retry should fail (max retries reached)
+	if q.Retry(item.ID) {
+		t.Error("Third retry should fail (max retries exceeded)")
+	}
+}
+
+func TestGenerateID(t *testing.T) {
+	id1 := generateID()
+	id2 := generateID()
+
+	if id1 == id2 {
+		t.Error("Generated IDs should be unique")
+	}
+
+	// Check ID format
+	if len(id1) < 3 {
+		t.Errorf("ID too short: %s", id1)
+	}
+}
+
+func TestDefaultQueueFile(t *testing.T) {
+	path := getDefaultQueueFile()
+	if path == "" {
+		t.Error("Default queue file path should not be empty")
+	}
+}
+
+func TestQueueConcurrentAccess(t *testing.T) {
+	q, _ := NewQueue(nil)
+	initialLen := q.Len()
+
+	done := make(chan bool, 10)
+
+	// Concurrent adds
+	for i := 0; i < 10; i++ {
+		go func(n int) {
+			q.Add("https://example.com/"+string(rune('a'+n))+".zip", "/tmp/"+string(rune('a'+n))+".zip", 5)
+			done <- true
+		}(i)
+	}
+
+	for i := 0; i < 10; i++ {
+		<-done
+	}
+
+	if q.Len() < initialLen+10 {
+		t.Errorf("Expected at least %d items after concurrent adds, got %d", initialLen+10, q.Len())
+	}
+}
+
+func TestQueueAutoSave(t *testing.T) {
+	tmpDir := t.TempDir()
+	queueFile := filepath.Join(tmpDir, "queue.json")
+
+	q, err := NewQueue(&QueueConfig{QueueFile: queueFile, AutoSave: true})
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+
+	// Check file exists
+	if _, err := os.Stat(queueFile); os.IsNotExist(err) {
+		t.Error("Queue file should exist with AutoSave enabled")
+	}
+}
+
+func TestQueueProgressUpdate(t *testing.T) {
+	q, err := NewQueue(nil)
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	item := q.Add("https://example.com/file.zip", "/tmp/file.zip", 5)
+
+	q.UpdateProgress(item.ID, 500, 1000)
+
+	items := q.GetAll()
+	for _, it := range items {
+		if it.ID == item.ID {
+			if it.Downloaded != 500 {
+				t.Errorf("Expected downloaded 500, got %d", it.Downloaded)
+			}
+			if it.Size != 1000 {
+				t.Errorf("Expected size 1000, got %d", it.Size)
+			}
+			break
+		}
+	}
+}