471 lines
9.8 KiB
Go
471 lines
9.8 KiB
Go
//go:build linux || freebsd
|
|||
|
|
// +build linux freebsd
|
||
|
|
|
||
|
|
package queue
|
||
|
|
|
||
|
|
import (
|
||
|
|
"crypto/rand"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"sync"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
// QueueItem represents a single item in the queue
|
||
|
|
type QueueItem struct {
|
||
|
|
ID string `json:"id"`
|
||
|
|
URL string `json:"url"`
|
||
|
|
Output string `json:"output"`
|
||
|
|
Priority int `json:"priority"` // 1-10, higher = more important
|
||
|
|
Status string `json:"status"` // pending, downloading, completed, failed, paused
|
||
|
|
AddedAt time.Time `json:"added_at"`
|
||
|
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||
|
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||
|
|
Size int64 `json:"size,omitempty"`
|
||
|
|
Downloaded int64 `json:"downloaded,omitempty"`
|
||
|
|
Error string `json:"error,omitempty"`
|
||
|
|
Retries int `json:"retries"`
|
||
|
|
MaxRetries int `json:"max_retries"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// QueueConfig configures the queue
|
||
|
|
type QueueConfig struct {
|
||
|
|
// MaxParallel is the maximum number of parallel downloads
|
||
|
|
MaxParallel int `json:"max_parallel"`
|
||
|
|
|
||
|
|
// QueueFile is the path to the queue file
|
||
|
|
QueueFile string `json:"queue_file"`
|
||
|
|
|
||
|
|
// AutoSave enables auto-saving on changes
|
||
|
|
AutoSave bool `json:"auto_save"`
|
||
|
|
|
||
|
|
// DefaultMaxRetries is the default maximum retry count for failed items
|
||
|
|
DefaultMaxRetries int `json:"default_max_retries"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// DefaultQueueConfig returns the default configuration
|
||
|
|
func DefaultQueueConfig() *QueueConfig {
|
||
|
|
return &QueueConfig{
|
||
|
|
MaxParallel: 3,
|
||
|
|
QueueFile: getDefaultQueueFile(),
|
||
|
|
AutoSave: true,
|
||
|
|
DefaultMaxRetries: 3,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Queue represents download queue
|
||
|
|
type Queue struct {
|
||
|
|
config *QueueConfig
|
||
|
|
items []*QueueItem
|
||
|
|
mu sync.RWMutex
|
||
|
|
dirty bool
|
||
|
|
}
|
||
|
|
|
||
|
|
// NewQueue creates a new queue
|
||
|
|
func NewQueue(cfg *QueueConfig) (*Queue, error) {
|
||
|
|
if cfg == nil {
|
||
|
|
cfg = DefaultQueueConfig()
|
||
|
|
}
|
||
|
|
|
||
|
|
q := &Queue{
|
||
|
|
config: cfg,
|
||
|
|
items: make([]*QueueItem, 0),
|
||
|
|
}
|
||
|
|
|
||
|
|
// Try to load existing queue
|
||
|
|
if err := q.Load(); err != nil && !os.IsNotExist(err) {
|
||
|
|
return nil, fmt.Errorf("failed to load queue: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
return q, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// Add adds a new download to the queue
|
||
|
|
func (q *Queue) Add(url, output string, priority int) *QueueItem {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
if priority < 1 {
|
||
|
|
priority = 5
|
||
|
|
}
|
||
|
|
if priority > 10 {
|
||
|
|
priority = 10
|
||
|
|
}
|
||
|
|
|
||
|
|
item := &QueueItem{
|
||
|
|
ID: generateID(),
|
||
|
|
URL: url,
|
||
|
|
Output: output,
|
||
|
|
Priority: priority,
|
||
|
|
Status: "pending",
|
||
|
|
AddedAt: time.Now(),
|
||
|
|
MaxRetries: q.config.DefaultMaxRetries,
|
||
|
|
}
|
||
|
|
|
||
|
|
q.items = append(q.items, item)
|
||
|
|
q.dirty = true
|
||
|
|
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
|
||
|
|
return item
|
||
|
|
}
|
||
|
|
|
||
|
|
// AddMultiple adds multiple downloads at once
|
||
|
|
func (q *Queue) AddMultiple(items []QueueItem) {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
for i := range items {
|
||
|
|
item := &items[i]
|
||
|
|
if item.ID == "" {
|
||
|
|
item.ID = generateID()
|
||
|
|
}
|
||
|
|
if item.Status == "" {
|
||
|
|
item.Status = "pending"
|
||
|
|
}
|
||
|
|
if item.AddedAt.IsZero() {
|
||
|
|
item.AddedAt = time.Now()
|
||
|
|
}
|
||
|
|
if item.MaxRetries == 0 {
|
||
|
|
item.MaxRetries = q.config.DefaultMaxRetries
|
||
|
|
}
|
||
|
|
q.items = append(q.items, item)
|
||
|
|
}
|
||
|
|
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Remove removes an item from the queue
|
||
|
|
func (q *Queue) Remove(id string) bool {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
for i, item := range q.items {
|
||
|
|
if item.ID == id {
|
||
|
|
q.items = append(q.items[:i], q.items[i+1:]...)
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetNext returns the next item to download (highest priority, oldest)
|
||
|
|
func (q *Queue) GetNext() *QueueItem {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
var best *QueueItem
|
||
|
|
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status != "pending" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if best == nil || item.Priority > best.Priority ||
|
||
|
|
(item.Priority == best.Priority && item.AddedAt.Before(best.AddedAt)) {
|
||
|
|
best = item
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return best
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetPending returns all pending items
|
||
|
|
func (q *Queue) GetPending() []*QueueItem {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
var pending []*QueueItem
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status == "pending" {
|
||
|
|
pending = append(pending, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return pending
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetDownloading returns all downloading items
|
||
|
|
func (q *Queue) GetDownloading() []*QueueItem {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
var downloading []*QueueItem
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status == "downloading" {
|
||
|
|
downloading = append(downloading, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return downloading
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetCompleted returns all completed items
|
||
|
|
func (q *Queue) GetCompleted() []*QueueItem {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
var completed []*QueueItem
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status == "completed" {
|
||
|
|
completed = append(completed, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return completed
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetFailed returns all failed items
|
||
|
|
func (q *Queue) GetFailed() []*QueueItem {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
var failed []*QueueItem
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status == "failed" {
|
||
|
|
failed = append(failed, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return failed
|
||
|
|
}
|
||
|
|
|
||
|
|
// UpdateStatus updates an item's status
|
||
|
|
func (q *Queue) UpdateStatus(id, status string) bool {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.ID == id {
|
||
|
|
item.Status = status
|
||
|
|
now := time.Now()
|
||
|
|
if status == "downloading" && item.StartedAt == nil {
|
||
|
|
item.StartedAt = &now
|
||
|
|
}
|
||
|
|
if status == "completed" || status == "failed" {
|
||
|
|
item.CompletedAt = &now
|
||
|
|
}
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
// UpdateProgress updates an item's progress
|
||
|
|
func (q *Queue) UpdateProgress(id string, downloaded, size int64) bool {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.ID == id {
|
||
|
|
item.Downloaded = downloaded
|
||
|
|
item.Size = size
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
// Retry marks a failed item for retry
|
||
|
|
func (q *Queue) Retry(id string) bool {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.ID == id {
|
||
|
|
if item.Retries < item.MaxRetries {
|
||
|
|
item.Retries++
|
||
|
|
item.Status = "pending"
|
||
|
|
item.Error = ""
|
||
|
|
item.StartedAt = nil
|
||
|
|
item.CompletedAt = nil
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
// ClearCompleted removes all completed items
|
||
|
|
func (q *Queue) ClearCompleted() int {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
count := 0
|
||
|
|
var remaining []*QueueItem
|
||
|
|
for _, item := range q.items {
|
||
|
|
if item.Status == "completed" {
|
||
|
|
count++
|
||
|
|
} else {
|
||
|
|
remaining = append(remaining, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if count > 0 {
|
||
|
|
q.items = remaining
|
||
|
|
q.dirty = true
|
||
|
|
if q.config.AutoSave {
|
||
|
|
q.saveUnsafe()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return count
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetAll returns all items
|
||
|
|
func (q *Queue) GetAll() []*QueueItem {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
result := make([]*QueueItem, len(q.items))
|
||
|
|
copy(result, q.items)
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
// Len returns the item count
|
||
|
|
func (q *Queue) Len() int {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
return len(q.items)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Save saves the queue to a file
|
||
|
|
func (q *Queue) Save() error {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
return q.saveUnsafe()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (q *Queue) saveUnsafe() error {
|
||
|
|
if !q.dirty {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// Ensure directory exists
|
||
|
|
dir := filepath.Dir(q.config.QueueFile)
|
||
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||
|
|
return fmt.Errorf("failed to create queue directory: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
data, err := json.MarshalIndent(q.items, "", " ")
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("failed to marshal queue: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Write atomically
|
||
|
|
tmpFile := q.config.QueueFile + ".tmp"
|
||
|
|
if err := os.WriteFile(tmpFile, data, 0600); err != nil {
|
||
|
|
return fmt.Errorf("failed to write queue: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := os.Rename(tmpFile, q.config.QueueFile); err != nil {
|
||
|
|
return fmt.Errorf("failed to rename queue file: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
q.dirty = false
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// Load loads the queue from a file
|
||
|
|
func (q *Queue) Load() error {
|
||
|
|
q.mu.Lock()
|
||
|
|
defer q.mu.Unlock()
|
||
|
|
|
||
|
|
data, err := os.ReadFile(q.config.QueueFile)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
var items []*QueueItem
|
||
|
|
if err := json.Unmarshal(data, &items); err != nil {
|
||
|
|
return fmt.Errorf("failed to unmarshal queue: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Reset downloading items to pending (they were interrupted)
|
||
|
|
for _, item := range items {
|
||
|
|
if item.Status == "downloading" {
|
||
|
|
item.Status = "pending"
|
||
|
|
item.StartedAt = nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
q.items = items
|
||
|
|
q.dirty = false
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetStats returns queue statistics
|
||
|
|
func (q *Queue) GetStats() QueueStats {
|
||
|
|
q.mu.RLock()
|
||
|
|
defer q.mu.RUnlock()
|
||
|
|
|
||
|
|
stats := QueueStats{}
|
||
|
|
for _, item := range q.items {
|
||
|
|
switch item.Status {
|
||
|
|
case "pending":
|
||
|
|
stats.Pending++
|
||
|
|
case "downloading":
|
||
|
|
stats.Downloading++
|
||
|
|
case "completed":
|
||
|
|
stats.Completed++
|
||
|
|
case "failed":
|
||
|
|
stats.Failed++
|
||
|
|
case "paused":
|
||
|
|
stats.Paused++
|
||
|
|
}
|
||
|
|
stats.TotalSize += item.Size
|
||
|
|
stats.TotalDownloaded += item.Downloaded
|
||
|
|
}
|
||
|
|
stats.Total = len(q.items)
|
||
|
|
return stats
|
||
|
|
}
|
||
|
|
|
||
|
|
// QueueStats represents queue statistics
|
||
|
|
type QueueStats struct {
|
||
|
|
Total int `json:"total"`
|
||
|
|
Pending int `json:"pending"`
|
||
|
|
Downloading int `json:"downloading"`
|
||
|
|
Completed int `json:"completed"`
|
||
|
|
Failed int `json:"failed"`
|
||
|
|
Paused int `json:"paused"`
|
||
|
|
TotalSize int64 `json:"total_size"`
|
||
|
|
TotalDownloaded int64 `json:"total_downloaded"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// generateID generates a unique ID with a random suffix to prevent collisions.
|
||
|
|
func generateID() string {
|
||
|
|
b := make([]byte, 4)
|
||
|
|
_, _ = rand.Read(b)
|
||
|
|
return fmt.Sprintf("q_%d_%x", time.Now().UnixNano(), b)
|
||
|
|
}
|
||
|
|
|
||
|
|
// getDefaultQueueFile returns the default path to the queue file
|
||
|
|
func getDefaultQueueFile() string {
|
||
|
|
if xdg := os.Getenv("XDG_STATE_HOME"); xdg != "" {
|
||
|
|
return filepath.Join(xdg, "goget", "queue.json")
|
||
|
|
}
|
||
|
|
if home, err := os.UserHomeDir(); err == nil {
|
||
|
|
return filepath.Join(home, ".local", "state", "goget", "queue.json")
|
||
|
|
}
|
||
|
|
return "goget.queue.json"
|
||
|
|
}
|