feat: initial goget release — modern IPv6-first download utility
This commit is contained in:
@@ -0,0 +1,431 @@
|
||||
//go:build linux || freebsd
|
||||
// +build linux freebsd
|
||||
|
||||
package metalink
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Metalink represents a metalink file (RFC 5854)
|
||||
type Metalink struct {
|
||||
XMLName xml.Name `xml:"metalink"`
|
||||
XMLNS string `xml:"xmlns,attr"`
|
||||
Version string `xml:"version,attr,omitempty"`
|
||||
|
||||
// Original URL of the metalink
|
||||
OriginURL string `xml:"-"`
|
||||
|
||||
// Files to download
|
||||
Files []File `xml:"file"`
|
||||
}
|
||||
|
||||
// File represents a single file in a metalink
|
||||
type File struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Size int64 `xml:"size,attr,omitempty"`
|
||||
|
||||
// Identifiers
|
||||
MetaURL MetaURL `xml:"metaurl,omitempty"`
|
||||
URLs []URL `xml:"url"`
|
||||
|
||||
// Hashes for verification
|
||||
Hashes []Hash `xml:"hash"`
|
||||
|
||||
// Language preferences
|
||||
Lang string `xml:"lang,attr,omitempty"`
|
||||
|
||||
// Localized names
|
||||
LocalName []LocalName `xml:"localname,omitempty"`
|
||||
|
||||
// Description
|
||||
Desc string `xml:"desc,omitempty"`
|
||||
|
||||
// Patches and signatures
|
||||
Patches []Patch `xml:"patch,omitempty"`
|
||||
Signatures []Signature `xml:"signature,omitempty"`
|
||||
}
|
||||
|
||||
// MetaURL links to another metalink (for mirror selection)
|
||||
type MetaURL struct {
|
||||
URL string `xml:",chardata"`
|
||||
Mediator string `xml:"mediator,attr,omitempty"`
|
||||
Priority int `xml:"priority,attr,omitempty"`
|
||||
Location string `xml:"location,attr,omitempty"`
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
// URL represents a single source for download
|
||||
type URL struct {
|
||||
URL string `xml:",chardata"`
|
||||
Location string `xml:"location,attr,omitempty"` // Geographic location
|
||||
Priority int `xml:"priority,attr,omitempty"` // 1-100, lower = higher priority
|
||||
MaxConn int `xml:"max-connections,attr,omitempty"`
|
||||
Type string `xml:"type,attr,omitempty"`
|
||||
}
|
||||
|
||||
// Hash represents a checksum for verification
|
||||
type Hash struct {
|
||||
Value string `xml:",chardata"`
|
||||
Type string `xml:"type,attr"` // sha-256, sha-512, md5, etc.
|
||||
}
|
||||
|
||||
// LocalName localized name of the file
|
||||
type LocalName struct {
|
||||
Name string `xml:",chardata"`
|
||||
Lang string `xml:"xml:lang,attr"`
|
||||
}
|
||||
|
||||
// Patch information for updating
|
||||
type Patch struct {
|
||||
URL string `xml:"url,attr"`
|
||||
MetaURL string `xml:"metaurl,attr,omitempty"`
|
||||
Size int64 `xml:"size,attr"`
|
||||
Hash Hash `xml:"hash,omitempty"`
|
||||
}
|
||||
|
||||
// Signature PGP/GPG signature
|
||||
type Signature struct {
|
||||
Value string `xml:",chardata"`
|
||||
Type string `xml:"type,attr"` // pgp, smime, etc.
|
||||
}
|
||||
|
||||
// Parse parses a metalink file
|
||||
func Parse(r io.Reader) (*Metalink, error) {
|
||||
var ml Metalink
|
||||
decoder := xml.NewDecoder(r)
|
||||
if err := decoder.Decode(&ml); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse metalink: %w", err)
|
||||
}
|
||||
return &ml, nil
|
||||
}
|
||||
|
||||
// ParseFile parses a metalink from a file
|
||||
func ParseFile(path string) (*Metalink, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open metalink file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ml, err := Parse(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ml.OriginURL = path
|
||||
return ml, nil
|
||||
}
|
||||
|
||||
// ParseURL fetches and parses a metalink from a URL
|
||||
func ParseURL(urlStr string) (*Metalink, error) {
|
||||
resp, err := http.Get(urlStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch metalink url: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("metalink url returned status %s", resp.Status)
|
||||
}
|
||||
|
||||
ml, err := Parse(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ml.OriginURL = urlStr
|
||||
return ml, nil
|
||||
}
|
||||
|
||||
// GetFiles returns all files from the metalink
|
||||
func (m *Metalink) GetFiles() []File {
|
||||
return m.Files
|
||||
}
|
||||
|
||||
// GetFileByName finds a file by name
|
||||
func (m *Metalink) GetFileByName(name string) *File {
|
||||
for i := range m.Files {
|
||||
if m.Files[i].Name == name {
|
||||
return &m.Files[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetURLs returns all URLs for the file sorted by priority
|
||||
func (f *File) GetURLs() []URL {
|
||||
// Sort by priority (lower = higher priority)
|
||||
sorted := make([]URL, len(f.URLs))
|
||||
copy(sorted, f.URLs)
|
||||
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].Priority == 0 {
|
||||
sorted[i].Priority = 50 // Default priority
|
||||
}
|
||||
if sorted[j].Priority == 0 {
|
||||
sorted[j].Priority = 50
|
||||
}
|
||||
return sorted[i].Priority < sorted[j].Priority
|
||||
})
|
||||
|
||||
return sorted
|
||||
}
|
||||
|
||||
// GetBestURL returns the URL with the highest priority
|
||||
func (f *File) GetBestURL() *URL {
|
||||
urls := f.GetURLs()
|
||||
if len(urls) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &urls[0]
|
||||
}
|
||||
|
||||
// GetHash gets a hash of the given type
|
||||
func (f *File) GetHash(hashType string) string {
|
||||
for _, h := range f.Hashes {
|
||||
if strings.EqualFold(h.Type, hashType) {
|
||||
return h.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetSHA256 returns the SHA-256 hash
|
||||
func (f *File) GetSHA256() string {
|
||||
return f.GetHash("sha-256")
|
||||
}
|
||||
|
||||
// GetSHA512 returns the SHA-512 hash
|
||||
func (f *File) GetSHA512() string {
|
||||
return f.GetHash("sha-512")
|
||||
}
|
||||
|
||||
// GetMD5 returns the MD5 hash
|
||||
func (f *File) GetMD5() string {
|
||||
return f.GetHash("md5")
|
||||
}
|
||||
|
||||
// HasHash checks if file has a hash of given type
|
||||
func (f *File) HasHash(hashType string) bool {
|
||||
return f.GetHash(hashType) != ""
|
||||
}
|
||||
|
||||
// GetSize returns the size of the file
|
||||
func (f *File) GetSize() int64 {
|
||||
return f.Size
|
||||
}
|
||||
|
||||
// IsValid checks whether the file has at least one URL
|
||||
func (f *File) IsValid() bool {
|
||||
return len(f.URLs) > 0 || f.MetaURL.URL != ""
|
||||
}
|
||||
|
||||
// GetURLsByLocation returns URLs filtered by location
|
||||
func (f *File) GetURLsByLocation(location string) []URL {
|
||||
var result []URL
|
||||
for _, u := range f.URLs {
|
||||
if u.Location == "" || strings.EqualFold(u.Location, location) {
|
||||
result = append(result, u)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetURLsByType returns URLs of the given type (http, https, ftp, etc.)
|
||||
func (f *File) GetURLsByType(urlType string) []URL {
|
||||
var result []URL
|
||||
for _, u := range f.URLs {
|
||||
if u.Type == "" || strings.EqualFold(u.Type, urlType) {
|
||||
result = append(result, u)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetAllURLs returns all URLs from all files
|
||||
func (m *Metalink) GetAllURLs() []DownloadSource {
|
||||
var sources []DownloadSource
|
||||
|
||||
for _, file := range m.Files {
|
||||
for _, u := range file.URLs {
|
||||
sources = append(sources, DownloadSource{
|
||||
FileName: file.Name,
|
||||
URL: u.URL,
|
||||
Priority: u.Priority,
|
||||
Size: file.Size,
|
||||
Hashes: file.Hashes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return sources
|
||||
}
|
||||
|
||||
// DownloadSource represents a single source for download
|
||||
type DownloadSource struct {
|
||||
FileName string
|
||||
URL string
|
||||
Priority int
|
||||
Size int64
|
||||
Hashes []Hash
|
||||
}
|
||||
|
||||
// GetUniqueURLs returns unique URLs across all files
|
||||
func (m *Metalink) GetUniqueURLs() []string {
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
|
||||
for _, file := range m.Files {
|
||||
for _, u := range file.URLs {
|
||||
if !seen[u.URL] {
|
||||
seen[u.URL] = true
|
||||
result = append(result, u.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Validate checks the validity of the metalink
|
||||
func (m *Metalink) Validate() error {
|
||||
if len(m.Files) == 0 {
|
||||
return fmt.Errorf("metalink contains no files")
|
||||
}
|
||||
|
||||
for i, file := range m.Files {
|
||||
if file.Name == "" {
|
||||
return fmt.Errorf("file %d has no name", i)
|
||||
}
|
||||
if !file.IsValid() {
|
||||
return fmt.Errorf("file %s has no valid urls", file.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTotalSize calculates the total size of all files
|
||||
func (m *Metalink) GetTotalSize() int64 {
|
||||
var total int64
|
||||
for _, f := range m.Files {
|
||||
total += f.Size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// SelectMirrors selects the best mirrors for download
|
||||
func (m *Metalink) SelectMirrors(maxMirrors int, preferredLocation string) []URL {
|
||||
var allURLs []URL
|
||||
|
||||
for _, file := range m.Files {
|
||||
allURLs = append(allURLs, file.GetURLs()...)
|
||||
}
|
||||
|
||||
// Sort by priority
|
||||
sort.Slice(allURLs, func(i, j int) bool {
|
||||
pi := allURLs[i].Priority
|
||||
pj := allURLs[j].Priority
|
||||
if pi == 0 {
|
||||
pi = 50
|
||||
}
|
||||
if pj == 0 {
|
||||
pj = 50
|
||||
}
|
||||
|
||||
// Prefer URLs matching preferred location
|
||||
if preferredLocation != "" {
|
||||
iMatch := strings.EqualFold(allURLs[i].Location, preferredLocation)
|
||||
jMatch := strings.EqualFold(allURLs[j].Location, preferredLocation)
|
||||
if iMatch && !jMatch {
|
||||
return true
|
||||
}
|
||||
if !iMatch && jMatch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return pi < pj
|
||||
})
|
||||
|
||||
// Return unique URLs up to maxMirrors
|
||||
seen := make(map[string]bool)
|
||||
var result []URL
|
||||
|
||||
for _, u := range allURLs {
|
||||
if !seen[u.URL] {
|
||||
seen[u.URL] = true
|
||||
result = append(result, u)
|
||||
if len(result) >= maxMirrors {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// IsMetalinkFile detects whether a file is a metalink by its extension
|
||||
func IsMetalinkFile(path string) bool {
|
||||
lower := strings.ToLower(path)
|
||||
return strings.HasSuffix(lower, ".metalink") ||
|
||||
strings.HasSuffix(lower, ".meta4") ||
|
||||
strings.HasSuffix(lower, ".metalink4")
|
||||
}
|
||||
|
||||
// IsMetalinkContentType detects a metalink by its Content-Type header
|
||||
func IsMetalinkContentType(contentType string) bool {
|
||||
ct := strings.ToLower(contentType)
|
||||
return strings.Contains(ct, "application/metalink+xml") ||
|
||||
strings.Contains(ct, "application/metalink4+xml")
|
||||
}
|
||||
|
||||
// ParseURLs parses a metalink from a string (for inline metalinks)
|
||||
func ParseURLs(metalinkXML string) (*Metalink, error) {
|
||||
return Parse(strings.NewReader(metalinkXML))
|
||||
}
|
||||
|
||||
// GetPreferredProtocol returns the preferred protocol from URLs
|
||||
func GetPreferredProtocol(urls []URL) string {
|
||||
// Prefer HTTPS > HTTP > FTP
|
||||
protocolPriority := map[string]int{
|
||||
"https": 1,
|
||||
"http": 2,
|
||||
"ftp": 3,
|
||||
"ftps": 2,
|
||||
}
|
||||
|
||||
bestProtocol := ""
|
||||
bestPriority := 999
|
||||
|
||||
for _, u := range urls {
|
||||
parsed, err := url.Parse(u.URL)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
prio := protocolPriority[parsed.Scheme]
|
||||
if prio == 0 {
|
||||
prio = 50 // Unknown protocol
|
||||
}
|
||||
|
||||
if prio < bestPriority {
|
||||
bestPriority = prio
|
||||
bestProtocol = parsed.Scheme
|
||||
}
|
||||
}
|
||||
|
||||
if bestProtocol == "" {
|
||||
return "https"
|
||||
}
|
||||
|
||||
return bestProtocol
|
||||
}
|
||||
Reference in New Issue
Block a user