feat: initial goget release — modern IPv6-first download utility

internal/recursive/filter.go

@@ -0,0 +1,211 @@
+//go:build linux || freebsd
+// +build linux freebsd
+
+package recursive
+
+import (
+	"mime"
+	"net/url"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// URLFilter filters URLs according to rules
+type URLFilter struct {
+	baseURL         *url.URL
+	maxDepth        int
+	followExternal  bool
+	excludePatterns []*regexp.Regexp
+	includePatterns []*regexp.Regexp
+	acceptPatterns  []string // Glob-style accept patterns
+	noParent        bool
+}
+
+// URLFilterConfig configures the URL filter
+type URLFilterConfig struct {
+	MaxDepth        int
+	FollowExternal  bool
+	ExcludePatterns []string
+	IncludePatterns []string
+	AcceptPatterns  []string // Glob-style accept patterns (e.g., "*.pdf,*.html")
+	NoParent        bool     // Do not ascend to parent directory
+}
+
+// NewURLFilter creates a new filter
+func NewURLFilter(baseURL *url.URL, cfg URLFilterConfig) *URLFilter {
+	filter := &URLFilter{
+		baseURL:         baseURL,
+		maxDepth:        cfg.MaxDepth,
+		followExternal:  cfg.FollowExternal,
+		excludePatterns: make([]*regexp.Regexp, 0),
+		includePatterns: make([]*regexp.Regexp, 0),
+		acceptPatterns:  cfg.AcceptPatterns,
+		noParent:        cfg.NoParent,
+	}
+
+	// Compile exclude patterns
+	for _, pattern := range cfg.ExcludePatterns {
+		if re, err := regexp.Compile(pattern); err == nil {
+			filter.excludePatterns = append(filter.excludePatterns, re)
+		}
+	}
+
+	// Compile include patterns
+	for _, pattern := range cfg.IncludePatterns {
+		if re, err := regexp.Compile(pattern); err == nil {
+			filter.includePatterns = append(filter.includePatterns, re)
+		}
+	}
+
+	return filter
+}
+
+// ShouldDownload determines whether a URL should be downloaded
+func (f *URLFilter) ShouldDownload(u *url.URL, currentDepth int) bool {
+	// Check depth
+	if currentDepth > f.maxDepth {
+		return false
+	}
+
+	// Check scheme
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return false
+	}
+
+	// Check domain (external links)
+	if !f.followExternal && u.Hostname() != f.baseURL.Hostname() {
+		return false
+	}
+
+	// Check exclude patterns
+	urlStr := u.String()
+	for _, re := range f.excludePatterns {
+		if re.MatchString(urlStr) {
+			return false
+		}
+	}
+
+	// Check include patterns (if specified, must match at least one)
+	if len(f.includePatterns) > 0 {
+		matched := false
+		for _, re := range f.includePatterns {
+			if re.MatchString(urlStr) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	// Check accept patterns (glob-style and MIME type)
+	if len(f.acceptPatterns) > 0 {
+		lastPath := u.Path
+		if idx := strings.LastIndex(lastPath, "/"); idx >= 0 {
+			lastPath = lastPath[idx+1:]
+		}
+		matched := false
+		for _, pattern := range f.acceptPatterns {
+			// MIME type pattern (contains "/")
+			if strings.Contains(pattern, "/") {
+				ext := filepath.Ext(lastPath)
+				if ext != "" {
+					mimeType := mime.TypeByExtension(ext)
+					if match, _ := filepath.Match(pattern, mimeType); match {
+						matched = true
+						break
+					}
+				}
+				continue
+			}
+			// Filename glob
+			if match, _ := filepath.Match(pattern, lastPath); match {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	// Check no-parent
+	if f.noParent && f.baseURL != nil {
+		basePath := strings.TrimRight(f.baseURL.Path, "/")
+		targetPath := strings.TrimRight(u.Path, "/")
+		if !strings.HasPrefix(targetPath, basePath) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// IsSameDomain determines whether the URL belongs to the same domain
+func IsSameDomain(base, target *url.URL) bool {
+	if base == nil || target == nil {
+		return false
+	}
+	return base.Hostname() == target.Hostname()
+}
+
+// IsSubPath determines whether target is a subpath of base
+func IsSubPath(base, target *url.URL) bool {
+	if base == nil || target == nil {
+		return false
+	}
+	return strings.HasPrefix(target.Path, base.Path)
+}
+
+// GetLocalPath converts a URL to a local file path
+func GetLocalPath(baseURL, targetURL *url.URL, outputDir string) string {
+	// Start with output directory
+	path := outputDir
+
+	// Add hostname as subdirectory to avoid conflicts
+	if targetURL.Hostname() != baseURL.Hostname() {
+		path = filepath.Join(path, targetURL.Hostname())
+	}
+
+	// Add URL path
+	if targetURL.Path != "" && targetURL.Path != "/" {
+		// Clean the path
+		cleanPath := strings.TrimPrefix(targetURL.Path, "/")
+		path = filepath.Join(path, cleanPath)
+	}
+
+	// If path ends with / or is empty, add index.html
+	if strings.HasSuffix(targetURL.Path, "/") || targetURL.Path == "" {
+		path = filepath.Join(path, "index.html")
+	}
+
+	// If path has no extension, treat as HTML
+	if filepath.Ext(path) == "" {
+		path = path + ".html"
+	}
+
+	return path
+}
+
+// GetFilename extracts the filename from a URL
+func GetFilename(u *url.URL) string {
+	path := u.Path
+	if path == "" || path == "/" {
+		return "index.html"
+	}
+
+	// Get last path segment
+	segments := strings.Split(strings.Trim(path, "/"), "/")
+	if len(segments) == 0 {
+		return "index.html"
+	}
+
+	filename := segments[len(segments)-1]
+	if filename == "" {
+		return "index.html"
+	}
+
+	return filename
+}