goget/internal/recursive/filter.go

//go:build linux || freebsd
// +build linux freebsd

package recursive

import (
	"mime"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
)

// URLFilter filters URLs according to rules
type URLFilter struct {
	baseURL         *url.URL
	maxDepth        int
	followExternal  bool
	excludePatterns []*regexp.Regexp
	includePatterns []*regexp.Regexp
	acceptPatterns  []string // Glob-style accept patterns
	noParent        bool
}

// URLFilterConfig configures the URL filter
type URLFilterConfig struct {
	MaxDepth        int
	FollowExternal  bool
	ExcludePatterns []string
	IncludePatterns []string
	AcceptPatterns  []string // Glob-style accept patterns (e.g., "*.pdf,*.html")
	NoParent        bool     // Do not ascend to parent directory
}

// NewURLFilter creates a new filter
func NewURLFilter(baseURL *url.URL, cfg URLFilterConfig) *URLFilter {
	filter := &URLFilter{
		baseURL:         baseURL,
		maxDepth:        cfg.MaxDepth,
		followExternal:  cfg.FollowExternal,
		excludePatterns: make([]*regexp.Regexp, 0),
		includePatterns: make([]*regexp.Regexp, 0),
		acceptPatterns:  cfg.AcceptPatterns,
		noParent:        cfg.NoParent,
	}

	// Compile exclude patterns
	for _, pattern := range cfg.ExcludePatterns {
		if re, err := regexp.Compile(pattern); err == nil {
			filter.excludePatterns = append(filter.excludePatterns, re)
		}
	}

	// Compile include patterns
	for _, pattern := range cfg.IncludePatterns {
		if re, err := regexp.Compile(pattern); err == nil {
			filter.includePatterns = append(filter.includePatterns, re)
		}
	}

	return filter
}

// ShouldDownload determines whether a URL should be downloaded
func (f *URLFilter) ShouldDownload(u *url.URL, currentDepth int) bool {
	// Check depth
	if currentDepth > f.maxDepth {
		return false
	}

	// Check scheme
	if u.Scheme != "http" && u.Scheme != "https" {
		return false
	}

	// Check domain (external links)
	if !f.followExternal && u.Hostname() != f.baseURL.Hostname() {
		return false
	}

	// Check exclude patterns
	urlStr := u.String()
	for _, re := range f.excludePatterns {
		if re.MatchString(urlStr) {
			return false
		}
	}

	// Check include patterns (if specified, must match at least one)
	if len(f.includePatterns) > 0 {
		matched := false
		for _, re := range f.includePatterns {
			if re.MatchString(urlStr) {
				matched = true
				break
			}
		}
		if !matched {
			return false
		}
	}

	// Check accept patterns (glob-style and MIME type)
	if len(f.acceptPatterns) > 0 {
		lastPath := u.Path
		if idx := strings.LastIndex(lastPath, "/"); idx >= 0 {
			lastPath = lastPath[idx+1:]
		}
		matched := false
		for _, pattern := range f.acceptPatterns {
			// MIME type pattern (contains "/")
			if strings.Contains(pattern, "/") {
				ext := filepath.Ext(lastPath)
				if ext != "" {
					mimeType := mime.TypeByExtension(ext)
					if match, _ := filepath.Match(pattern, mimeType); match {
						matched = true
						break
					}
				}
				continue
			}
			// Filename glob
			if match, _ := filepath.Match(pattern, lastPath); match {
				matched = true
				break
			}
		}
		if !matched {
			return false
		}
	}

	// Check no-parent
	if f.noParent && f.baseURL != nil {
		basePath := strings.TrimRight(f.baseURL.Path, "/")
		targetPath := strings.TrimRight(u.Path, "/")
		if !strings.HasPrefix(targetPath, basePath) {
			return false
		}
	}

	return true
}

// IsSameDomain determines whether the URL belongs to the same domain
func IsSameDomain(base, target *url.URL) bool {
	if base == nil || target == nil {
		return false
	}
	return base.Hostname() == target.Hostname()
}

// IsSubPath determines whether target is a subpath of base
func IsSubPath(base, target *url.URL) bool {
	if base == nil || target == nil {
		return false
	}
	return strings.HasPrefix(target.Path, base.Path)
}

// GetLocalPath converts a URL to a local file path
func GetLocalPath(baseURL, targetURL *url.URL, outputDir string) string {
	// Start with output directory
	path := outputDir

	// Add hostname as subdirectory to avoid conflicts
	if targetURL.Hostname() != baseURL.Hostname() {
		path = filepath.Join(path, targetURL.Hostname())
	}

	// Add URL path
	if targetURL.Path != "" && targetURL.Path != "/" {
		// Clean the path
		cleanPath := strings.TrimPrefix(targetURL.Path, "/")
		path = filepath.Join(path, cleanPath)
	}

	// If path ends with / or is empty, add index.html
	if strings.HasSuffix(targetURL.Path, "/") || targetURL.Path == "" {
		path = filepath.Join(path, "index.html")
	}

	// If path has no extension, treat as HTML
	if filepath.Ext(path) == "" {
		path = path + ".html"
	}

	return path
}

// GetFilename extracts the filename from a URL
func GetFilename(u *url.URL) string {
	path := u.Path
	if path == "" || path == "/" {
		return "index.html"
	}

	// Get last path segment
	segments := strings.Split(strings.Trim(path, "/"), "/")
	if len(segments) == 0 {
		return "index.html"
	}

	filename := segments[len(segments)-1]
	if filename == "" {
		return "index.html"
	}

	return filename
}