//go:build linux || freebsd // +build linux freebsd package recursive import ( "mime" "net/url" "path/filepath" "regexp" "strings" ) // URLFilter filters URLs according to rules type URLFilter struct { baseURL *url.URL maxDepth int followExternal bool excludePatterns []*regexp.Regexp includePatterns []*regexp.Regexp acceptPatterns []string // Glob-style accept patterns noParent bool } // URLFilterConfig configures the URL filter type URLFilterConfig struct { MaxDepth int FollowExternal bool ExcludePatterns []string IncludePatterns []string AcceptPatterns []string // Glob-style accept patterns (e.g., "*.pdf,*.html") NoParent bool // Do not ascend to parent directory } // NewURLFilter creates a new filter func NewURLFilter(baseURL *url.URL, cfg URLFilterConfig) *URLFilter { filter := &URLFilter{ baseURL: baseURL, maxDepth: cfg.MaxDepth, followExternal: cfg.FollowExternal, excludePatterns: make([]*regexp.Regexp, 0), includePatterns: make([]*regexp.Regexp, 0), acceptPatterns: cfg.AcceptPatterns, noParent: cfg.NoParent, } // Compile exclude patterns for _, pattern := range cfg.ExcludePatterns { if re, err := regexp.Compile(pattern); err == nil { filter.excludePatterns = append(filter.excludePatterns, re) } } // Compile include patterns for _, pattern := range cfg.IncludePatterns { if re, err := regexp.Compile(pattern); err == nil { filter.includePatterns = append(filter.includePatterns, re) } } return filter } // ShouldDownload determines whether a URL should be downloaded func (f *URLFilter) ShouldDownload(u *url.URL, currentDepth int) bool { // Check depth if currentDepth > f.maxDepth { return false } // Check scheme if u.Scheme != "http" && u.Scheme != "https" { return false } // Check domain (external links) if !f.followExternal && u.Hostname() != f.baseURL.Hostname() { return false } // Check exclude patterns urlStr := u.String() for _, re := range f.excludePatterns { if re.MatchString(urlStr) { return false } } // Check include patterns (if specified, must match at least one) if len(f.includePatterns) > 0 { matched := false for _, re := range f.includePatterns { if re.MatchString(urlStr) { matched = true break } } if !matched { return false } } // Check accept patterns (glob-style and MIME type) if len(f.acceptPatterns) > 0 { lastPath := u.Path if idx := strings.LastIndex(lastPath, "/"); idx >= 0 { lastPath = lastPath[idx+1:] } matched := false for _, pattern := range f.acceptPatterns { // MIME type pattern (contains "/") if strings.Contains(pattern, "/") { ext := filepath.Ext(lastPath) if ext != "" { mimeType := mime.TypeByExtension(ext) if match, _ := filepath.Match(pattern, mimeType); match { matched = true break } } continue } // Filename glob if match, _ := filepath.Match(pattern, lastPath); match { matched = true break } } if !matched { return false } } // Check no-parent if f.noParent && f.baseURL != nil { basePath := strings.TrimRight(f.baseURL.Path, "/") targetPath := strings.TrimRight(u.Path, "/") if !strings.HasPrefix(targetPath, basePath) { return false } } return true } // IsSameDomain determines whether the URL belongs to the same domain func IsSameDomain(base, target *url.URL) bool { if base == nil || target == nil { return false } return base.Hostname() == target.Hostname() } // IsSubPath determines whether target is a subpath of base func IsSubPath(base, target *url.URL) bool { if base == nil || target == nil { return false } return strings.HasPrefix(target.Path, base.Path) } // GetLocalPath converts a URL to a local file path func GetLocalPath(baseURL, targetURL *url.URL, outputDir string) string { // Start with output directory path := outputDir // Add hostname as subdirectory to avoid conflicts if targetURL.Hostname() != baseURL.Hostname() { path = filepath.Join(path, targetURL.Hostname()) } // Add URL path if targetURL.Path != "" && targetURL.Path != "/" { // Clean the path cleanPath := strings.TrimPrefix(targetURL.Path, "/") path = filepath.Join(path, cleanPath) } // If path ends with / or is empty, add index.html if strings.HasSuffix(targetURL.Path, "/") || targetURL.Path == "" { path = filepath.Join(path, "index.html") } // If path has no extension, treat as HTML if filepath.Ext(path) == "" { path = path + ".html" } return path } // GetFilename extracts the filename from a URL func GetFilename(u *url.URL) string { path := u.Path if path == "" || path == "/" { return "index.html" } // Get last path segment segments := strings.Split(strings.Trim(path, "/"), "/") if len(segments) == 0 { return "index.html" } filename := segments[len(segments)-1] if filename == "" { return "index.html" } return filename }