449 lines
12 KiB
Go
449 lines
12 KiB
Go
|
//go:build linux || freebsd
|
||
|
|
// +build linux freebsd
|
||
|
|
|
||
|
|
package linkrewrite
|
||
|
|
|
||
|
|
import (
|
||
|
|
"net/url"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"sync"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"golang.org/x/net/html"
|
||
|
|
)
|
||
|
|
|
||
|
|
func parseHTML(t *testing.T, htmlStr string) *html.Node {
|
||
|
|
t.Helper()
|
||
|
|
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("failed to parse html: %v", err)
|
||
|
|
}
|
||
|
|
return doc
|
||
|
|
}
|
||
|
|
|
||
|
|
func getAttr(t *testing.T, n *html.Node, tag, attrName string) string {
|
||
|
|
t.Helper()
|
||
|
|
var find func(*html.Node)
|
||
|
|
var val string
|
||
|
|
find = func(n *html.Node) {
|
||
|
|
if n.Type == html.ElementNode && n.Data == tag {
|
||
|
|
for _, a := range n.Attr {
|
||
|
|
if a.Key == attrName {
|
||
|
|
val = a.Val
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||
|
|
find(c)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
find(n)
|
||
|
|
return val
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNew(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com/")
|
||
|
|
outDir := "/tmp/mirror"
|
||
|
|
rw := New(base, outDir)
|
||
|
|
|
||
|
|
if rw.baseURL.String() != "https://example.com/" {
|
||
|
|
t.Errorf("expected baseURL https://example.com/, got %s", rw.baseURL.String())
|
||
|
|
}
|
||
|
|
if rw.outputDir != outDir {
|
||
|
|
t.Errorf("expected outputDir /tmp/mirror, got %s", rw.outputDir)
|
||
|
|
}
|
||
|
|
if rw.visited == nil {
|
||
|
|
t.Error("expected visited map to be initialized")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRegister(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
pageURL, _ := url.Parse("https://example.com/about.html")
|
||
|
|
rw.Register(pageURL, "/out/about.html")
|
||
|
|
|
||
|
|
rw.visitedMu.RLock()
|
||
|
|
got := rw.visited["https://example.com/about.html"]
|
||
|
|
rw.visitedMu.RUnlock()
|
||
|
|
|
||
|
|
if got != "/out/about.html" {
|
||
|
|
t.Errorf("expected /out/about.html, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksNoRegistered(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><a href="/page">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 0 {
|
||
|
|
t.Errorf("expected 0 conversions, got %d", converted)
|
||
|
|
}
|
||
|
|
// href should remain unchanged
|
||
|
|
got := getAttr(t, doc, "a", "href")
|
||
|
|
if got != "/page" {
|
||
|
|
t.Errorf("expected /page, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksAnchorHref(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
pageURL, _ := url.Parse("https://example.com/page.html")
|
||
|
|
rw.Register(pageURL, "/out/page.html")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><a href="/page.html">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "a", "href")
|
||
|
|
if got != "page.html" {
|
||
|
|
t.Errorf("expected relative path, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksImageSrc(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
imgURL, _ := url.Parse("https://example.com/images/logo.png")
|
||
|
|
rw.Register(imgURL, "/out/images/logo.png")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><img src="/images/logo.png"></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "img", "src")
|
||
|
|
if got != "images/logo.png" {
|
||
|
|
t.Errorf("expected images/logo.png, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksFormAction(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
formURL, _ := url.Parse("https://example.com/submit")
|
||
|
|
rw.Register(formURL, "/out/submit.html")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><form action="/submit"></form></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "form", "action")
|
||
|
|
if got != "submit.html" {
|
||
|
|
t.Errorf("expected submit.html, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksMediaElements(t *testing.T) {
|
||
|
|
tests := []struct{ tag, urlStr, localPath string }{
|
||
|
|
{"script", "https://example.com/app.js", "/out/js/app.js"},
|
||
|
|
{"video", "https://example.com/video.mp4", "/out/video.mp4"},
|
||
|
|
{"audio", "https://example.com/sound.mp3", "/out/sound.mp3"},
|
||
|
|
{"source", "https://example.com/video.webm", "/out/video.webm"},
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, tt := range tests {
|
||
|
|
t.Run(tt.tag, func(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
u, _ := url.Parse(tt.urlStr)
|
||
|
|
rw.Register(u, tt.localPath)
|
||
|
|
|
||
|
|
htmlStr := `<html><body><` + tt.tag + ` src="` + tt.urlStr + `"></` + tt.tag + `></body></html>`
|
||
|
|
doc := parseHTML(t, htmlStr)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksSkipsProtocols(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
|
||
|
|
prefixes := []string{"#section", "javascript:void(0)", "mailto:a@b.com", "tel:+123", "data:text/plain", "about:blank"}
|
||
|
|
|
||
|
|
for _, prefix := range prefixes {
|
||
|
|
t.Run(prefix, func(t *testing.T) {
|
||
|
|
rw := New(base, "/out")
|
||
|
|
doc := parseHTML(t, `<html><body><a href="`+prefix+`">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
if converted != 0 {
|
||
|
|
t.Errorf("expected 0 conversions for %s, got %d", prefix, converted)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksRelativeResolution(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com/subdir/")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
pageURL, _ := url.Parse("https://example.com/subdir/page.html")
|
||
|
|
rw.Register(pageURL, "/out/subdir/page.html")
|
||
|
|
|
||
|
|
// Relative URL should resolve against base
|
||
|
|
doc := parseHTML(t, `<html><body><a href="page.html">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/subdir/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "a", "href")
|
||
|
|
if got != "page.html" {
|
||
|
|
t.Errorf("expected page.html, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksNestedRelative(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
imgURL, _ := url.Parse("https://example.com/assets/img/photo.jpg")
|
||
|
|
rw.Register(imgURL, "/out/assets/img/photo.jpg")
|
||
|
|
|
||
|
|
// page.html is in /out/blog/ — image should resolve to ../assets/img/photo.jpg
|
||
|
|
doc := parseHTML(t, `<html><body><img src="/assets/img/photo.jpg"></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/blog/page.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "img", "src")
|
||
|
|
if got != "../assets/img/photo.jpg" {
|
||
|
|
t.Errorf("expected ../assets/img/photo.jpg, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksMetaRefresh(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
targetURL, _ := url.Parse("https://example.com/new-page.html")
|
||
|
|
rw.Register(targetURL, "/out/new-page.html")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><head><meta http-equiv="refresh" content="5; url=/new-page.html"></head></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "meta", "content")
|
||
|
|
if !strings.Contains(got, "url=new-page.html") {
|
||
|
|
t.Errorf("expected content to contain url=new-page.html, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksOGImage(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
imgURL, _ := url.Parse("https://example.com/og-image.jpg")
|
||
|
|
rw.Register(imgURL, "/out/og-image.jpg")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><head><meta property="og:image" content="/og-image.jpg"></head></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "meta", "content")
|
||
|
|
if got != "og-image.jpg" {
|
||
|
|
t.Errorf("expected og-image.jpg, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksAreaTag(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
mapURL, _ := url.Parse("https://example.com/section.html")
|
||
|
|
rw.Register(mapURL, "/out/section.html")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><map><area href="/section.html"></map></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksLinkTag(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
cssURL, _ := url.Parse("https://example.com/style.css")
|
||
|
|
rw.Register(cssURL, "/out/style.css")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><head><link rel="stylesheet" href="/style.css"></head></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "link", "href")
|
||
|
|
if got != "style.css" {
|
||
|
|
t.Errorf("expected style.css, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksMultipleConversions(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
urls := []struct{ abs, local string }{
|
||
|
|
{"https://example.com/a.html", "/out/a.html"},
|
||
|
|
{"https://example.com/img1.png", "/out/img1.png"},
|
||
|
|
{"https://example.com/img2.png", "/out/img2.png"},
|
||
|
|
}
|
||
|
|
for _, u := range urls {
|
||
|
|
pu, _ := url.Parse(u.abs)
|
||
|
|
rw.Register(pu, u.local)
|
||
|
|
}
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body>
|
||
|
|
<a href="/a.html">link</a>
|
||
|
|
<img src="/img1.png">
|
||
|
|
<img src="/img2.png">
|
||
|
|
</body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 3 {
|
||
|
|
t.Errorf("expected 3 conversions, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksOnlyRegistered(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
// Only register one of two URLs
|
||
|
|
regURL, _ := url.Parse("https://example.com/registered.html")
|
||
|
|
rw.Register(regURL, "/out/registered.html")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body>
|
||
|
|
<a href="/registered.html">yes</a>
|
||
|
|
<a href="/unregistered.html">no</a>
|
||
|
|
</body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksEmptyDocument(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
doc := parseHTML(t, ``)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
if converted != 0 {
|
||
|
|
t.Errorf("expected 0 conversions for empty doc, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksNoHtmlElement(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `just text`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
if converted != 0 {
|
||
|
|
t.Errorf("expected 0 conversions for text-only doc, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksForwardSlashConversion(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
pageURL, _ := url.Parse("https://example.com/dir/page.html")
|
||
|
|
rw.Register(pageURL, filepath.FromSlash("/out/dir/page.html"))
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><body><a href="/dir/page.html">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, filepath.FromSlash("/out/index.html"))
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "a", "href")
|
||
|
|
// Should use forward slashes
|
||
|
|
if !strings.Contains(got, "/") && strings.Contains(got, "\\") {
|
||
|
|
t.Errorf("expected forward-slash path, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRegisterConcurrent(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
for i := 0; i < 100; i++ {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(n int) {
|
||
|
|
defer wg.Done()
|
||
|
|
u, _ := url.Parse("https://example.com/page" + string(rune('0'+n%10)) + ".html")
|
||
|
|
rw.Register(u, "/out/p.html")
|
||
|
|
}(i)
|
||
|
|
}
|
||
|
|
wg.Wait()
|
||
|
|
|
||
|
|
rw.visitedMu.RLock()
|
||
|
|
count := len(rw.visited)
|
||
|
|
rw.visitedMu.RUnlock()
|
||
|
|
|
||
|
|
if count == 0 {
|
||
|
|
t.Error("expected visited map to have entries after concurrent registration")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksAbsoluteURLWithPath(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com/blog/")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
// Register with absolute URL
|
||
|
|
pageURL, _ := url.Parse("https://example.com/blog/post.html")
|
||
|
|
rw.Register(pageURL, "/out/blog/post.html")
|
||
|
|
|
||
|
|
// Absolute href in HTML
|
||
|
|
doc := parseHTML(t, `<html><body><a href="https://example.com/blog/post.html">link</a></body></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/blog/index.html")
|
||
|
|
|
||
|
|
if converted != 1 {
|
||
|
|
t.Errorf("expected 1 conversion, got %d", converted)
|
||
|
|
}
|
||
|
|
got := getAttr(t, doc, "a", "href")
|
||
|
|
if got != "post.html" {
|
||
|
|
t.Errorf("expected post.html, got %s", got)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRewriteLinksMetaRefreshNoRegistered(t *testing.T) {
|
||
|
|
base, _ := url.Parse("https://example.com")
|
||
|
|
rw := New(base, "/out")
|
||
|
|
|
||
|
|
doc := parseHTML(t, `<html><head><meta http-equiv="refresh" content="5; url=/nonexistent"></head></html>`)
|
||
|
|
converted := rw.RewriteLinks(doc, "/out/index.html")
|
||
|
|
|
||
|
|
if converted != 0 {
|
||
|
|
t.Errorf("expected 0 conversions, got %d", converted)
|
||
|
|
}
|
||
|
|
}
|