51 lines
1.4 KiB
Go
51 lines
1.4 KiB
Go
|
//go:build linux || freebsd
|
||
|
|
// +build linux freebsd
|
||
|
|
|
||
|
|
// Package warc provides Web ARChive (WARC) format reading and writing.
|
||
|
|
// It implements WARC/1.0 as defined by ISO 28500.
|
||
|
|
package warc
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Writer writes WARC records to a file.
|
||
|
|
type Writer struct {
|
||
|
|
path string
|
||
|
|
}
|
||
|
|
|
||
|
|
// NewWriter creates a new WARC writer that writes records to the given file.
|
||
|
|
// Each call to WriteRecord or WriteRecordFile overwrites the file.
|
||
|
|
func NewWriter(path string) *Writer {
|
||
|
|
return &Writer{path: path}
|
||
|
|
}
|
||
|
|
|
||
|
|
// WriteRecord writes a WARC response record from raw byte data.
|
||
|
|
// targetURI is the original URL of the downloaded resource.
|
||
|
|
func (w *Writer) WriteRecord(targetURI string, data []byte) error {
|
||
|
|
now := time.Now().UTC().Format(time.RFC3339)
|
||
|
|
warcData := fmt.Sprintf(
|
||
|
|
"WARC/1.0\r\n"+
|
||
|
|
"WARC-Type: response\r\n"+
|
||
|
|
"WARC-Date: %s\r\n"+
|
||
|
|
"WARC-Target-URI: %s\r\n"+
|
||
|
|
"Content-Length: %d\r\n"+
|
||
|
|
"\r\n"+
|
||
|
|
"%s\r\n\r\n",
|
||
|
|
now, targetURI, len(data), string(data))
|
||
|
|
|
||
|
|
return os.WriteFile(w.path, []byte(warcData), 0644)
|
||
|
|
}
|
||
|
|
|
||
|
|
// WriteRecordFile reads the file at filePath and writes it as a WARC record.
|
||
|
|
// targetURI is the original URL of the downloaded resource.
|
||
|
|
func (w *Writer) WriteRecordFile(targetURI, filePath string) error {
|
||
|
|
data, err := os.ReadFile(filePath)
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("failed to read file for WARC: %w", err)
|
||
|
|
}
|
||
|
|
return w.WriteRecord(targetURI, data)
|
||
|
|
}
|