//go:build linux || freebsd // +build linux freebsd // Package warc provides Web ARChive (WARC) format reading and writing. // It implements WARC/1.0 as defined by ISO 28500. package warc import ( "fmt" "os" "time" ) // Writer writes WARC records to a file. type Writer struct { path string } // NewWriter creates a new WARC writer that writes records to the given file. // Each call to WriteRecord or WriteRecordFile overwrites the file. func NewWriter(path string) *Writer { return &Writer{path: path} } // WriteRecord writes a WARC response record from raw byte data. // targetURI is the original URL of the downloaded resource. func (w *Writer) WriteRecord(targetURI string, data []byte) error { now := time.Now().UTC().Format(time.RFC3339) warcData := fmt.Sprintf( "WARC/1.0\r\n"+ "WARC-Type: response\r\n"+ "WARC-Date: %s\r\n"+ "WARC-Target-URI: %s\r\n"+ "Content-Length: %d\r\n"+ "\r\n"+ "%s\r\n\r\n", now, targetURI, len(data), string(data)) return os.WriteFile(w.path, []byte(warcData), 0644) } // WriteRecordFile reads the file at filePath and writes it as a WARC record. // targetURI is the original URL of the downloaded resource. func (w *Writer) WriteRecordFile(targetURI, filePath string) error { data, err := os.ReadFile(filePath) if err != nil { return fmt.Errorf("failed to read file for WARC: %w", err) } return w.WriteRecord(targetURI, data) }