feat: initial release
Assisted-by: GLM 5.3 Flash
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package io
|
||||
|
||||
// Parity between the hand-rolled CSV tokenizer and the encoding/csv
|
||||
// implementation it replaced. The legacy reader below is the oracle:
|
||||
// the differential test walks a corpus of inputs through both and
|
||||
// demands the same error text, the same shape and the same float bits,
|
||||
// and the fuzz target hunts for an input where they part ways.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"math"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sourcedock.dev/petrbalvin/tensor/internal/base"
|
||||
"sourcedock.dev/petrbalvin/tensor/internal/core"
|
||||
)
|
||||
|
||||
// loadCSVReaderLegacy is the encoding/csv reader the tokenizer
|
||||
// replaced, kept verbatim, byte order mark, buffer growth included.
|
||||
func loadCSVReaderLegacy(r io.Reader, skipHeader bool) (*core.Array, error) {
|
||||
br := bufio.NewReader(r)
|
||||
prefix, _ := br.Peek(3)
|
||||
if len(prefix) == 3 && prefix[0] == 0xEF && prefix[1] == 0xBB && prefix[2] == 0xBF {
|
||||
_, _, _ = br.ReadRune() // consume the mark
|
||||
}
|
||||
cr := csv.NewReader(br)
|
||||
cr.FieldsPerRecord = -1 // allow variable; validated manually
|
||||
// The record is consumed before the next Read, so the reader may
|
||||
// reuse its slice: one row buffer serves the whole file.
|
||||
cr.ReuseRecord = true
|
||||
vals := make([]float64, 0)
|
||||
rows, cols := 0, 0
|
||||
header := skipHeader
|
||||
var ragged, badValue error
|
||||
for {
|
||||
rec, err := cr.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, base.Errf("LoadCSV: %w", err)
|
||||
}
|
||||
if header {
|
||||
header = false
|
||||
continue
|
||||
}
|
||||
if rows == 0 {
|
||||
cols = len(rec)
|
||||
} else if len(rec) != cols {
|
||||
if ragged == nil {
|
||||
ragged = base.Errf("LoadCSV: row %d has %d fields, want %d", rows, len(rec), cols)
|
||||
}
|
||||
rows++
|
||||
continue
|
||||
}
|
||||
if badValue == nil {
|
||||
if need := len(vals) + len(rec); need > cap(vals) {
|
||||
grown := make([]float64, len(vals), max(2*cap(vals), need))
|
||||
copy(grown, vals)
|
||||
vals = grown
|
||||
}
|
||||
for j, f := range rec {
|
||||
v, perr := strconv.ParseFloat(f, 64)
|
||||
if perr != nil {
|
||||
badValue = base.Errf("LoadCSV: row %d col %d: %w", rows, j, perr)
|
||||
break
|
||||
}
|
||||
vals = append(vals, v)
|
||||
}
|
||||
}
|
||||
rows++
|
||||
}
|
||||
if ragged != nil {
|
||||
return nil, ragged
|
||||
}
|
||||
if badValue != nil {
|
||||
return nil, badValue
|
||||
}
|
||||
return core.FromFloats(vals, rows, cols)
|
||||
}
|
||||
|
||||
// csvParityCases collects the inputs whose treatment the tokenizer has
|
||||
// to match byte for byte: the shapes a numeric table takes, the quote
|
||||
// grammar, the line-ending conventions and every defect the reader
|
||||
// reports.
|
||||
var csvParityCases = []struct {
|
||||
name string
|
||||
in string
|
||||
header bool
|
||||
}{
|
||||
{"plain table", "1.5,2.25\n3.125,4\n", false},
|
||||
{"last row without a newline", "1,2\n3,4", false},
|
||||
{"single row without a newline", "1,2", false},
|
||||
{"crlf", "1,2\r\n3,4\r\n", false},
|
||||
{"mixed line endings", "1,2\r\n3,4\n5,6\r\n", false},
|
||||
{"blank lines between records", "1,2\n\n3,4\n\n", false},
|
||||
{"blank line only", "\n\n\n", false},
|
||||
{"empty input", "", false},
|
||||
{"empty field", "1,,3\n4,5,6\n", false},
|
||||
{"trailing empty field", "1,2,\n3,4,5\n", false},
|
||||
{"single comma", ",", false},
|
||||
{"spaces stay in the field", " 1 , 2 \n3,4\n", false},
|
||||
{"lone carriage return inside a line", "1,2\r3,4\n", false},
|
||||
{"carriage return before eof", "1,2\r", false},
|
||||
{"carriage return blank line", "1,2\n\r\n3,4\n", false},
|
||||
{"bom then table", "\xEF\xBB\xBF1.5,2.5\n3,4\n", false},
|
||||
{"quoted values", "1,\"2\"\n\"3\",4\n", false},
|
||||
{"quoted header", "\"a\",\"b\"\n1,2\n", true},
|
||||
{"quoted comma", "\"1,2\",3\n4,5,6\n", false},
|
||||
{"escaped quote", "\"a\"\"b\",2\n", false},
|
||||
{"quote run at field end", "\"a\"\"\",2\n", false},
|
||||
{"empty quoted fields", "\"\",\"\",\"\"\n", false},
|
||||
{"multiline quoted field", "\"1\n2\",3\n4,5,6\n", false},
|
||||
{"multiline quoted field with crlf", "\"1\r\n2\",3\n4,5,6\n", false},
|
||||
{"blank line inside a quoted field", "\"a\n\nb\",1\n", false},
|
||||
{"quote only field mid table", "1,2\n\"3\",4\n", false},
|
||||
{"header skip", "a,b\n1,2\n", true},
|
||||
{"header skip with blank line", "a,b\n\n1,2\n", true},
|
||||
{"ragged rows", "1,2\n3\n4,5,6\n", false},
|
||||
{"ragged rows with header", "a,b\n1,2\n3\n", true},
|
||||
{"two bad values", "1,x\n4,y\n", false},
|
||||
{"bad value after ragged row", "1,2\n3\n4,x\n", false},
|
||||
{"bare quote", "a\"b,1\n", false},
|
||||
{"bare quote in a later field", "1,b\"c,2\n", false},
|
||||
{"text after a closing quote", "1,\"a\"b,2\n", false},
|
||||
{"text after a closing quote on a later line", "1\n\"a\"b,2\n", false},
|
||||
{"unterminated quote at eof", "1,2\n3,\"ab", false},
|
||||
{"unterminated quote at end of line", "1,2\n3,\"ab\n", false},
|
||||
{"unterminated quote after a multiline field", "1,\"a\nb", false},
|
||||
{"unterminated quote in the header", "\"a,b\n1,2\n", true},
|
||||
{"only a header", "a,b\n", true},
|
||||
{"only a header without skip", "a,b\n", false},
|
||||
{"range overflow", "1e309,2\n", false},
|
||||
{"range underflow", "1,1e-400\n", false},
|
||||
{"hex float", "0x1p-2,2\n", false},
|
||||
{"infinity spelling", "Inf,+Inf,-inf\n", false},
|
||||
{"nan spelling", "NaN,-nan\n", false},
|
||||
{"utf-8 in a field", "λ,2\n", false},
|
||||
{"wide utf-8 in a field", "1,𝄞\n", false},
|
||||
}
|
||||
|
||||
// csvParityExtra builds the generated cases the fixed list cannot
|
||||
// spell: records longer than the read buffer, quoted or not, and a
|
||||
// generated table in the shape SaveCSV emits.
|
||||
func csvParityExtra(t *testing.T) []struct {
|
||||
name string
|
||||
in string
|
||||
header bool
|
||||
} {
|
||||
t.Helper()
|
||||
var sb strings.Builder
|
||||
for j := range 40000 {
|
||||
if j > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
sb.WriteString(strconv.Itoa(j))
|
||||
}
|
||||
longLine := sb.String()
|
||||
var qb strings.Builder
|
||||
qb.WriteString("1,\"")
|
||||
qb.WriteString(strings.Repeat("234567890\n", 10000))
|
||||
qb.WriteString("\",2\n")
|
||||
return []struct {
|
||||
name string
|
||||
in string
|
||||
header bool
|
||||
}{
|
||||
{"record longer than the read buffer", longLine + "\n1,2\n", false},
|
||||
{"long record without a newline", longLine, false},
|
||||
{"long quoted field across the buffer", qb.String(), false},
|
||||
{"ragged long record", longLine + "\n1\n", false},
|
||||
{"long last field without a newline", "1," + strings.Repeat("2", 70000), false},
|
||||
}
|
||||
}
|
||||
|
||||
// checkCSVParity runs one input through both readers and demands the
|
||||
// same outcome: the same error text, or the same shape and the same
|
||||
// float bits.
|
||||
func checkCSVParity(t *testing.T, name, in string, skipHeader bool) {
|
||||
t.Helper()
|
||||
want, wantErr := loadCSVReaderLegacy(strings.NewReader(in), skipHeader)
|
||||
got, gotErr := LoadCSVReader(strings.NewReader(in), skipHeader)
|
||||
switch {
|
||||
case wantErr != nil && gotErr != nil:
|
||||
if wantErr.Error() != gotErr.Error() {
|
||||
t.Fatalf("%s: error %q, want %q", name, gotErr, wantErr)
|
||||
}
|
||||
return
|
||||
case wantErr != nil || gotErr != nil:
|
||||
t.Fatalf("%s: error mismatch: legacy %v, tokenizer %v", name, wantErr, gotErr)
|
||||
}
|
||||
if !slices.Equal(want.Shape(), got.Shape()) {
|
||||
t.Fatalf("%s: shape %v, want %v", name, got.Shape(), want.Shape())
|
||||
}
|
||||
wv, gv := want.RawFloats(), got.RawFloats()
|
||||
for i := range wv {
|
||||
if math.Float64bits(wv[i]) != math.Float64bits(gv[i]) {
|
||||
t.Fatalf("%s: value %d is %v (%#x), want %v (%#x)", name, i, gv[i], math.Float64bits(gv[i]), wv[i], math.Float64bits(wv[i]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadCSVParity pins the tokenizer to the encoding/csv reader it
|
||||
// replaced, one input at a time.
|
||||
func TestLoadCSVParity(t *testing.T) {
|
||||
for _, tc := range csvParityCases {
|
||||
checkCSVParity(t, tc.name, tc.in, tc.header)
|
||||
checkCSVParity(t, tc.name+" with header flag", tc.in, !tc.header)
|
||||
}
|
||||
for _, tc := range csvParityExtra(t) {
|
||||
checkCSVParity(t, tc.name, tc.in, tc.header)
|
||||
checkCSVParity(t, tc.name+" with header flag", tc.in, !tc.header)
|
||||
}
|
||||
// A generated table, the shape the reader is built for.
|
||||
var sb strings.Builder
|
||||
a, _ := core.FromFloats([]float64{1.5, -2.25, 3.125, 4, 5.5, -6.75, 1e-9, -0, 0.1, 2, 3, 4}, 4, 3)
|
||||
if err := SaveCSVWriter(&sb, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkCSVParity(t, "saved table", sb.String(), false)
|
||||
checkCSVParity(t, "saved table with header", "c0,c1,c2\n"+sb.String(), true)
|
||||
}
|
||||
|
||||
// FuzzLoadCSVParity hunts for the input where the tokenizer and the
|
||||
// encoding/csv reader part ways.
|
||||
func FuzzLoadCSVParity(f *testing.F) {
|
||||
for _, tc := range csvParityCases {
|
||||
f.Add(tc.in, tc.header)
|
||||
}
|
||||
f.Add(strings.Repeat("1.5,", 300)+"1.5\n", false)
|
||||
f.Fuzz(func(t *testing.T, in string, skipHeader bool) {
|
||||
checkCSVParity(t, "fuzz", in, skipHeader)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user