// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) // SPDX-License-Identifier: MIT package io import ( "encoding/binary" "fmt" "math" "os" "path/filepath" "runtime" "strings" "testing" "sourcedock.dev/petrbalvin/tensor/internal/core" ) // Regression tests for hostile inputs across the io package: the size // arithmetic of the HDF5 reader, the NetCDF record-size // pre-pass, the FITS table encoders and decoders, the mmap count // conversion, and the CSV shape contract. Every hostile file below is // built by hand, byte by byte, because the point of each test is a // declared size the writer of a well-formed file would never produce. // h5HostileFile returns an n-byte HDF5 file with the signature and a // version 0 superblock (the classic layout: eight-byte addresses and // lengths) whose root object header sits at offset 96. func h5HostileFile(n int) []byte { f := make([]byte, n) copy(f, hdf5Magic) f[8] = 0 // superblock version 0 f[13] = 8 f[14] = 8 binary.LittleEndian.PutUint64(f[32:], math.MaxUint64) // free space undefined binary.LittleEndian.PutUint64(f[40:], uint64(n)) // end of file binary.LittleEndian.PutUint64(f[48:], math.MaxUint64) // driver undefined binary.LittleEndian.PutUint64(f[64:], 96) // root object header return f } // h5Msg is one object header message: its type and its body bytes. type h5Msg struct { typ uint16 body []byte } // h5ObjectHeader writes a version 1 object header at off carrying msgs // and returns the offset just past its message region. func h5ObjectHeader(f []byte, off int, msgs ...h5Msg) int { f[off] = 1 binary.LittleEndian.PutUint16(f[off+2:], uint16(len(msgs))) binary.LittleEndian.PutUint32(f[off+4:], 1) // reference count region := off + 16 size := 0 for _, m := range msgs { size = alignUp(size+8+len(m.body), 8) } binary.LittleEndian.PutUint32(f[off+8:], uint32(size)) for _, m := range msgs { binary.LittleEndian.PutUint16(f[region:], m.typ) binary.LittleEndian.PutUint16(f[region+2:], uint16(len(m.body))) copy(f[region+8:], m.body) region += alignUp(8+len(m.body), 8) } return region } // h5Dataspace renders a version 1 dataspace message with the given // extents, each written as an eight-byte length. func h5Dataspace(dims ...uint64) []byte { m := make([]byte, 8+8*len(dims)) m[0] = 1 // version 1 if len(dims) == 0 { return m } m[1] = byte(len(dims)) for i, d := range dims { binary.LittleEndian.PutUint64(m[8+8*i:], d) } return m } // h5FloatType renders a version 1 floating-point datatype message of the // given element width. func h5FloatType(size uint32) []byte { m := make([]byte, 20) m[0] = 0x11 // version 1, class 1 (floating-point) binary.LittleEndian.PutUint32(m[4:], size) return m } // h5ChunkLayoutV4 renders a version 4 chunked layout message whose chunk // dimensions are eight bytes wide, the flag the version 4 message // carries, so an extent beyond 2^32 can be declared at all. func h5ChunkLayoutV4(btree uint64, dims ...uint64) []byte { m := make([]byte, 3+8+1+8*len(dims)) m[0] = 4 m[1] = 2 // chunked m[2] = byte(len(dims)) binary.LittleEndian.PutUint64(m[3:], btree) m[11] = 1 // eight-byte chunk dimensions for i, d := range dims { binary.LittleEndian.PutUint64(m[12+8*i:], d) } return m } // h5ChunkLayoutV3 renders a version 3 chunked layout message with // four-byte chunk dimensions. func h5ChunkLayoutV3(btree uint64, dims ...uint32) []byte { m := make([]byte, 3+8+4*len(dims)) m[0] = 3 m[1] = 2 m[2] = byte(len(dims)) binary.LittleEndian.PutUint64(m[3:], btree) for i, d := range dims { binary.LittleEndian.PutUint32(m[11+4*i:], d) } return m } // h5ContiguousLayout renders a version 3 contiguous layout message. func h5ContiguousLayout(addr, size uint64) []byte { m := make([]byte, 2+8+8) m[0] = 3 m[1] = 1 // contiguous binary.LittleEndian.PutUint64(m[2:], addr) binary.LittleEndian.PutUint64(m[10:], size) return m } // h5ChunkTree writes a one-entry leaf chunk B-tree at off for a dataset // of the given rank: one chunk of size stored bytes at chunkAt, with no // filter mask, at the given chunk offsets. The key carries one more // offset than the dataset has axes, the element size, which is why the // entry is 8+(8*(rank+1))+offSize bytes long. func h5ChunkTree(f []byte, off, rank int, size uint32, offsets []uint64, chunkAt uint64) { copy(f[off:], hdf5Tree) f[off+4] = 1 // chunk tree f[off+5] = 0 // leaf level binary.LittleEndian.PutUint16(f[off+6:], 1) p := off + 24 binary.LittleEndian.PutUint32(f[p:], size) binary.LittleEndian.PutUint32(f[p+4:], 0) // filter mask for i := range rank { binary.LittleEndian.PutUint64(f[p+8+8*i:], offsets[i]) } binary.LittleEndian.PutUint64(f[p+8+8*rank:], 8) // the element-size key binary.LittleEndian.PutUint64(f[p+8+8*(rank+1):], chunkAt) } // TestLoadHDF5ChunkExtentWrap pins the chunk-dimension bound. The chunk // shape 2^32 by 2^29 float64 elements is 2^64 bytes, which wraps to // zero: the "chunk above the reader budget" guard and the "chunk holds a // full chunk" guard both saw 0, so the chunk strides were walked out of // the stored chunk (a slice out of range on a 2^61-element declaration, // and a silent read of the file's other bytes on a milder one). func TestLoadHDF5ChunkExtentWrap(t *testing.T) { cases := []struct { name string dims []uint64 // the chunk's shape plus the element-size slot }{ {"2^32 by 2^29 elements", []uint64{1 << 32, 1 << 29, 8}}, {"2^61 elements", []uint64{1 << 61, 1, 8}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { const btree, chunkAt, n = 320, 448, 512 f := h5HostileFile(n) end := h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(2, 1)}, h5Msg{hdf5MsgDatatype, h5FloatType(8)}, h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV4(btree, tc.dims...)}, ) if end > btree { t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree) } h5ChunkTree(f, btree, 2, 16, []uint64{0, 0}, chunkAt) path := writeHostile(t, "chunkextent.h5", f) sets, err := LoadHDF5(path) if err == nil { t.Fatalf("LoadHDF5 accepted a chunk whose extent wraps: %d datasets", len(sets)) } if !strings.Contains(err.Error(), "budget") { t.Fatalf("error = %v, want the reader-budget refusal", err) } }) } } // TestLoadHDF5DatasetExtentWrap pins the dataspace bound on both storage // classes: a shape of 2^32 by 2^32 float64 elements wraps the byte count // to zero, which passed the cap, so the output buffer was allocated // empty while the copy needed a whole element (the chunked class // panicked), and the contiguous class returned a dataset claiming 2^64 // elements with an empty payload and no error. func TestLoadHDF5DatasetExtentWrap(t *testing.T) { t.Run("chunked", func(t *testing.T) { const btree, chunkAt, n = 320, 448, 512 f := h5HostileFile(n) end := h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(1<<32, 1<<32)}, h5Msg{hdf5MsgDatatype, h5FloatType(8)}, h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV3(btree, 1, 1, 8)}, ) if end > btree { t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree) } h5ChunkTree(f, btree, 2, 8, []uint64{0, 0}, chunkAt) sets, err := LoadHDF5(writeHostile(t, "shapedwrap.h5", f)) if err == nil { t.Fatalf("LoadHDF5 accepted a wrapping dataspace: %d datasets", len(sets)) } if !strings.Contains(err.Error(), "budget") { t.Fatalf("error = %v, want the reader-budget refusal", err) } }) t.Run("contiguous", func(t *testing.T) { const dataAt, n = 448, 512 f := h5HostileFile(n) h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(1<<32, 1<<32)}, h5Msg{hdf5MsgDatatype, h5FloatType(8)}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(dataAt, 8)}, ) sets, err := LoadHDF5(writeHostile(t, "contigwrap.h5", f)) if err == nil { shape := []int(nil) if len(sets) > 0 { shape = sets[0].Shape } t.Fatalf("LoadHDF5 returned %v for a wrapping dataspace, want an error", shape) } if !strings.Contains(err.Error(), "budget") { t.Fatalf("error = %v, want the reader-budget refusal", err) } }) } // TestLoadHDF5AttributeExtentWrap pins the attribute dataspace bound: a // 3 by 2^62 element dataspace wraps its element count to a negative // number in the multiply-first check, which let it past the bounds test // and into an allocation with a negative capacity. The attribute must be // refused (dropped), not accepted with a wrapped value. func TestLoadHDF5AttributeExtentWrap(t *testing.T) { const n = 512 f := h5HostileFile(n) // The attribute message: version, flags, then the three size fields // with their fields behind them. A 2^62 extent is far past the // message, so no legal value can follow it. attr := make([]byte, 88) attr[0] = 1 binary.LittleEndian.PutUint16(attr[2:], 2) // name size binary.LittleEndian.PutUint16(attr[4:], 20) // datatype size binary.LittleEndian.PutUint16(attr[6:], 24) // dataspace size copy(attr[8:], "n\x00") copy(attr[16:], h5FloatType(8)) copy(attr[40:], h5Dataspace(3, 1<<62)) // A hard link to a one-element dataset, so the attribute's fate is // observable: an accepted attribute lands in the dataset's map. link := func(addr uint64) []byte { return binary.LittleEndian.AppendUint64([]byte{1, 0, 1, 'd'}, addr) } datasetAt := 288 end := h5ObjectHeader(f, 96, h5Msg{hdf5MsgLink, link(uint64(datasetAt))}, h5Msg{hdf5MsgAttribute, attr}, ) if end > datasetAt { t.Fatalf("the test object header runs to %d, past the dataset at %d", end, datasetAt) } h5ObjectHeader(f, datasetAt, h5Msg{hdf5MsgDataspace, h5Dataspace(1)}, h5Msg{hdf5MsgDatatype, h5FloatType(8)}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(448, 8)}, ) sets, err := LoadHDF5(writeHostile(t, "attrextent.h5", f)) if err != nil { t.Fatalf("LoadHDF5 refused the file: %v", err) } if len(sets) != 1 || sets[0].Path != "/d" { t.Fatalf("datasets = %d, want the linked /d", len(sets)) } if v, ok := sets[0].Attrs["n"]; ok { t.Fatalf("the attribute with a wrapping dataspace was accepted as %q", v) } } // TestLoadHDF5MessageCountBounded pins the object header preallocation: // the slice used to be sized from the header's declared message count // (65535 messages of 32 bytes is 2 MiB) before the region that would // have to hold them was consulted, so 16 bytes of file ordered a // two-megabyte allocation. func TestLoadHDF5MessageCountBounded(t *testing.T) { f := h5HostileFile(160) f[96] = 1 binary.LittleEndian.PutUint16(f[98:], 65535) // declared messages binary.LittleEndian.PutUint32(f[100:], 1) // reference count binary.LittleEndian.PutUint32(f[104:], 16) // the region that holds them path := writeHostile(t, "msgcount.h5", f) runtime.GC() var before, after runtime.MemStats runtime.ReadMemStats(&before) _, err := LoadHDF5(path) runtime.ReadMemStats(&after) if err == nil { t.Fatal("LoadHDF5 accepted a header whose messages run past the region") } if used := after.TotalAlloc - before.TotalAlloc; used > 512<<10 { t.Fatalf("loading a 160-byte file allocated %d bytes: the declared message count still sizes the preallocation", used) } } // TestLoadNetCDFRecordSizeWrap pins the record-size pre-pass. It used to // multiply every record variable's non-record dimensions before any // variable was validated, so two hostile declarations inflated the // record size for an innocuous variable to 2^54 bytes: one element per // record then made (numrecs-1)*recordSize exactly 2^64, which wrapped to // zero, the span check passed and the reader walked 2^54 bytes into a // 4 KiB file. func TestLoadNetCDFRecordSizeWrap(t *testing.T) { const numrecs = 1<<10 + 1 // The arithmetic the reader performed: two slabs of 2^25 by 2^25 // doubles are 2^53 bytes each and the innocuous variable's slab is // four, so the record size is 2^54+4 and the last record's slab // offset wraps to 2^64+4096, i.e. 4096. A file larger than that // passes the span check and the following records then index far // past the file. recordSize := 2*int64(1<<53) + 4 if span := (numrecs-1)*recordSize + 1; span != 1<<12+1 { t.Fatalf("construction is wrong: the wrapped span is %d, want %d", span, 1<<12+1) } var b []byte b = append(b, 'C', 'D', 'F', 1) b = binary.BigEndian.AppendUint32(b, numrecs) // dim_list: the record dimension leads, then one one-element // dimension and the two hostile ones. b = binary.BigEndian.AppendUint32(b, ncTagDimension) b = binary.BigEndian.AppendUint32(b, 4) b = hostileNCName(b, "rec") b = binary.BigEndian.AppendUint32(b, 0) b = hostileNCName(b, "one") b = binary.BigEndian.AppendUint32(b, 1) b = hostileNCName(b, "big1") b = binary.BigEndian.AppendUint32(b, 1<<25) b = hostileNCName(b, "big2") b = binary.BigEndian.AppendUint32(b, 1<<25) // gatt_list: absent. b = binary.BigEndian.AppendUint32(b, 0) b = binary.BigEndian.AppendUint32(b, 0) // var_list: the innocuous record variable, then the two that inflate // the record size. b = binary.BigEndian.AppendUint32(b, ncTagVariable) b = binary.BigEndian.AppendUint32(b, 3) b = hostileNCName(b, "a") b = binary.BigEndian.AppendUint32(b, 2) // rank: rec, one b = binary.BigEndian.AppendUint32(b, 0) b = binary.BigEndian.AppendUint32(b, 1) b = binary.BigEndian.AppendUint32(b, 0) // variable attributes: absent b = binary.BigEndian.AppendUint32(b, 0) b = binary.BigEndian.AppendUint32(b, ncTypeByte) b = binary.BigEndian.AppendUint32(b, 4) // vsize b = binary.BigEndian.AppendUint32(b, 0) // begin for i := range 2 { b = hostileNCName(b, fmt.Sprintf("b%d", i)) b = binary.BigEndian.AppendUint32(b, 3) // rank: rec, big1, big2 b = binary.BigEndian.AppendUint32(b, 0) b = binary.BigEndian.AppendUint32(b, 2) b = binary.BigEndian.AppendUint32(b, 3) b = binary.BigEndian.AppendUint32(b, 0) // variable attributes: absent b = binary.BigEndian.AppendUint32(b, 0) b = binary.BigEndian.AppendUint32(b, ncTypeDouble) b = binary.BigEndian.AppendUint32(b, 0) // vsize b = binary.BigEndian.AppendUint32(b, 0) // begin } // A file long enough for the innocuous variable's own records and for // the wrapped span the buggy check computes. for len(b) < 8192 { b = append(b, 0) } path := writeHostile(t, "recsize-wrap.nc", b) _, _, _, err := LoadNetCDF(path) if err == nil { t.Fatal("LoadNetCDF accepted a record variable whose dimensions cannot fit the file") } if !strings.Contains(err.Error(), "more elements than the file holds") { t.Fatalf("error = %v, want the per-variable element bound", err) } } // TestSaveFITSTableASCIIFloat32 pins the ASCII float encoder for a // float32 column. The encoder read the float64 payload first and only // then substituted the float32 one, but RawFloats is nil for every // float32 array, so the first row of an ordinary float32 column indexed // a nil slice. func TestSaveFITSTableASCIIFloat32(t *testing.T) { vals := []float32{1.5, -2.25, 3e8} f32 := core.New(core.Float32, len(vals)) copy(f32.RawFloat32s(), vals) cols := []FITSTableColumn{{Name: "F", Form: "E12.4", Data: f32}} path := filepath.Join(t.TempDir(), "f32ascii.fits") if err := SaveFITSTable(path, true, cols, nil); err != nil { t.Fatalf("SaveFITSTable: %v", err) } table, err := LoadFITSTable(path) if err != nil { t.Fatalf("LoadFITSTable: %v", err) } for i, want := range vals { if got := table.Columns[0].FloatAt(i); got != float64(want) { t.Fatalf("F[%d] = %g, want %g", i, got, float64(want)) } } } // TestLoadFITSTableFortranNumbers pins the number forms a Fortran writer // puts in an ASCII table: strconv.ParseFloat accepts neither the Dw.d // exponent ("1.5D+03") nor the exponent-less form ("1.5+03"), and one // unparseable cell refused the whole table. func TestLoadFITSTableFortranNumbers(t *testing.T) { hdr := cardBlock( card("XTENSION= 'TABLE '"), card("BITPIX = 8"), card("NAXIS = 2"), card("NAXIS1 = 20"), card("NAXIS2 = 4"), card("TFIELDS = 1"), card("TBCOL1 = 1"), card("TFORM1 = 'D20.12'"), card("END"), ) rows := []string{"1.5D+03", "1.5+03", "-2.5d-2", "1.5E+03"} payload := make([]byte, 0, 20*len(rows)) for _, r := range rows { payload = append(payload, fmt.Sprintf("%-20s", r)...) } path := writeHostile(t, "fortran.fits", append(hdr, payload...)) table, err := LoadFITSTable(path) if err != nil { t.Fatalf("LoadFITSTable refused a Fortran table: %v", err) } want := []float64{1500, 1500, -0.025, 1500} for i, w := range want { if got := table.Columns[0].FloatAt(i); got != w { t.Fatalf("row %d (%q) = %g, want %g", i, rows[i], got, w) } } } // TestLoadFITSTableScaleOverflow pins the integral-scaling bound. The // guard used 9.3e18, which is above MaxInt64 (9.223372036854775807e18), // so a scaled value in [2^63, 9.3e18) reached an out-of-range // float-to-integer conversion: the column kept the int dtype and the // value came back as MinInt64 instead of the documented promotion to // float64. func TestLoadFITSTableScaleOverflow(t *testing.T) { const raw = int64(1) << 62 cols := []FITSTableColumn{{Name: "V", Form: "K", Data: mustInts(t, []int64{raw}, 1)}} path := filepath.Join(t.TempDir(), "scaled.fits") if err := SaveFITSTable(path, false, cols, map[string]string{"TSCAL1": "2", "TZERO1": "0"}); err != nil { t.Fatalf("SaveFITSTable: %v", err) } table, err := LoadFITSTable(path) if err != nil { t.Fatalf("LoadFITSTable: %v", err) } col := table.Columns[0] want := float64(raw) * 2 // 2^63, exactly representable in float64 if col.Dtype() != core.Float { t.Fatalf("dtype = %s, want float64: a scaling past MaxInt64 must promote the column", col.Dtype()) } if got := col.FloatAt(0); got != want { t.Fatalf("scaled value = %g, want %g", got, want) } } // TestMapCountsWithoutWrap pins the mmap count arithmetic: the byte // length used to be int64(n)*elementSize, which wraps for a hostile // count, so the wrapped length passed the file-size checks and the typed // view was then built with the original count, which unsafe.Slice // rejects with a panic instead of an error. func TestMapCountsWithoutWrap(t *testing.T) { path := filepath.Join(t.TempDir(), "small.bin") if err := SaveNativeFloats(path, []float64{1, 2, 3, 4}); err != nil { t.Fatal(err) } // 2^62+3 float64 values are 2^65+24 bytes, which wraps to 24: the // file holds 32. The others already failed the non-positive-length // check by luck and must keep failing it. for _, n := range []int{1<<62 + 3, 1 << 61, 1 << 60, math.MaxInt} { if _, _, err := MapFloats(path, 0, n); err == nil { t.Errorf("MapFloats(n = %d) accepted a count the file cannot hold", n) } if _, _, err := MapFloat32s(path, 0, n); err == nil { t.Errorf("MapFloat32s(n = %d) accepted a count the file cannot hold", n) } if _, _, err := MapInts(path, 0, n); err == nil { t.Errorf("MapInts(n = %d) accepted a count the file cannot hold", n) } } } // TestSaveCSVZeroColumns pins the shape contract: an array with rows and // no columns wrote one empty record per row, and encoding/csv reads // blank lines as no records at all, so (3,0) came back as (0,0). It is // refused instead, because no CSV text can carry the column count. func TestSaveCSVZeroColumns(t *testing.T) { a, err := core.FromFloats(nil, 3, 0) if err != nil { t.Fatalf("FromFloats(nil, 3, 0): %v", err) } var sb strings.Builder if err := SaveCSVWriter(&sb, a); err == nil { t.Fatalf("SaveCSVWriter accepted the shape %v and wrote %q, which reads back as a different shape", a.Shape(), sb.String()) } // A 2-D array with columns still round-trips. ok := mustFloats(t, []float64{1, 2, 3, 4, 5, 6}, 2, 3) var back strings.Builder if err := SaveCSVWriter(&back, ok); err != nil { t.Fatalf("SaveCSVWriter: %v", err) } got, err := LoadCSVReader(strings.NewReader(back.String()), false) if err != nil { t.Fatalf("LoadCSVReader: %v", err) } if got.Shape()[0] != 2 || got.Shape()[1] != 3 { t.Fatalf("round trip gave %v, want [2 3]", got.Shape()) } } // TestIOSourceTypography pins the plain-ASCII rule for the package's own // source: an em dash, an en dash, a Unicode minus, a middle dot or a // multiplication sign in a comment or a message is a typographic symbol // where the repository's style asks for ASCII. func TestIOSourceTypography(t *testing.T) { banned := []struct { r rune name string }{ {'\u2014', "em dash"}, {'\u2013', "en dash"}, {'\u2212', "Unicode minus"}, {'\u00b7', "middle dot"}, {'\u00d7', "multiplication sign"}, } entries, err := os.ReadDir(".") if err != nil { t.Fatalf("ReadDir: %v", err) } checked := 0 for _, e := range entries { name := e.Name() if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { continue } data, err := os.ReadFile(name) if err != nil { t.Fatalf("ReadFile(%s): %v", name, err) } checked++ for _, b := range banned { if strings.ContainsRune(string(data), b.r) { t.Errorf("%s carries a %s (U+%04X)", name, b.name, b.r) } } } if checked == 0 { t.Fatal("no source files were checked") } }