petrbalvin/tensor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity
Files
7ce7041f2f9b5821032f58e294a1865190c73055
tensor/io/extent_wrap_pins_test.go
T

574 lines
21 KiB
Go
Raw Normal View History

feat: initial release
2026-09-03 10:00:00 +02:00
// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: MIT
package io
import (
"encoding/binary"
"fmt"
"math"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"sourcedock.dev/petrbalvin/tensor/internal/core"
)
// Regression tests for hostile inputs across the io package: the size
// arithmetic of the HDF5 reader, the NetCDF record-size
// pre-pass, the FITS table encoders and decoders, the mmap count
// conversion, and the CSV shape contract. Every hostile file below is
// built by hand, byte by byte, because the point of each test is a
// declared size the writer of a well-formed file would never produce.
// h5HostileFile returns an n-byte HDF5 file with the signature and a
// version 0 superblock (the classic layout: eight-byte addresses and
// lengths) whose root object header sits at offset 96.
func h5HostileFile(n int) []byte {
f := make([]byte, n)
copy(f, hdf5Magic)
f[8] = 0 // superblock version 0
f[13] = 8
f[14] = 8
binary.LittleEndian.PutUint64(f[32:], math.MaxUint64) // free space undefined
binary.LittleEndian.PutUint64(f[40:], uint64(n)) // end of file
binary.LittleEndian.PutUint64(f[48:], math.MaxUint64) // driver undefined
binary.LittleEndian.PutUint64(f[64:], 96) // root object header
return f
}
// h5Msg is one object header message: its type and its body bytes.
type h5Msg struct {
typ uint16
body []byte
}
// h5ObjectHeader writes a version 1 object header at off carrying msgs
// and returns the offset just past its message region.
func h5ObjectHeader(f []byte, off int, msgs ...h5Msg) int {
f[off] = 1
binary.LittleEndian.PutUint16(f[off+2:], uint16(len(msgs)))
binary.LittleEndian.PutUint32(f[off+4:], 1) // reference count
region := off + 16
size := 0
for _, m := range msgs {
size = alignUp(size+8+len(m.body), 8)
}
binary.LittleEndian.PutUint32(f[off+8:], uint32(size))
for _, m := range msgs {
binary.LittleEndian.PutUint16(f[region:], m.typ)
binary.LittleEndian.PutUint16(f[region+2:], uint16(len(m.body)))
copy(f[region+8:], m.body)
region += alignUp(8+len(m.body), 8)
}
return region
}
// h5Dataspace renders a version 1 dataspace message with the given
// extents, each written as an eight-byte length.
func h5Dataspace(dims ...uint64) []byte {
m := make([]byte, 8+8*len(dims))
m[0] = 1 // version 1
if len(dims) == 0 {
return m
}
m[1] = byte(len(dims))
for i, d := range dims {
binary.LittleEndian.PutUint64(m[8+8*i:], d)
}
return m
}
// h5FloatType renders a version 1 floating-point datatype message of the
// given element width.
func h5FloatType(size uint32) []byte {
m := make([]byte, 20)
m[0] = 0x11 // version 1, class 1 (floating-point)
binary.LittleEndian.PutUint32(m[4:], size)
return m
}
// h5ChunkLayoutV4 renders a version 4 chunked layout message whose chunk
// dimensions are eight bytes wide, the flag the version 4 message
// carries, so an extent beyond 2^32 can be declared at all.
func h5ChunkLayoutV4(btree uint64, dims ...uint64) []byte {
m := make([]byte, 3+8+1+8*len(dims))
m[0] = 4
m[1] = 2 // chunked
m[2] = byte(len(dims))
binary.LittleEndian.PutUint64(m[3:], btree)
m[11] = 1 // eight-byte chunk dimensions
for i, d := range dims {
binary.LittleEndian.PutUint64(m[12+8*i:], d)
}
return m
}
// h5ChunkLayoutV3 renders a version 3 chunked layout message with
// four-byte chunk dimensions.
func h5ChunkLayoutV3(btree uint64, dims ...uint32) []byte {
m := make([]byte, 3+8+4*len(dims))
m[0] = 3
m[1] = 2
m[2] = byte(len(dims))
binary.LittleEndian.PutUint64(m[3:], btree)
for i, d := range dims {
binary.LittleEndian.PutUint32(m[11+4*i:], d)
}
return m
}
// h5ContiguousLayout renders a version 3 contiguous layout message.
func h5ContiguousLayout(addr, size uint64) []byte {
m := make([]byte, 2+8+8)
m[0] = 3
m[1] = 1 // contiguous
binary.LittleEndian.PutUint64(m[2:], addr)
binary.LittleEndian.PutUint64(m[10:], size)
return m
}
// h5ChunkTree writes a one-entry leaf chunk B-tree at off for a dataset
// of the given rank: one chunk of size stored bytes at chunkAt, with no
// filter mask, at the given chunk offsets. The key carries one more
// offset than the dataset has axes, the element size, which is why the
// entry is 8+(8*(rank+1))+offSize bytes long.
func h5ChunkTree(f []byte, off, rank int, size uint32, offsets []uint64, chunkAt uint64) {
copy(f[off:], hdf5Tree)
f[off+4] = 1 // chunk tree
f[off+5] = 0 // leaf level
binary.LittleEndian.PutUint16(f[off+6:], 1)
p := off + 24
binary.LittleEndian.PutUint32(f[p:], size)
binary.LittleEndian.PutUint32(f[p+4:], 0) // filter mask
for i := range rank {
binary.LittleEndian.PutUint64(f[p+8+8*i:], offsets[i])
}
binary.LittleEndian.PutUint64(f[p+8+8*rank:], 8) // the element-size key
binary.LittleEndian.PutUint64(f[p+8+8*(rank+1):], chunkAt)
}
// TestLoadHDF5ChunkExtentWrap pins the chunk-dimension bound. The chunk
// shape 2^32 by 2^29 float64 elements is 2^64 bytes, which wraps to
// zero: the "chunk above the reader budget" guard and the "chunk holds a
// full chunk" guard both saw 0, so the chunk strides were walked out of
// the stored chunk (a slice out of range on a 2^61-element declaration,
// and a silent read of the file's other bytes on a milder one).
func TestLoadHDF5ChunkExtentWrap(t *testing.T) {
cases := []struct {
name string
dims []uint64 // the chunk's shape plus the element-size slot
}{
{"2^32 by 2^29 elements", []uint64{1 << 32, 1 << 29, 8}},
{"2^61 elements", []uint64{1 << 61, 1, 8}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
const btree, chunkAt, n = 320, 448, 512
f := h5HostileFile(n)
end := h5ObjectHeader(f, 96,
h5Msg{hdf5MsgDataspace, h5Dataspace(2, 1)},
h5Msg{hdf5MsgDatatype, h5FloatType(8)},
h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV4(btree, tc.dims...)},
)
if end > btree {
t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree)
}
h5ChunkTree(f, btree, 2, 16, []uint64{0, 0}, chunkAt)
path := writeHostile(t, "chunkextent.h5", f)
sets, err := LoadHDF5(path)
if err == nil {
t.Fatalf("LoadHDF5 accepted a chunk whose extent wraps: %d datasets", len(sets))
}
if !strings.Contains(err.Error(), "budget") {
t.Fatalf("error = %v, want the reader-budget refusal", err)
}
})
}
}
// TestLoadHDF5DatasetExtentWrap pins the dataspace bound on both storage
// classes: a shape of 2^32 by 2^32 float64 elements wraps the byte count
// to zero, which passed the cap, so the output buffer was allocated
// empty while the copy needed a whole element (the chunked class
// panicked), and the contiguous class returned a dataset claiming 2^64
// elements with an empty payload and no error.
func TestLoadHDF5DatasetExtentWrap(t *testing.T) {
t.Run("chunked", func(t *testing.T) {
const btree, chunkAt, n = 320, 448, 512
f := h5HostileFile(n)
end := h5ObjectHeader(f, 96,
h5Msg{hdf5MsgDataspace, h5Dataspace(1<<32, 1<<32)},
h5Msg{hdf5MsgDatatype, h5FloatType(8)},
h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV3(btree, 1, 1, 8)},
)
if end > btree {
t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree)
}
h5ChunkTree(f, btree, 2, 8, []uint64{0, 0}, chunkAt)
sets, err := LoadHDF5(writeHostile(t, "shapedwrap.h5", f))
if err == nil {
t.Fatalf("LoadHDF5 accepted a wrapping dataspace: %d datasets", len(sets))
}
if !strings.Contains(err.Error(), "budget") {
t.Fatalf("error = %v, want the reader-budget refusal", err)
}
})
t.Run("contiguous", func(t *testing.T) {
const dataAt, n = 448, 512
f := h5HostileFile(n)
h5ObjectHeader(f, 96,
h5Msg{hdf5MsgDataspace, h5Dataspace(1<<32, 1<<32)},
h5Msg{hdf5MsgDatatype, h5FloatType(8)},
h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(dataAt, 8)},
)
sets, err := LoadHDF5(writeHostile(t, "contigwrap.h5", f))
if err == nil {
shape := []int(nil)
if len(sets) > 0 {
shape = sets[0].Shape
}
t.Fatalf("LoadHDF5 returned %v for a wrapping dataspace, want an error", shape)
}
if !strings.Contains(err.Error(), "budget") {
t.Fatalf("error = %v, want the reader-budget refusal", err)
}
})
}
// TestLoadHDF5AttributeExtentWrap pins the attribute dataspace bound: a
// 3 by 2^62 element dataspace wraps its element count to a negative
// number in the multiply-first check, which let it past the bounds test
// and into an allocation with a negative capacity. The attribute must be
// refused (dropped), not accepted with a wrapped value.
func TestLoadHDF5AttributeExtentWrap(t *testing.T) {
const n = 512
f := h5HostileFile(n)
// The attribute message: version, flags, then the three size fields
// with their fields behind them. A 2^62 extent is far past the
// message, so no legal value can follow it.
attr := make([]byte, 88)
attr[0] = 1
binary.LittleEndian.PutUint16(attr[2:], 2) // name size
binary.LittleEndian.PutUint16(attr[4:], 20) // datatype size
binary.LittleEndian.PutUint16(attr[6:], 24) // dataspace size
copy(attr[8:], "n\x00")
copy(attr[16:], h5FloatType(8))
copy(attr[40:], h5Dataspace(3, 1<<62))
// A hard link to a one-element dataset, so the attribute's fate is
// observable: an accepted attribute lands in the dataset's map.
link := func(addr uint64) []byte {
return binary.LittleEndian.AppendUint64([]byte{1, 0, 1, 'd'}, addr)
}
datasetAt := 288
end := h5ObjectHeader(f, 96,
h5Msg{hdf5MsgLink, link(uint64(datasetAt))},
h5Msg{hdf5MsgAttribute, attr},
)
if end > datasetAt {
t.Fatalf("the test object header runs to %d, past the dataset at %d", end, datasetAt)
}
h5ObjectHeader(f, datasetAt,
h5Msg{hdf5MsgDataspace, h5Dataspace(1)},
h5Msg{hdf5MsgDatatype, h5FloatType(8)},
h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(448, 8)},
)
sets, err := LoadHDF5(writeHostile(t, "attrextent.h5", f))
if err != nil {
t.Fatalf("LoadHDF5 refused the file: %v", err)
}
if len(sets) != 1 || sets[0].Path != "/d" {
t.Fatalf("datasets = %d, want the linked /d", len(sets))
}
if v, ok := sets[0].Attrs["n"]; ok {
t.Fatalf("the attribute with a wrapping dataspace was accepted as %q", v)
}
}
// TestLoadHDF5MessageCountBounded pins the object header preallocation:
// the slice used to be sized from the header's declared message count
// (65535 messages of 32 bytes is 2 MiB) before the region that would
// have to hold them was consulted, so 16 bytes of file ordered a
// two-megabyte allocation.
func TestLoadHDF5MessageCountBounded(t *testing.T) {
f := h5HostileFile(160)
f[96] = 1
binary.LittleEndian.PutUint16(f[98:], 65535) // declared messages
binary.LittleEndian.PutUint32(f[100:], 1) // reference count
binary.LittleEndian.PutUint32(f[104:], 16) // the region that holds them
path := writeHostile(t, "msgcount.h5", f)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
_, err := LoadHDF5(path)
runtime.ReadMemStats(&after)
if err == nil {
t.Fatal("LoadHDF5 accepted a header whose messages run past the region")
}
if used := after.TotalAlloc - before.TotalAlloc; used > 512<<10 {
t.Fatalf("loading a 160-byte file allocated %d bytes: the declared message count still sizes the preallocation", used)
}
}
// TestLoadNetCDFRecordSizeWrap pins the record-size pre-pass. It used to
// multiply every record variable's non-record dimensions before any
// variable was validated, so two hostile declarations inflated the
// record size for an innocuous variable to 2^54 bytes: one element per
// record then made (numrecs-1)*recordSize exactly 2^64, which wrapped to
// zero, the span check passed and the reader walked 2^54 bytes into a
// 4 KiB file.
func TestLoadNetCDFRecordSizeWrap(t *testing.T) {
const numrecs = 1<<10 + 1
// The arithmetic the reader performed: two slabs of 2^25 by 2^25
// doubles are 2^53 bytes each and the innocuous variable's slab is
// four, so the record size is 2^54+4 and the last record's slab
// offset wraps to 2^64+4096, i.e. 4096. A file larger than that
// passes the span check and the following records then index far
// past the file.
recordSize := 2*int64(1<<53) + 4
if span := (numrecs-1)*recordSize + 1; span != 1<<12+1 {
t.Fatalf("construction is wrong: the wrapped span is %d, want %d", span, 1<<12+1)
}
var b []byte
b = append(b, 'C', 'D', 'F', 1)
b = binary.BigEndian.AppendUint32(b, numrecs)
// dim_list: the record dimension leads, then one one-element
// dimension and the two hostile ones.
b = binary.BigEndian.AppendUint32(b, ncTagDimension)
b = binary.BigEndian.AppendUint32(b, 4)
b = hostileNCName(b, "rec")
b = binary.BigEndian.AppendUint32(b, 0)
b = hostileNCName(b, "one")
b = binary.BigEndian.AppendUint32(b, 1)
b = hostileNCName(b, "big1")
b = binary.BigEndian.AppendUint32(b, 1<<25)
b = hostileNCName(b, "big2")
b = binary.BigEndian.AppendUint32(b, 1<<25)
// gatt_list: absent.
b = binary.BigEndian.AppendUint32(b, 0)
b = binary.BigEndian.AppendUint32(b, 0)
// var_list: the innocuous record variable, then the two that inflate
// the record size.
b = binary.BigEndian.AppendUint32(b, ncTagVariable)
b = binary.BigEndian.AppendUint32(b, 3)
b = hostileNCName(b, "a")
b = binary.BigEndian.AppendUint32(b, 2) // rank: rec, one
b = binary.BigEndian.AppendUint32(b, 0)
b = binary.BigEndian.AppendUint32(b, 1)
b = binary.BigEndian.AppendUint32(b, 0) // variable attributes: absent
b = binary.BigEndian.AppendUint32(b, 0)
b = binary.BigEndian.AppendUint32(b, ncTypeByte)
b = binary.BigEndian.AppendUint32(b, 4) // vsize
b = binary.BigEndian.AppendUint32(b, 0) // begin
for i := range 2 {
b = hostileNCName(b, fmt.Sprintf("b%d", i))
b = binary.BigEndian.AppendUint32(b, 3) // rank: rec, big1, big2
b = binary.BigEndian.AppendUint32(b, 0)
b = binary.BigEndian.AppendUint32(b, 2)
b = binary.BigEndian.AppendUint32(b, 3)
b = binary.BigEndian.AppendUint32(b, 0) // variable attributes: absent
b = binary.BigEndian.AppendUint32(b, 0)
b = binary.BigEndian.AppendUint32(b, ncTypeDouble)
b = binary.BigEndian.AppendUint32(b, 0) // vsize
b = binary.BigEndian.AppendUint32(b, 0) // begin
}
// A file long enough for the innocuous variable's own records and for
// the wrapped span the buggy check computes.
for len(b) < 8192 {
b = append(b, 0)
}
path := writeHostile(t, "recsize-wrap.nc", b)
_, _, _, err := LoadNetCDF(path)
if err == nil {
t.Fatal("LoadNetCDF accepted a record variable whose dimensions cannot fit the file")
}
if !strings.Contains(err.Error(), "more elements than the file holds") {
t.Fatalf("error = %v, want the per-variable element bound", err)
}
}
// TestSaveFITSTableASCIIFloat32 pins the ASCII float encoder for a
// float32 column. The encoder read the float64 payload first and only
// then substituted the float32 one, but RawFloats is nil for every
// float32 array, so the first row of an ordinary float32 column indexed
// a nil slice.
func TestSaveFITSTableASCIIFloat32(t *testing.T) {
vals := []float32{1.5, -2.25, 3e8}
f32 := core.New(core.Float32, len(vals))
copy(f32.RawFloat32s(), vals)
cols := []FITSTableColumn{{Name: "F", Form: "E12.4", Data: f32}}
path := filepath.Join(t.TempDir(), "f32ascii.fits")
if err := SaveFITSTable(path, true, cols, nil); err != nil {
t.Fatalf("SaveFITSTable: %v", err)
}
table, err := LoadFITSTable(path)
if err != nil {
t.Fatalf("LoadFITSTable: %v", err)
}
for i, want := range vals {
if got := table.Columns[0].FloatAt(i); got != float64(want) {
t.Fatalf("F[%d] = %g, want %g", i, got, float64(want))
}
}
}
// TestLoadFITSTableFortranNumbers pins the number forms a Fortran writer
// puts in an ASCII table: strconv.ParseFloat accepts neither the Dw.d
// exponent ("1.5D+03") nor the exponent-less form ("1.5+03"), and one
// unparseable cell refused the whole table.
func TestLoadFITSTableFortranNumbers(t *testing.T) {
hdr := cardBlock(
card("XTENSION= 'TABLE '"),
card("BITPIX = 8"),
card("NAXIS = 2"),
card("NAXIS1 = 20"),
card("NAXIS2 = 4"),
card("TFIELDS = 1"),
card("TBCOL1 = 1"),
card("TFORM1 = 'D20.12'"),
card("END"),
)
rows := []string{"1.5D+03", "1.5+03", "-2.5d-2", "1.5E+03"}
payload := make([]byte, 0, 20*len(rows))
for _, r := range rows {
payload = append(payload, fmt.Sprintf("%-20s", r)...)
}
path := writeHostile(t, "fortran.fits", append(hdr, payload...))
table, err := LoadFITSTable(path)
if err != nil {
t.Fatalf("LoadFITSTable refused a Fortran table: %v", err)
}
want := []float64{1500, 1500, -0.025, 1500}
for i, w := range want {
if got := table.Columns[0].FloatAt(i); got != w {
t.Fatalf("row %d (%q) = %g, want %g", i, rows[i], got, w)
}
}
}
// TestLoadFITSTableScaleOverflow pins the integral-scaling bound. The
// guard used 9.3e18, which is above MaxInt64 (9.223372036854775807e18),
// so a scaled value in [2^63, 9.3e18) reached an out-of-range
// float-to-integer conversion: the column kept the int dtype and the
// value came back as MinInt64 instead of the documented promotion to
// float64.
func TestLoadFITSTableScaleOverflow(t *testing.T) {
const raw = int64(1) << 62
cols := []FITSTableColumn{{Name: "V", Form: "K", Data: mustInts(t, []int64{raw}, 1)}}
path := filepath.Join(t.TempDir(), "scaled.fits")
if err := SaveFITSTable(path, false, cols, map[string]string{"TSCAL1": "2", "TZERO1": "0"}); err != nil {
t.Fatalf("SaveFITSTable: %v", err)
}
table, err := LoadFITSTable(path)
if err != nil {
t.Fatalf("LoadFITSTable: %v", err)
}
col := table.Columns[0]
want := float64(raw) * 2 // 2^63, exactly representable in float64
if col.Dtype() != core.Float {
t.Fatalf("dtype = %s, want float64: a scaling past MaxInt64 must promote the column", col.Dtype())
}
if got := col.FloatAt(0); got != want {
t.Fatalf("scaled value = %g, want %g", got, want)
}
}
// TestMapCountsWithoutWrap pins the mmap count arithmetic: the byte
// length used to be int64(n)*elementSize, which wraps for a hostile
// count, so the wrapped length passed the file-size checks and the typed
// view was then built with the original count, which unsafe.Slice
// rejects with a panic instead of an error.
func TestMapCountsWithoutWrap(t *testing.T) {
path := filepath.Join(t.TempDir(), "small.bin")
if err := SaveNativeFloats(path, []float64{1, 2, 3, 4}); err != nil {
t.Fatal(err)
}
// 2^62+3 float64 values are 2^65+24 bytes, which wraps to 24: the
// file holds 32. The others already failed the non-positive-length
// check by luck and must keep failing it.
for _, n := range []int{1<<62 + 3, 1 << 61, 1 << 60, math.MaxInt} {
if _, _, err := MapFloats(path, 0, n); err == nil {
t.Errorf("MapFloats(n = %d) accepted a count the file cannot hold", n)
}
if _, _, err := MapFloat32s(path, 0, n); err == nil {
t.Errorf("MapFloat32s(n = %d) accepted a count the file cannot hold", n)
}
if _, _, err := MapInts(path, 0, n); err == nil {
t.Errorf("MapInts(n = %d) accepted a count the file cannot hold", n)
}
}
}
// TestSaveCSVZeroColumns pins the shape contract: an array with rows and
// no columns wrote one empty record per row, and encoding/csv reads
// blank lines as no records at all, so (3,0) came back as (0,0). It is
// refused instead, because no CSV text can carry the column count.
func TestSaveCSVZeroColumns(t *testing.T) {
a, err := core.FromFloats(nil, 3, 0)
if err != nil {
t.Fatalf("FromFloats(nil, 3, 0): %v", err)
}
var sb strings.Builder
if err := SaveCSVWriter(&sb, a); err == nil {
t.Fatalf("SaveCSVWriter accepted the shape %v and wrote %q, which reads back as a different shape",
a.Shape(), sb.String())
}
// A 2-D array with columns still round-trips.
ok := mustFloats(t, []float64{1, 2, 3, 4, 5, 6}, 2, 3)
var back strings.Builder
if err := SaveCSVWriter(&back, ok); err != nil {
t.Fatalf("SaveCSVWriter: %v", err)
}
got, err := LoadCSVReader(strings.NewReader(back.String()), false)
if err != nil {
t.Fatalf("LoadCSVReader: %v", err)
}
if got.Shape()[0] != 2 || got.Shape()[1] != 3 {
t.Fatalf("round trip gave %v, want [2 3]", got.Shape())
}
}
// TestIOSourceTypography pins the plain-ASCII rule for the package's own
// source: an em dash, an en dash, a Unicode minus, a middle dot or a
// multiplication sign in a comment or a message is a typographic symbol
// where the repository's style asks for ASCII.
func TestIOSourceTypography(t *testing.T) {
banned := []struct {
r rune
name string
}{
{'\u2014', "em dash"},
{'\u2013', "en dash"},
{'\u2212', "Unicode minus"},
{'\u00b7', "middle dot"},
{'\u00d7', "multiplication sign"},
}
entries, err := os.ReadDir(".")
if err != nil {
t.Fatalf("ReadDir: %v", err)
}
checked := 0
for _, e := range entries {
name := e.Name()
if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
continue
}
data, err := os.ReadFile(name)
if err != nil {
t.Fatalf("ReadFile(%s): %v", name, err)
}
checked++
for _, b := range banned {
if strings.ContainsRune(string(data), b.r) {
t.Errorf("%s carries a %s (U+%04X)", name, b.name, b.r)
}
}
}
if checked == 0 {
t.Fatal("no source files were checked")
}
}
Reference in New Issue Copy Permalink