tensor/io/hdf5_continuation_pins_test.go

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: MIT

package io

import (
	"encoding/binary"
	"math"
	"strings"
	"testing"
)

// Regression pins: HDF5 headers the reference library writes
// once they outgrow their first block, byte orders and unallocated
// storage the reader used to answer silently, and the CSV byte order
// mark every spreadsheet writes.

// h5Link renders a continuation link message body: the block address
// and its length, eight bytes each in the hostile layout.
func h5Link(addr, length uint64) []byte {
	b := make([]byte, 16)
	binary.LittleEndian.PutUint64(b, addr)
	binary.LittleEndian.PutUint64(b[8:], length)
	return b
}

// h5ContinuationBlock writes a flat message-list block at off and
// returns the end offset, everything eight-aligned.
func h5ContinuationBlock(f []byte, off int, msgs ...h5Msg) int {
	for _, m := range msgs {
		binary.LittleEndian.PutUint16(f[off:], m.typ)
		binary.LittleEndian.PutUint16(f[off+2:], uint16(len(m.body)))
		copy(f[off+8:], m.body)
		off += alignUp(8+len(m.body), 8)
	}
	return off
}

// TestLoadHDF5ChainedContinuation: the header walk followed exactly
// one continuation block and dropped every message of the second and
// later ones, so a legal attribute-rich file lost its datasets. Here
// the dataspace sits in the header, the datatype in the first
// continuation block and the layout in the second: only a walk that
// follows the whole chain can assemble the dataset.
func TestLoadHDF5ChainedContinuation(t *testing.T) {
	const blockA, blockB, dataAt, n = 448, 640, 800, 1024
	f := h5HostileFile(n)
	end := h5ObjectHeader(f, 96,
		h5Msg{hdf5MsgDataspace, h5Dataspace(2)},
		h5Msg{hdf5MsgContinuation, h5Link(blockA, 112)},
	)
	if end > blockA {
		t.Fatalf("the header runs to %d, past the first block at %d", end, blockA)
	}
	endA := h5ContinuationBlock(f, blockA,
		h5Msg{hdf5MsgDatatype, h5FloatType(8)},
		h5Msg{hdf5MsgContinuation, h5Link(blockB, 88)},
	)
	if endA > blockB {
		t.Fatalf("block A runs to %d, past block B at %d", endA, blockB)
	}
	h5ContinuationBlock(f, blockB,
		h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(dataAt, 16)},
	)
	binary.LittleEndian.PutUint64(f[dataAt:], math.Float64bits(1.5))
	binary.LittleEndian.PutUint64(f[dataAt+8:], math.Float64bits(-2.5))
	sets, err := LoadHDF5(writeHostile(t, "chained.h5", f))
	if err != nil {
		t.Fatalf("LoadHDF5: %v", err)
	}
	if len(sets) != 1 {
		t.Fatalf("datasets = %d, want 1", len(sets))
	}
	v0 := sets[0].Values.FloatAt(0)
	if v0 != 1.5 || sets[0].Values.FloatAt(1) != -2.5 {
		t.Fatalf("values = %v, %v, want 1.5 and -2.5", v0, sets[0].Values.FloatAt(1))
	}
}

// TestLoadHDF5ContinuationCycle: a continuation block listing itself
// must be an error, not an infinite walk.
func TestLoadHDF5ContinuationCycle(t *testing.T) {
	const blockA, n = 448, 640
	f := h5HostileFile(n)
	h5ObjectHeader(f, 96,
		h5Msg{hdf5MsgDataspace, h5Dataspace(2)},
		h5Msg{hdf5MsgContinuation, h5Link(blockA, 40)},
	)
	// The block's only entry is a link back to itself.
	h5ContinuationBlock(f, blockA,
		h5Msg{hdf5MsgContinuation, h5Link(blockA, 40)},
	)
	if _, err := LoadHDF5(writeHostile(t, "cycle.h5", f)); err == nil || !strings.Contains(err.Error(), "twice") {
		t.Fatalf("a self-referencing continuation block: err = %v", err)
	}
}

// TestLoadHDF5BigEndianRefusal: the byte-order bit of the datatype was
// parsed and never checked, so a big-endian dataset decoded as
// byte-swapped noise with no error.
func TestLoadHDF5BigEndianRefusal(t *testing.T) {
	const dataAt, n = 448, 512
	f := h5HostileFile(n)
	beType := h5FloatType(8)
	beType[1] = 0x01 // class bit field: bit 0 set means big-endian
	h5ObjectHeader(f, 96,
		h5Msg{hdf5MsgDataspace, h5Dataspace(2)},
		h5Msg{hdf5MsgDatatype, beType},
		h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(dataAt, 16)},
	)
	if _, err := LoadHDF5(writeHostile(t, "bigendian.h5", f)); err == nil || !strings.Contains(err.Error(), "big-endian") {
		t.Fatalf("a big-endian dataset: err = %v", err)
	}
}

// TestLoadHDF5UnallocatedContiguous: the undefined storage address on
// a contiguous dataset refused the whole file; an empty dataset there
// is legal (nothing was ever allocated) and must load as empty, while
// a non-empty one names what is missing.
func TestLoadHDF5UnallocatedContiguous(t *testing.T) {
	t.Run("empty", func(t *testing.T) {
		f := h5HostileFile(512)
		h5ObjectHeader(f, 96,
			h5Msg{hdf5MsgDataspace, h5Dataspace(0)},
			h5Msg{hdf5MsgDatatype, h5FloatType(8)},
			h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(math.MaxUint64, 0)},
		)
		sets, err := LoadHDF5(writeHostile(t, "empty.h5", f))
		if err != nil {
			t.Fatalf("an empty unallocated dataset: %v", err)
		}
		if len(sets) != 1 || sets[0].Values.Len() != 0 {
			t.Fatalf("datasets = %d, want one empty dataset", len(sets))
		}
	})
	t.Run("non-empty", func(t *testing.T) {
		f := h5HostileFile(512)
		h5ObjectHeader(f, 96,
			h5Msg{hdf5MsgDataspace, h5Dataspace(2)},
			h5Msg{hdf5MsgDatatype, h5FloatType(8)},
			h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(math.MaxUint64, 16)},
		)
		if _, err := LoadHDF5(writeHostile(t, "noalloc.h5", f)); err == nil || !strings.Contains(err.Error(), "never allocated") {
			t.Fatalf("a non-empty unallocated dataset: err = %v", err)
		}
	})
}

// TestLoadCSVSkipBOM: a leading UTF-8 byte order mark used to glue
// itself onto the first field and fail the whole load with a strconv
// error.
func TestLoadCSVSkipBOM(t *testing.T) {
	const in = "\xEF\xBB\xBF1.5,2.5\n3,4\n"
	a, err := LoadCSVReader(strings.NewReader(in), false)
	if err != nil {
		t.Fatalf("LoadCSVReader with a BOM: %v", err)
	}
	if a.FloatAt(0) != 1.5 || a.FloatAt(1) != 2.5 || a.FloatAt(2) != 3 || a.FloatAt(3) != 4 {
		t.Fatalf("values = %v", a)
	}
}