tensor/io/hdf5write_test.go

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: MIT

package io

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"

	"sourcedock.dev/petrbalvin/tensor/internal/core"
)

// The writer tests round-trip through the reader: every file the
// writer produces is read back with LoadHDF5, the verified decoders
// of hdf5.go, and compared element for element, integers exactly and
// floats bit for bit. The reference fixtures under testdata/h5 close
// the circle: read, written, read again.

// hdf5WriteRead writes one file and reads it back, the shape every
// test below takes.
func hdf5WriteRead(t *testing.T, sets []HDF5Dataset, groupAttrs map[string]map[string]string, opts HDF5WriteOptions) []HDF5Dataset {
	t.Helper()
	path := filepath.Join(t.TempDir(), "written.h5")
	if err := SaveHDF5(path, sets, groupAttrs, opts); err != nil {
		t.Fatalf("SaveHDF5: %v", err)
	}
	back, err := LoadHDF5(path)
	if err != nil {
		t.Fatalf("LoadHDF5: %v", err)
	}
	return back
}

// hdf5CompareSets compares two dataset listings by path: shapes, dtypes,
// values and attributes, integers exactly and floats bit for bit. Both
// listings are compared in the reader's path order.
func hdf5CompareSets(t *testing.T, want, got []HDF5Dataset) {
	t.Helper()
	byPath := func(a, b HDF5Dataset) int { return strings.Compare(a.Path, b.Path) }
	slices.SortFunc(want, byPath)
	slices.SortFunc(got, byPath)
	if len(want) != len(got) {
		t.Fatalf("datasets = %d, want %d", len(got), len(want))
	}
	for i := range want {
		w, g := want[i], got[i]
		if g.Path != w.Path {
			t.Fatalf("dataset %d path = %q, want %q", i, g.Path, w.Path)
		}
		if !slices.Equal(g.Shape, w.Shape) {
			t.Fatalf("%s shape = %v, want %v", w.Path, g.Shape, w.Shape)
		}
		if g.Values.Dtype() != w.Values.Dtype() {
			t.Fatalf("%s dtype = %s, want %s", w.Path, g.Values.Dtype(), w.Values.Dtype())
		}
		switch w.Values.Dtype() {
		case core.Int:
			gi, wi := g.Values.RawInts(), w.Values.RawInts()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Float:
			gf, wf := g.Values.RawFloats(), w.Values.RawFloats()
			for j := range wf {
				if math.Float64bits(gf[j]) != math.Float64bits(wf[j]) {
					t.Fatalf("%s[%d] = %v, want %v (bit-exact)", w.Path, j, gf[j], wf[j])
				}
			}
		case core.Float32:
			gf, wf := g.Values.RawFloat32s(), w.Values.RawFloat32s()
			for j := range wf {
				if math.Float32bits(gf[j]) != math.Float32bits(wf[j]) {
					t.Fatalf("%s[%d] = %v, want %v (bit-exact)", w.Path, j, gf[j], wf[j])
				}
			}
		case core.Bool:
			gb, wb := g.Values.RawBools(), w.Values.RawBools()
			for j := range wb {
				if gb[j] != wb[j] {
					t.Fatalf("%s[%d] = %v, want %v", w.Path, j, gb[j], wb[j])
				}
			}
		case core.Int8:
			gi, wi := g.Values.RawInt8s(), w.Values.RawInt8s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Uint8:
			gi, wi := g.Values.RawUint8s(), w.Values.RawUint8s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Int16:
			gi, wi := g.Values.RawInt16s(), w.Values.RawInt16s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Uint16:
			gi, wi := g.Values.RawUint16s(), w.Values.RawUint16s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Int32:
			gi, wi := g.Values.RawInt32s(), w.Values.RawInt32s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		case core.Uint32:
			gi, wi := g.Values.RawUint32s(), w.Values.RawUint32s()
			for j := range wi {
				if gi[j] != wi[j] {
					t.Fatalf("%s[%d] = %d, want %d", w.Path, j, gi[j], wi[j])
				}
			}
		default:
			t.Fatalf("%s: the comparison pins no values for dtype %s", w.Path, w.Values.Dtype())
		}
		if len(g.Attrs) != len(w.Attrs) {
			t.Fatalf("%s attrs = %v, want %v", w.Path, g.Attrs, w.Attrs)
		}
		for k, v := range w.Attrs {
			if g.Attrs[k] != v {
				t.Fatalf("%s attr %q = %q, want %q", w.Path, k, g.Attrs[k], v)
			}
		}
	}
}

// TestHDF5WriteRoundTrip writes every numeric dtype in every rank the
// model carries, contiguous, and reads the values back exactly.
func TestHDF5WriteRoundTrip(t *testing.T) {
	floats, err := core.FromFloats([]float64{1.5, -2.5, math.Pi, math.MaxFloat64, math.SmallestNonzeroFloat64, 0, math.Inf(-1)}, 7)
	if err != nil {
		t.Fatal(err)
	}
	matrix, err := core.FromFloats([]float64{1, 2, 3, 4, 5, 6}, 2, 3)
	if err != nil {
		t.Fatal(err)
	}
	cube, err := core.FromFloats([]float64{1, 2, 3, 4, 5, 6, 7, 8}, 2, 2, 2)
	if err != nil {
		t.Fatal(err)
	}
	singles, err := core.FromFloat32s([]float32{1, -1.5, 3.25, 1e-20, math.MaxFloat32}, 5)
	if err != nil {
		t.Fatal(err)
	}
	ints, err := core.FromInts([]int64{0, 1, -1, math.MaxInt64, math.MinInt64 + 1, 1 << 40}, 2, 3)
	if err != nil {
		t.Fatal(err)
	}
	empty, err := core.FromInts([]int64{}, 0, 3)
	if err != nil {
		t.Fatal(err)
	}
	want := []HDF5Dataset{
		{Path: "/empty", Shape: empty.Shape(), Values: empty},
		{Path: "/floats", Shape: floats.Shape(), Values: floats},
		{Path: "/ints", Shape: ints.Shape(), Values: ints},
		{Path: "/matrix", Shape: matrix.Shape(), Values: matrix},
		{Path: "/singles", Shape: singles.Shape(), Values: singles},
		{Path: "/cube", Shape: cube.Shape(), Values: cube},
	}
	got := hdf5WriteRead(t, want, nil, HDF5WriteOptions{})
	hdf5CompareSets(t, want, got)
	// The same again through the latest layout, which stores every
	// dataset contiguously with checksummed headers.
	got = hdf5WriteRead(t, want, nil, HDF5WriteOptions{Latest: true})
	hdf5CompareSets(t, want, got)
}

// TestHDF5WriteGroupsAndAttrs writes nested groups with attributes on
// the root, the groups and the datasets, in both layouts, and checks
// what the reader reports: the dataset's own attributes and the ones
// inherited from the groups above it.
func TestHDF5WriteGroupsAndAttrs(t *testing.T) {
	for _, latest := range []bool{false, true} {
		inner, err := core.FromFloats([]float64{1, 2}, 2)
		if err != nil {
			t.Fatal(err)
		}
		outer, err := core.FromInts([]int64{7}, 1)
		if err != nil {
			t.Fatal(err)
		}
		sets := []HDF5Dataset{
			{Path: "/a/b/inner", Shape: inner.Shape(), Values: inner, Attrs: map[string]string{"units": "m/s"}},
			{Path: "/outer", Shape: outer.Shape(), Values: outer, Attrs: map[string]string{"note": "top level"}},
		}
		groupAttrs := map[string]map[string]string{
			"/":    {"title": "grouped"},
			"/a":   {"units": "group units"},
			"/a/b": {"depth": "2"},
		}
		got := hdf5WriteRead(t, sets, groupAttrs, HDF5WriteOptions{Latest: latest})
		byPath := map[string]HDF5Dataset{}
		for _, d := range got {
			byPath[d.Path] = d
		}
		innerGot, ok := byPath["/a/b/inner"]
		if !ok {
			t.Fatalf("latest=%v: /a/b/inner is missing in %v", latest, hdf5Paths(got))
		}
		// The dataset's own attribute and everything inherited from the
		// groups above it, the nearest group winning.
		for k, want := range map[string]string{"units": "m/s", "title": "grouped", "depth": "2"} {
			if got := innerGot.Attrs[k]; got != want {
				t.Fatalf("latest=%v: /a/b/inner attr %q = %q, want %q", latest, k, got, want)
			}
		}
		outerGot := byPath["/outer"]
		for k, want := range map[string]string{"note": "top level", "title": "grouped"} {
			if got := outerGot.Attrs[k]; got != want {
				t.Fatalf("latest=%v: /outer attr %q = %q, want %q", latest, k, got, want)
			}
		}
	}
}

// hdf5Paths lists the paths of a dataset listing, for error messages.
func hdf5Paths(sets []HDF5Dataset) []string {
	out := make([]string, len(sets))
	for i, d := range sets {
		out[i] = d.Path
	}
	return out
}

// TestHDF5WriteChunked writes chunked datasets through the deflate
// filter, the only route this writer takes to chunked storage: whole
// dataset chunks, edge chunks hanging past the shape, and enough
// chunks to force a multi-level chunk B-tree.
func TestHDF5WriteChunked(t *testing.T) {
	// ChunkBytes 64 splits the float64 axis into eight-element chunks;
	// 598 elements make 75 chunks, which is past the sixty-four
	// children one chunk B-tree node holds, and the last chunk hangs
	// six elements past the shape.
	want := make([]float64, 598)
	for i := range want {
		want[i] = float64(i) * 1.5
	}
	values, err := core.FromFloats(want, 598)
	if err != nil {
		t.Fatal(err)
	}
	// A three-dimensional dataset whose extent is not a whole number of
	// chunks along the trailing axis: the edge chunks are stored padded.
	rows := make([]float64, 0, 5*3*7)
	for i := range 5 * 3 * 7 {
		rows = append(rows, float64(i))
	}
	matrix, err := core.FromFloats(rows, 5, 3, 7)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		{Path: "/long", Shape: values.Shape(), Values: values},
		{Path: "/box", Shape: matrix.Shape(), Values: matrix},
	}
	got := hdf5WriteRead(t, sets, nil, HDF5WriteOptions{Gzip: 1, ChunkBytes: 64})
	hdf5CompareSets(t, sets, got)
}

// TestHDF5WriteFilters round-trips the deflate and shuffle filters at
// several levels, alone and together, on data whose chunking crosses
// chunk B-tree leaves.
func TestHDF5WriteFilters(t *testing.T) {
	want := make([]float64, 1000)
	for i := range want {
		// Runs of equal values compress; the tail differs so the
		// round-trip is not trivially constant.
		if i < 900 {
			want[i] = float64(i / 10)
		} else {
			want[i] = math.Sin(float64(i))
		}
	}
	values, err := core.FromFloats(want, 1000)
	if err != nil {
		t.Fatal(err)
	}
	ints, err := core.FromInts([]int64{5, 3, 5, 3, 5, 3, 5, 3}, 8)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		{Path: "/gzip_shuffle", Shape: values.Shape(), Values: values},
		{Path: "/ints", Shape: ints.Shape(), Values: ints},
	}
	for _, opts := range []HDF5WriteOptions{
		{Gzip: 1, ChunkBytes: 128},
		{Gzip: 6, ChunkBytes: 128},
		{Gzip: 9, ChunkBytes: 128},
		{Gzip: -1, ChunkBytes: 128},
		{Shuffle: true, ChunkBytes: 128},
		{Shuffle: true, Gzip: 4, ChunkBytes: 64},
		{Shuffle: true, Gzip: 6},
	} {
		got := hdf5WriteRead(t, sets, nil, opts)
		hdf5CompareSets(t, sets, got)
	}
	// The deflate stage genuinely compresses: eight hundred bytes of
	// repeating values deflate to a fraction of themselves. (The file
	// as a whole carries the B-tree nodes the format allocates, which
	// dwarf the data of a test-sized dataset.)
	flat := bytes.Repeat([]byte{0, 0, 0, 0, 0, 0, 0xF0, 0x3F}, 100)
	small, err := (&hdf5Writer{}).deflateChunk(flat, 6)
	if err != nil {
		t.Fatal(err)
	}
	if len(small) >= len(flat) {
		t.Fatalf("the deflate filter stored %d bytes for %d of data", len(small), len(flat))
	}
}

// TestHDF5WriteWideGroup writes a group with more children than two
// symbol table nodes hold, which forces a multi-level group B-tree,
// and reads every child back by path.
func TestHDF5WriteWideGroup(t *testing.T) {
	// 260 children: 33 symbol table nodes, which is past the
	// thirty-two children one group B-tree node holds, so the tree
	// grows a second level.
	var want []HDF5Dataset
	for i := range 260 {
		v, err := core.FromInts([]int64{int64(i)}, 1)
		if err != nil {
			t.Fatal(err)
		}
		want = append(want, HDF5Dataset{Path: fmt.Sprintf("/g/d%03d", i), Shape: v.Shape(), Values: v})
	}
	for _, latest := range []bool{false, true} {
		got := hdf5WriteRead(t, want, nil, HDF5WriteOptions{Latest: latest})
		hdf5CompareSets(t, want, got)
	}
}

// TestHDF5WriteAttrs pins the attribute round-trip: whole numbers
// return as int64 attributes, decimals as float64 ones, bracketed
// lists as arrays, and any other text as a fixed-length string.
func TestHDF5WriteAttrs(t *testing.T) {
	values, err := core.FromInts([]int64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		{Path: "/x", Shape: values.Shape(), Values: values, Attrs: map[string]string{
			"n":       "7",
			"neg":     "-12",
			"x":       "2.5",
			"tiny":    "1e-300",
			"arr":     "[1, 2, 3]",
			"floats":  "[0.5, -1.25, 2.75]",
			"label":   "hello world",
			"short":   "K",
			"unicode": "vlno moe",
			"spacey":  "  padded  ",
		}},
	}
	got := hdf5WriteRead(t, sets, nil, HDF5WriteOptions{})
	for k, want := range sets[0].Attrs {
		if got[0].Attrs[k] != want {
			t.Fatalf("attr %q = %q, want %q", k, got[0].Attrs[k], want)
		}
	}
	// A float64 attribute of a whole value reads back as "3" and is
	// rewritten as the int64 attribute "3": the value is preserved,
	// the type is the writer's inference.
	floaty, err := core.FromFloats([]float64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	sets = []HDF5Dataset{{Path: "/x", Shape: floaty.Shape(), Values: floaty, Attrs: map[string]string{"v": "3"}}}
	got = hdf5WriteRead(t, sets, nil, HDF5WriteOptions{})
	if got[0].Attrs["v"] != "3" {
		t.Fatalf("attr v = %q, want %q", got[0].Attrs["v"], "3")
	}
	// Non-numeric array elements are refused by name.
	sets[0].Attrs = map[string]string{"bad": "[1, x]"}
	path := filepath.Join(t.TempDir(), "bad.h5")
	if err := SaveHDF5(path, sets, nil); err == nil {
		t.Fatal("expected an error for a non-numeric array attribute")
	} else if !strings.Contains(err.Error(), "not a number") {
		t.Fatalf("error = %v, want a not-a-number refusal", err)
	}
}

// TestHDF5WriteText writes fixed-length string datasets, checks the
// datatype message they produce against the reader's own decoder, and
// confirms the reader refuses them exactly as its documentation says.
func TestHDF5WriteText(t *testing.T) {
	path := filepath.Join(t.TempDir(), "text.h5")
	texts := []HDF5TextDataset{
		{Path: "/words", Shape: []int{4}, Text: []string{"forty two", "hi", "", "K"}},
		{Path: "/grid", Shape: []int{2, 2}, Text: []string{"a", "bb", "ccc", "dddd"}},
	}
	if err := SaveHDF5Text(path, texts, HDF5WriteOptions{}); err != nil {
		t.Fatalf("SaveHDF5Text: %v", err)
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	// The words dataset stores four elements of nine bytes: the
	// datatype message is the fixed-length string class the reader
	// decodes for attributes, and its width is the longest element.
	pattern := []byte{0x13, 0x01, 0, 0}
	pattern = binary.LittleEndian.AppendUint32(pattern, 9)
	at := bytes.Index(raw, pattern)
	if at < 0 {
		t.Fatalf("no nine-byte fixed-string datatype message in %d bytes", len(raw))
	}
	dtype, err := decodeType(raw[at:at+8], true)
	if err != nil {
		t.Fatalf("decodeType: %v", err)
	}
	if dtype.class != 3 || dtype.width != 9 {
		t.Fatalf("datatype = class %d width %d, want class 3 width 9", dtype.class, dtype.width)
	}
	// The reader refuses string datasets by name.
	if _, err := LoadHDF5(path); err == nil {
		t.Fatal("expected the reader to refuse a string dataset")
	} else if !strings.Contains(err.Error(), "string datasets are not supported") {
		t.Fatalf("error = %v, want the string dataset refusal", err)
	}
	// The latest layout carries them too, still refused by the reader.
	path = filepath.Join(t.TempDir(), "text3.h5")
	if err := SaveHDF5Text(path, texts[:1], HDF5WriteOptions{Latest: true}); err != nil {
		t.Fatalf("SaveHDF5Text: %v", err)
	}
	if _, err := LoadHDF5(path); err == nil {
		t.Fatal("expected the reader to refuse a latest-format string dataset")
	}
	// Filters are refused by name rather than silently dropped.
	if err := SaveHDF5Text(filepath.Join(t.TempDir(), "z.h5"), texts, HDF5WriteOptions{Gzip: 6}); err == nil {
		t.Fatal("expected an error for filters on string datasets")
	}
	// A element count that does not fill the shape is refused.
	if err := SaveHDF5Text(filepath.Join(t.TempDir(), "s.h5"), []HDF5TextDataset{{Path: "/s", Shape: []int{3}, Text: []string{"a", "b"}}}, HDF5WriteOptions{}); err == nil {
		t.Fatal("expected an error for a short text list")
	}
}

// TestHDF5WriteLatest pins the latest layout against its checksums:
// the file reads back, and a single flipped byte inside either
// checksummed region, the superblock or an object header, refuses the
// read. The classic layout carries no such checksums, and a flipped
// data byte there reads back corrupted but without error.
func TestHDF5WriteLatest(t *testing.T) {
	values, err := core.FromFloats([]float64{1, 2, 3}, 3)
	if err != nil {
		t.Fatal(err)
	}
	deep, err := core.FromInts([]int64{9, 8}, 2)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		// The root attribute reaches every dataset on read, the
		// reader's inheritance rule, so the expected listings carry it.
		{Path: "/g/d", Shape: values.Shape(), Values: values, Attrs: map[string]string{"title": "latest"}},
		{Path: "/g/h/e", Shape: deep.Shape(), Values: deep, Attrs: map[string]string{"title": "latest"}},
	}
	path := filepath.Join(t.TempDir(), "latest.h5")
	if err := SaveHDF5(path, sets, map[string]map[string]string{"/": {"title": "latest"}}, HDF5WriteOptions{Latest: true}); err != nil {
		t.Fatal(err)
	}
	got, err := LoadHDF5(path)
	if err != nil {
		t.Fatalf("LoadHDF5: %v", err)
	}
	hdf5CompareSets(t, sets, got)
	// The superblock carries a lookup3 checksum over its first forty
	// bytes; byte 11 is the consistency flags, which nothing else
	// reads. The root object header, found the way the reader finds
	// it, checksums signature to last message, and the writer lays the
	// datasets' data out in front of it, so a flip in the first bytes
	// past the superblock lands in unchecksummed data instead.
	whole, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if got := hdf5Lookup3(whole[:44]); got != binary.LittleEndian.Uint32(whole[44:]) {
		t.Fatalf("the written superblock fails its own checksum: %#08x", got)
	}
	f, err := newHDF5File(whole)
	if err != nil {
		t.Fatal(err)
	}
	for _, c := range []struct {
		name string
		at   int
	}{
		{"superblock", 11},
		{"object header", int(f.rootAddress) + 8},
	} {
		corrupt := slices.Clone(whole)
		corrupt[c.at] ^= 0xff
		bad := filepath.Join(t.TempDir(), "corrupt.h5")
		if err := os.WriteFile(bad, corrupt, 0o644); err != nil {
			t.Fatal(err)
		}
		if _, err := LoadHDF5(bad); err == nil {
			t.Fatalf("expected an error for a corrupted %s", c.name)
		} else if !strings.Contains(err.Error(), "checksum") {
			t.Fatalf("corrupted %s: error = %v, want a checksum refusal", c.name, err)
		}
	}
	// The classic layout carries no checksums: a flipped data byte
	// reads back changed, silently.
	classic := filepath.Join(t.TempDir(), "classic.h5")
	if err := SaveHDF5(classic, sets[:1], nil); err != nil {
		t.Fatal(err)
	}
	whole, err = os.ReadFile(classic)
	if err != nil {
		t.Fatal(err)
	}
	// The dataset holds 1, 2, 3: the little-endian bytes of the value 1
	// sit in the data region and nowhere else in a file this small.
	one := binary.LittleEndian.AppendUint64(nil, math.Float64bits(1))
	at := bytes.Index(whole, one)
	if at < 0 {
		t.Fatal("the value 1 is not in the file")
	}
	whole[at] ^= 0xff
	flipped := filepath.Join(t.TempDir(), "flipped.h5")
	if err := os.WriteFile(flipped, whole, 0o644); err != nil {
		t.Fatal(err)
	}
	back, err := LoadHDF5(flipped)
	if err != nil {
		t.Fatalf("LoadHDF5: %v", err)
	}
	if back[0].Values.RawFloats()[0] == 1 {
		t.Fatal("the flipped byte did not reach the values")
	}
}

// TestHDF5WriteFixtureRoundTrip reads each reference fixture, writes
// it back and reads again: paths, shapes, dtypes, values and
// attributes must agree exactly, in the dtypes the fixtures store
// natively. fletcher.h5 is stored chunked and compressed; the
// round-trip rewrites it contiguous, so only the values and attributes
// are the contract there. fixture.h5's int32 dataset is written as
// int32, which the writer stores at its own width and the reader lands
// int32 again.
func TestHDF5WriteFixtureRoundTrip(t *testing.T) {
	for _, c := range []struct {
		name   string
		latest bool
	}{
		{"fixture.h5", false},
		{"fletcher.h5", false},
		{"latest.h5", true},
	} {
		sets, err := LoadHDF5(h5Fixture(t, c.name))
		if err != nil {
			t.Fatalf("%s: %v", c.name, err)
		}
		got := hdf5WriteRead(t, sets, nil, HDF5WriteOptions{Latest: c.latest})
		hdf5CompareSets(t, sets, got)
	}
}

// TestHDF5WriteRefusals pins the errors: every malformed input is
// refused with a message that names the defect, before any bytes are
// written.
func TestHDF5WriteRefusals(t *testing.T) {
	dir := t.TempDir()
	values, err := core.FromInts([]int64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	good := HDF5Dataset{Path: "/x", Shape: values.Shape(), Values: values}
	bad := func(sets []HDF5Dataset, attrs map[string]map[string]string, opts HDF5WriteOptions) error {
		return SaveHDF5(filepath.Join(dir, "bad.h5"), sets, attrs, opts)
	}
	for _, c := range []struct {
		name string
		err  error
		text string
	}{
		{"relative path", bad([]HDF5Dataset{{Path: "x", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "absolute"},
		{"root path", bad([]HDF5Dataset{{Path: "/", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "root"},
		{"empty segment", bad([]HDF5Dataset{{Path: "/a//b", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "empty segment"},
		{"trailing slash", bad([]HDF5Dataset{{Path: "/a/", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "empty segment"},
		{"duplicate", bad([]HDF5Dataset{good, good}, nil, HDF5WriteOptions{}), "twice"},
		{"group conflict", bad([]HDF5Dataset{good, {Path: "/x/y", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "both a dataset and a group"},
		{"no values", bad([]HDF5Dataset{{Path: "/x"}}, nil, HDF5WriteOptions{}), "no values"},
		{"unknown attribute group", bad([]HDF5Dataset{good}, map[string]map[string]string{"/nope": {"a": "1"}}, HDF5WriteOptions{}), "does not name a group"},
		{"empty attribute name", bad([]HDF5Dataset{good}, map[string]map[string]string{"/": {"": "1"}}, HDF5WriteOptions{}), "empty name"},
		{"nul in path", bad([]HDF5Dataset{{Path: "/a\x00b", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}), "NUL"},
		{"nul in attribute", bad([]HDF5Dataset{good}, map[string]map[string]string{"/": {"a\x00": "1"}}, HDF5WriteOptions{}), "NUL"},
		{"latest with gzip", bad([]HDF5Dataset{good}, nil, HDF5WriteOptions{Latest: true, Gzip: 6}), "version 2 B-tree"},
		{"latest with shuffle", bad([]HDF5Dataset{good}, nil, HDF5WriteOptions{Latest: true, Shuffle: true}), "version 2 B-tree"},
		{"gzip level", bad([]HDF5Dataset{good}, nil, HDF5WriteOptions{Gzip: 10}), "gzip level"},
		{"negative chunk target", bad([]HDF5Dataset{good}, nil, HDF5WriteOptions{ChunkBytes: -1}), "negative"},
		{"disagreeing shape", bad([]HDF5Dataset{{Path: "/x", Shape: []int{2, 1}, Values: values}}, nil, HDF5WriteOptions{}), "declares a shape"},
	} {
		if c.err == nil {
			t.Errorf("%s: expected an error, got none", c.name)
			continue
		}
		if !strings.Contains(c.err.Error(), c.text) {
			t.Errorf("%s: error = %v, want it to name %q", c.name, c.err, c.text)
		}
	}
	// A complex dataset is refused: the format carries it, the core
	// does not.
	complexes, err := core.FromComplexes([]complex128{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	err = bad([]HDF5Dataset{{Path: "/z", Shape: complexes.Shape(), Values: complexes}}, nil, HDF5WriteOptions{})
	if err == nil || !strings.Contains(err.Error(), "not supported") {
		t.Fatalf("complex dtype: error = %v, want an unsupported-dtype refusal", err)
	}
	// A chunk target so small that the chunk count explodes is refused
	// with the lever named; the shape is laid out directly because
	// materialising that many elements would cost more than the
	// refusal is worth.
	chunky := &hdf5Writer{opts: HDF5WriteOptions{ChunkBytes: 8}}
	_, _, cerr := chunky.writeChunks(&hdf5OutSet{path: "/w", shape: []int{hdf5MaxChunks + 1}, width: 8})
	if cerr == nil || !strings.Contains(cerr.Error(), "chunks") {
		t.Fatalf("tiny chunks: error = %v, want a chunk-count refusal", cerr)
	}
	// A text shape whose extents wrap the element count onto zero
	// matches an empty Text slice exactly, so the length check alone
	// cannot refuse it: the byte budget must, before a header goes out
	// declaring 2^64 elements.
	terr := SaveHDF5Text(filepath.Join(dir, "bad-text.h5"),
		[]HDF5TextDataset{{Path: "/t", Shape: []int{4294967296, 4294967296}, Text: nil}})
	if terr == nil || !strings.Contains(terr.Error(), "budget") {
		t.Fatalf("wrapped text extents: error = %v, want the byte-budget refusal", terr)
	}
	// Nothing above may have left a valid file behind: the refusals
	// happen before a byte is written.
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		t.Errorf("a refused write left %q behind", e.Name())
	}
}

// TestHDF5WriteEmptyFilteredRoundTrip keeps a dataset with no elements
// out of the chunked path: the filters have nothing to compress and a
// zero extent divides the chunk grid into zero-sized chunks, so the
// dataset is stored contiguous at an undefined address in both
// layouts.
func TestHDF5WriteEmptyFilteredRoundTrip(t *testing.T) {
	empty, err := core.FromFloats(nil, 0, 3)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{{Path: "/empty", Shape: empty.Shape(), Values: empty}}
	got := hdf5WriteRead(t, sets, nil, HDF5WriteOptions{Gzip: 6, Shuffle: true})
	hdf5CompareSets(t, sets, got)
	// The latest layout stores the empty dataset contiguous too; the
	// filtered latest combination is refused by design and pinned in
	// the refusals test.
	got = hdf5WriteRead(t, sets, nil, HDF5WriteOptions{Latest: true})
	hdf5CompareSets(t, sets, got)
}

// TestHDF5WriteLatestGroupBadAttrRefused pins the error the latest
// layout returns for a group attribute whose text does not parse: the
// refusal must come out of the write, not be overwritten by the
// header encode that follows it.
func TestHDF5WriteLatestGroupBadAttrRefused(t *testing.T) {
	values, err := core.FromFloats([]float64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{{Path: "/leaf", Shape: values.Shape(), Values: values}}
	attrs := map[string]map[string]string{"/": {"bad": "[1, x]"}}
	path := filepath.Join(t.TempDir(), "bad.h5")
	err = SaveHDF5(path, sets, attrs, HDF5WriteOptions{Latest: true})
	if err == nil {
		t.Fatal("SaveHDF5 with a malformed group attribute answered nil")
	}
	if !strings.Contains(err.Error(), "bad") {
		t.Fatalf("error = %v, want it to name the attribute", err)
	}
}

// TestHDF5WriteInterleavedNamesRoundTrip pins the child order the
// classic symbol table requires: the heap offsets and the B-tree keys
// sort in one merged name order, so a subgroup whose name interleaves
// with the datasets' is reachable. Round-trips both layouts.
func TestHDF5WriteInterleavedNamesRoundTrip(t *testing.T) {
	values, err := core.FromFloats([]float64{1, 2, 3}, 3)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		{Path: "/alpha", Shape: values.Shape(), Values: values},
		{Path: "/box", Shape: values.Shape(), Values: values},
		{Path: "/deep/nested", Shape: values.Shape(), Values: values},
		{Path: "/f32", Shape: values.Shape(), Values: values},
		{Path: "/zoom/in", Shape: values.Shape(), Values: values},
	}
	for _, latest := range []bool{false, true} {
		got := hdf5WriteRead(t, sets, nil, HDF5WriteOptions{Latest: latest})
		hdf5CompareSets(t, sets, got)
	}
}

// TestHDF5WriteChunkTreeKeyConvention parses a written chunk B-tree
// node and pins the key convention the reference files store: every
// real chunk key carries 0 in the trailing element-size coordinate and
// only the node's closing sentinel carries the element size, which is
// what orders it past the node's chunks. Coordinates ascend through
// the node.
func TestHDF5WriteChunkTreeKeyConvention(t *testing.T) {
	want := make([]float64, 90)
	for i := range want {
		want[i] = float64(i)
	}
	values, err := core.FromFloats(want, 10, 9)
	if err != nil {
		t.Fatal(err)
	}
	w := &hdf5Writer{opts: HDF5WriteOptions{ChunkBytes: 4 * 8}}
	s, err := hdf5PlanSet("SaveHDF5", &HDF5Dataset{Path: "/grid", Shape: values.Shape(), Values: values})
	if err != nil {
		t.Fatal(err)
	}
	tree, chunk, err := w.writeChunks(s)
	if err != nil {
		t.Fatal(err)
	}
	// One node: a 3x3 chunk grid sits far under the fanout.
	node := w.buf[tree:]
	if string(node[:4]) != "TREE" {
		t.Fatalf("chunk tree signature = %q, want TREE", node[:4])
	}
	if node[5] != 0 {
		t.Fatalf("chunk tree level = %d, want a leaf", node[5])
	}
	entries := int(binary.LittleEndian.Uint16(node[6:]))
	if entries != 9 {
		t.Fatalf("chunk tree entries = %d, want 9 (a 3x3 grid of %v)", entries, chunk)
	}
	keySize := 8 + 8*(len(s.shape)+1)
	p := 24
	for i := 0; i <= entries; i++ {
		coords := make([]uint64, len(s.shape)+1)
		for j := range coords {
			coords[j] = binary.LittleEndian.Uint64(node[p+8+8*j:])
		}
		last := i == entries
		if want := uint64(s.width); last {
			if coords[len(coords)-1] != want {
				t.Fatalf("sentinel trailing coordinate = %d, want the element size %d", coords[len(coords)-1], want)
			}
		} else {
			if coords[len(coords)-1] != 0 {
				t.Fatalf("chunk %d trailing coordinate = %d, want 0", i, coords[len(coords)-1])
			}
			if i > 0 && coords[len(coords)-2] <= binary.LittleEndian.Uint64(node[p-keySize+8+8*(len(coords)-2):]) && coords[0] == binary.LittleEndian.Uint64(node[p-keySize+8:]) {
				t.Fatalf("chunk %d coordinates do not advance past chunk %d", i, i-1)
			}
		}
		p += keySize
		if !last {
			p += 8
		}
	}
}

// TestHDF5WriteDeterministicBytes pins the writer's determinism claim:
// the same content supplied in different dataset and attribute orders
// writes byte for bit the same file.
func TestHDF5WriteDeterministicBytes(t *testing.T) {
	a, err := core.FromFloats([]float64{1, 2, 3}, 3)
	if err != nil {
		t.Fatal(err)
	}
	b, err := core.FromInts([]int64{7, 8}, 2)
	if err != nil {
		t.Fatal(err)
	}
	files := make([][]byte, 2)
	for i, sets := range [][]HDF5Dataset{
		{{Path: "/a", Shape: a.Shape(), Values: a}, {Path: "/b", Shape: b.Shape(), Values: b}},
		{{Path: "/b", Shape: b.Shape(), Values: b}, {Path: "/a", Shape: a.Shape(), Values: a}},
	} {
		path := filepath.Join(t.TempDir(), "det.h5")
		if err := SaveHDF5(path, sets, map[string]map[string]string{"/": {"x": "1", "y": "2"}}, HDF5WriteOptions{}); err != nil {
			t.Fatal(err)
		}
		files[i], err = os.ReadFile(path)
		if err != nil {
			t.Fatal(err)
		}
	}
	if !bytes.Equal(files[0], files[1]) {
		t.Fatalf("two writes of the same content in different orders differ in %d of %d bytes", countByteDiffs(files[0], files[1]), len(files[0]))
	}
}

func countByteDiffs(a, b []byte) int {
	n := 0
	for i := range a {
		if a[i] != b[i] {
			n++
		}
	}
	return n
}

// TestHDF5WriteChildOrderBytes pins the merged child order at the byte
// level, the pin the round-trip test cannot be: the reader walks the
// symbol nodes linearly and stays blind to their order, while the
// reference library binary-searches both the symbol node records and
// the B-tree keys on the local heap offsets. Ten root children whose
// names interleave groups and datasets span two symbol node leaves:
// their entries must read in one ascending heap-offset order matching
// the alphabetical names, and the group B-tree must route to the two
// leaves in order with closing keys on each leaf's last offset.
func TestHDF5WriteChildOrderBytes(t *testing.T) {
	values, err := core.FromFloats([]float64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	sets := []HDF5Dataset{
		{Path: "/ann", Shape: values.Shape(), Values: values},
		{Path: "/bee/x", Shape: values.Shape(), Values: values},
		{Path: "/cee", Shape: values.Shape(), Values: values},
		{Path: "/dee/x", Shape: values.Shape(), Values: values},
		{Path: "/eff", Shape: values.Shape(), Values: values},
		{Path: "/gee/x", Shape: values.Shape(), Values: values},
		{Path: "/its/x", Shape: values.Shape(), Values: values},
		{Path: "/jay", Shape: values.Shape(), Values: values},
		{Path: "/mill", Shape: values.Shape(), Values: values},
		{Path: "/zoo/x", Shape: values.Shape(), Values: values},
	}
	path := filepath.Join(t.TempDir(), "order.h5")
	if err := SaveHDF5(path, sets, nil, HDF5WriteOptions{}); err != nil {
		t.Fatal(err)
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	// The root group's leaves: every subgroup carries a symbol node of
	// exactly one child, so the leaves of the root are the nodes with
	// more. Eight children fit one leaf of eight slots, so ten spill
	// into two, eight and two.
	var leaves []int
	for at := range hdf5ScanSignature(raw, []byte("SNOD")) {
		if int(binary.LittleEndian.Uint16(raw[at+6:])) > 1 {
			leaves = append(leaves, at)
		}
	}
	if len(leaves) != 2 {
		t.Fatalf("the root group spans %d symbol node leaves, want 2", len(leaves))
	}
	// The local heap's data segment: the header names its address.
	heap := raw[hdf5FirstSignature(raw, []byte("HEAP")):]
	dataAddr := binary.LittleEndian.Uint64(heap[24:])
	nameAt := func(off uint64) string {
		start := int(dataAddr + off)
		end := start
		for raw[end] != 0 {
			end++
		}
		return string(raw[start:end])
	}
	var offsets []uint64
	var names []string
	for _, at := range leaves {
		entries := int(binary.LittleEndian.Uint16(raw[at+6:]))
		for i := range entries {
			off := binary.LittleEndian.Uint64(raw[at+8+40*i:])
			if len(offsets) > 0 && off <= offsets[len(offsets)-1] {
				t.Fatalf("heap offsets not ascending across the leaves: %d then %d", offsets[len(offsets)-1], off)
			}
			offsets = append(offsets, off)
			names = append(names, nameAt(off))
		}
	}
	want := []string{"ann", "bee", "cee", "dee", "eff", "gee", "its", "jay", "mill", "zoo"}
	if !slices.Equal(names, want) {
		t.Fatalf("children in heap-offset order = %v, want %v", names, want)
	}
	// The root group's B-tree: one child pointer per leaf, keyed by the
	// leaf's last offset. It is the tree with two children (each
	// subgroup carries a tree of its own, with one).
	treeAt := -1
	for i := range hdf5ScanSignature(raw, []byte("TREE")) {
		if raw[i+4] == 0 && int(binary.LittleEndian.Uint16(raw[i+6:])) == len(leaves) {
			treeAt = i
			break
		}
	}
	if treeAt < 0 {
		t.Fatal("no group B-tree carries the two root leaves")
	}
	prefix := make([]int, len(leaves)+1)
	for i, at := range leaves {
		prefix[i+1] = prefix[i] + int(binary.LittleEndian.Uint16(raw[at+6:]))
	}
	kids := int(binary.LittleEndian.Uint16(raw[treeAt+6:]))
	p := treeAt + 24
	// The node reads key0, then child and closing key per child.
	if key := binary.LittleEndian.Uint64(raw[p:]); key != 0 {
		t.Fatalf("the first B-tree key = %d, want the null name 0", key)
	}
	p += 8
	prev := uint64(0)
	for i := range kids {
		child := int(binary.LittleEndian.Uint64(raw[p:]))
		p += 8
		if child != leaves[i] {
			t.Fatalf("B-tree child %d points at %d, want the symbol node at %d", i, child, leaves[i])
		}
		key := binary.LittleEndian.Uint64(raw[p:])
		p += 8
		last := offsets[prefix[i+1]-1]
		if key != last {
			t.Fatalf("the closing key %d of child %d, want its leaf's last offset %d", key, i, last)
		}
		if key <= prev {
			t.Fatalf("B-tree keys not ascending at child %d: %d then %d", i, prev, key)
		}
		prev = key
	}
}

// hdf5ScanSignature yields the offsets of every signature block.
func hdf5ScanSignature(raw, sig []byte) func(yield func(int) bool) {
	return func(yield func(int) bool) {
		for i := 0; i+4 <= len(raw); i++ {
			if string(raw[i:i+4]) == string(sig) {
				if !yield(i) {
					return
				}
			}
		}
	}
}

// hdf5FirstSignature returns the offset of the first signature block.
func hdf5FirstSignature(raw, sig []byte) int {
	for at := range hdf5ScanSignature(raw, sig) {
		return at
	}
	return -1
}

// TestHDF5WriteHeaderBoundRefusals pins the four bounds the header
// encoders and the plan enforce: a version 1 message past the
// countable length, a single version 2 message past its own countable
// length, an attribute name past the length the attribute message
// counts, and a text dataset past the rank the format allows.
func TestHDF5WriteHeaderBoundRefusals(t *testing.T) {
	dir := t.TempDir()
	values, err := core.FromFloats([]float64{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	huge := strings.Repeat("x", 70000)
	for _, c := range []struct {
		name  string
		write func() error
		text  string
	}{
		{"classic message past 0xffff", func() error {
			return SaveHDF5(filepath.Join(dir, "v1.h5"),
				[]HDF5Dataset{{Path: "/d", Shape: values.Shape(), Values: values, Attrs: map[string]string{"big": huge}}},
				nil, HDF5WriteOptions{})
		}, "a version 1 header stores"},
		{"latest message past 0xffff", func() error {
			return SaveHDF5(filepath.Join(dir, "v2.h5"),
				[]HDF5Dataset{{Path: "/d", Shape: values.Shape(), Values: values, Attrs: map[string]string{"big": huge}}},
				nil, HDF5WriteOptions{Latest: true})
		}, "a version 2 header stores"},
		{"attribute name past 0xffff", func() error {
			return SaveHDF5(filepath.Join(dir, "name.h5"),
				[]HDF5Dataset{{Path: "/d", Shape: values.Shape(), Values: values, Attrs: map[string]string{huge: "1"}}},
				nil, HDF5WriteOptions{})
		}, "attribute message counts"},
		{"text dataset past the rank", func() error {
			shape := make([]int, hdf5MaxRank+1)
			return SaveHDF5Text(filepath.Join(dir, "rank.h5"),
				[]HDF5TextDataset{{Path: "/t", Shape: shape, Text: nil}}, HDF5WriteOptions{})
		}, "dimensions"},
	} {
		err := c.write()
		if err == nil {
			t.Errorf("%s: expected an error, got none", c.name)
			continue
		}
		if !strings.Contains(err.Error(), c.text) {
			t.Errorf("%s: error = %v, want it to name %q", c.name, err, c.text)
		}
	}
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		t.Errorf("a refused write left %q behind", e.Name())
	}
}

// hdf5NarrowSets builds one dataset per dtype the narrow write round
// added, shaped [5, 3] with the values at each dtype's extremes beside
// deterministic filler. A small chunk target splits the trailing axis
// of this shape into one- or two-column chunks, so the round-trip
// configs below exercise multi-chunk grids and, for the one-byte
// dtypes, edge chunks that hang past the shape.
func hdf5NarrowSets(t *testing.T) []HDF5Dataset {
	t.Helper()
	must := func(values *core.Array, err error) *core.Array {
		if err != nil {
			t.Fatal(err)
		}
		return values
	}
	mk := []HDF5Dataset{}
	add := func(path string, values *core.Array) {
		mk = append(mk, HDF5Dataset{Path: path, Shape: values.Shape(), Values: values})
	}
	add("/bool", must(core.FromBools([]bool{
		false, true, false,
		true, true, false,
		false, true, true,
		false, true, false,
		false, true, true}, 5, 3)))
	add("/int8", must(core.FromInt8s([]int8{
		-128, 127, 0,
		-1, 7, -7,
		42, -42, 100,
		-100, 3, -3,
		1, -56, 64}, 5, 3)))
	add("/uint8", must(core.FromUint8s([]uint8{
		0, 255, 1,
		7, 42, 100,
		3, 200, 64,
		128, 12, 99,
		254, 17, 8}, 5, 3)))
	add("/int16", must(core.FromInt16s([]int16{
		-32768, 32767, 0,
		-1, 256, -256,
		1000, -1000, 3,
		-3, 32766, -32767,
		17, 42, -7}, 5, 3)))
	add("/uint16", must(core.FromUint16s([]uint16{
		0, 65535, 1,
		256, 1000, 3,
		32768, 32767, 65534,
		17, 42, 999,
		8, 12, 5}, 5, 3)))
	add("/int32", must(core.FromInt32s([]int32{
		-2147483648, 2147483647, 0,
		-1, 65536, -65536,
		1000000, -1000000, 3,
		-3, 2147483646, -2147483647,
		17, 42, -7}, 5, 3)))
	add("/uint32", must(core.FromUint32s([]uint32{
		0, 4294967295, 1,
		65536, 1000000, 3,
		2147483648, 2147483647, 4294967294,
		17, 42, 999999,
		8, 12, 5}, 5, 3)))
	return mk
}

// TestHDF5WriteNarrowRoundTrip writes every dtype the narrow write
// round added, contiguous in both layouts and chunked through the
// deflate and shuffle filters at several chunk targets, and pins the
// round trip: the landing dtype equals the written dtype (int16 reads
// back int16, bool lands bool) and every value returns exactly, at
// each dtype's extremes.
func TestHDF5WriteNarrowRoundTrip(t *testing.T) {
	sets := hdf5NarrowSets(t)
	for _, opts := range []HDF5WriteOptions{
		{},
		{Latest: true},
		{Gzip: 1, ChunkBytes: 12},
		{Shuffle: true, ChunkBytes: 12},
		{Gzip: 6, Shuffle: true, ChunkBytes: 32},
	} {
		got := hdf5WriteRead(t, sets, nil, opts)
		hdf5CompareSets(t, sets, got)
	}
	// Zero-length narrow datasets, shape [0]: one contiguous and one
	// chunked config. The round trip has no values to compare, so the
	// pin is the path, the shape and the landing dtype per dataset.
	empty := hdf5NarrowEmptySets(t)
	for _, opts := range []HDF5WriteOptions{
		{},
		{Gzip: 1, ChunkBytes: 12},
	} {
		got := hdf5WriteRead(t, empty, nil, opts)
		hdf5CompareSets(t, empty, got)
	}
}

// hdf5NarrowEmptySets builds one shape-[0] dataset per narrow dtype:
// the zero-extent round trip the narrow write path must also carry.
func hdf5NarrowEmptySets(t *testing.T) []HDF5Dataset {
	t.Helper()
	must := func(values *core.Array, err error) *core.Array {
		if err != nil {
			t.Fatal(err)
		}
		return values
	}
	mk := []HDF5Dataset{}
	add := func(path string, values *core.Array) {
		mk = append(mk, HDF5Dataset{Path: path, Shape: values.Shape(), Values: values})
	}
	add("/bool0", must(core.FromBools([]bool{}, 0)))
	add("/int80", must(core.FromInt8s([]int8{}, 0)))
	add("/uint80", must(core.FromUint8s([]uint8{}, 0)))
	add("/int160", must(core.FromInt16s([]int16{}, 0)))
	add("/uint160", must(core.FromUint16s([]uint16{}, 0)))
	add("/int320", must(core.FromInt32s([]int32{}, 0)))
	add("/uint320", must(core.FromUint32s([]uint32{}, 0)))
	return mk
}

// TestHDF5WriteBoolMessage pins the boolean enumeration datatype
// message the writer emits, byte for byte, against the reader's own
// decoder: class 8 with two members in the class bit field and the
// reserved byte zero, a complete one-byte unsigned little-endian
// fixed-point base, the member names each NUL terminated and padded in
// their own field to a multiple of eight bytes, and the packed member
// values 0 and 1 behind the names. The payload bytes are the member
// values themselves.
func TestHDF5WriteBoolMessage(t *testing.T) {
	values, err := core.FromBools([]bool{false, true, false}, 3)
	if err != nil {
		t.Fatal(err)
	}
	path := filepath.Join(t.TempDir(), "bool.h5")
	if err := SaveHDF5(path, []HDF5Dataset{{Path: "/mask", Shape: values.Shape(), Values: values}}, nil, HDF5WriteOptions{}); err != nil {
		t.Fatalf("SaveHDF5: %v", err)
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	pattern := []byte{
		0x18, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // class 8, two members, one-byte values
		0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, // the unsigned base
		'F', 'A', 'L', 'S', 'E', 0, 0, 0,
		'T', 'R', 'U', 'E', 0, 0, 0, 0,
		0x00, 0x01, // FALSE = 0, TRUE = 1
	}
	at := bytes.Index(raw, pattern)
	if at < 0 {
		t.Fatalf("no boolean enumeration datatype message in %d bytes", len(raw))
	}
	dtype, err := decodeType(raw[at:at+len(pattern)], false)
	if err != nil {
		t.Fatalf("decodeType on the written message: %v", err)
	}
	if dtype.class != 8 || !dtype.isBool || dtype.width != 1 {
		t.Fatalf("datatype = class %d width %d isBool %v, want the one-byte boolean convention", dtype.class, dtype.width, dtype.isBool)
	}
	// The payload: the member values, one byte per element, which the
	// round-trip test reads back as the written booleans.
	if bytes.Index(raw, []byte{0x00, 0x01, 0x00}) < 0 {
		t.Fatal("the boolean payload 0, 1, 0 is not in the file")
	}
}

// TestHDF5WriteNarrowRefusals pins the refusal for a dtype outside the
// stored set: float16 and complex are named against the full list the
// writer stores, and the refusal happens before any bytes are written.
// The loud defaults of the payload and message switches are reached
// directly: a datatype class the plan never produces fails loudly
// rather than writing silent zeros or a file without a datatype
// message.
func TestHDF5WriteNarrowRefusals(t *testing.T) {
	const stored = "the writer stores bool, int8, uint8, int16, uint16, int32, uint32, int64, float32 and float64"
	dir := t.TempDir()
	halves, err := core.FromFloat16s([]float64{1.5}, 1)
	if err != nil {
		t.Fatal(err)
	}
	complexes, err := core.FromComplexes([]complex128{1}, 1)
	if err != nil {
		t.Fatal(err)
	}
	for _, c := range []struct {
		name   string
		values *core.Array
	}{
		{"float16", halves},
		{"complex", complexes},
	} {
		path := filepath.Join(dir, c.name+".h5")
		err := SaveHDF5(path, []HDF5Dataset{{Path: "/x", Shape: c.values.Shape(), Values: c.values}}, nil, HDF5WriteOptions{})
		if err == nil {
			t.Fatalf("%s: expected an unsupported-dtype refusal", c.name)
		}
		if want := "dtype " + c.name + " is not supported; " + stored; !strings.Contains(err.Error(), want) {
			t.Fatalf("%s: error = %v, want it to carry %q", c.name, err, want)
		}
		if _, statErr := os.Stat(path); !os.IsNotExist(statErr) {
			t.Fatalf("%s: the refused write left %q behind", c.name, path)
		}
	}
	unknown := &hdf5OutSet{path: "/bad", class: 2, width: 4, nbytes: 4}
	if err := unknown.encode(make([]byte, 4)); err == nil || !strings.Contains(err.Error(), "cannot serialise") {
		t.Fatalf("encode of class 2: err = %v, want the serialisation refusal", err)
	}
	w := &hdf5Writer{}
	if _, err := w.writeDataset(unknown); err == nil || !strings.Contains(err.Error(), "no datatype message") {
		t.Fatalf("writeDataset of class 2: err = %v, want the missing-message refusal", err)
	}
	odd := &hdf5OutSet{path: "/odd", class: 0, width: 3, nbytes: 3}
	if err := odd.encode(make([]byte, 3)); err == nil || !strings.Contains(err.Error(), "cannot serialise") {
		t.Fatalf("encode of a three-byte fixed-point payload: err = %v, want the serialisation refusal", err)
	}
}

// TestHDF5WriteNarrowDeterministicBytes pins the determinism claim for
// the narrow dtypes: the same content supplied in different dataset
// orders writes byte for bit the same file, contiguous in both
// layouts, chunked through shuffle alone and chunked through shuffle
// and deflate.
func TestHDF5WriteNarrowDeterministicBytes(t *testing.T) {
	sets := hdf5NarrowSets(t)
	reordered := slices.Clone(sets)
	slices.Reverse(reordered)
	for _, opts := range []HDF5WriteOptions{
		{},
		{Latest: true},
		{Shuffle: true, ChunkBytes: 12},
		{Gzip: 4, Shuffle: true, ChunkBytes: 12},
	} {
		files := make([][]byte, 2)
		for i, in := range [][]HDF5Dataset{sets, reordered} {
			path := filepath.Join(t.TempDir(), "det.h5")
			if err := SaveHDF5(path, in, nil, opts); err != nil {
				t.Fatal(err)
			}
			raw, err := os.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			files[i] = raw
		}
		if !bytes.Equal(files[0], files[1]) {
			t.Fatalf("opts %v: two writes of the same narrow content differ in %d of %d bytes",
				opts, countByteDiffs(files[0], files[1]), len(files[0]))
		}
	}
}