// Copyright (c) 2026 Petr BalvĂ­n (https://petrbalvin.org) // SPDX-License-Identifier: MIT package io import ( "encoding/binary" "math" "os" "path/filepath" "slices" "strings" "testing" "sourcedock.dev/petrbalvin/tensor/internal/core" ) // The HDF5 fixtures under testdata/h5 were written by the HDF5 // reference library, and every value below was read back from them // independently: the expected values in these tests are what the // reference reports, not what this reader // produces. // // fixture.h5: an int32 dataset stored contiguously, a float64 dataset // chunked, gzip compressed and shuffled, a float32 dataset // in a group, and string attributes on the root and the // group (variable-length, so they live in a global heap) // fletcher.h5: a float64 dataset chunked, gzip compressed, with the // fletcher32 checksum filter on top // latest.h5: written with libver="latest", so superblock version 3 // and object header version 2, with a float64 dataset /d // and one /g/e in a group func h5Fixture(t *testing.T, name string) string { t.Helper() return filepath.Join("testdata", "h5", name) } // TestLoadHDF5Values pins the reader against the reference-written fixture. func TestLoadHDF5Values(t *testing.T) { sets, err := LoadHDF5(h5Fixture(t, "fixture.h5")) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 3 { t.Fatalf("datasets = %d, want 3", len(sets)) } byPath := map[string]HDF5Dataset{} for _, d := range sets { byPath[d.Path] = d } // The paths come back sorted. if paths := []string{sets[0].Path, sets[1].Path, sets[2].Path}; paths[0] != "/floats" || paths[1] != "/g/f32" || paths[2] != "/ints" { t.Fatalf("paths = %v, want [/floats /g/f32 /ints]", paths) } ints, ok := byPath["/ints"] if !ok { t.Fatal("/ints is missing") } if s := ints.Shape; len(s) != 2 || s[0] != 2 || s[1] != 3 { t.Fatalf("/ints shape = %v, want [2 3]", s) } // The fixture's int32 dataset lands the native int32 dtype: the // reader keeps the width the file stores instead of widening it. if ints.Values.Dtype() != core.Int32 { t.Fatalf("/ints dtype = %s, want int32", ints.Values.Dtype()) } for i, want := range []int32{1, 2, 3, 4, 5, 6} { if got := ints.Values.RawInt32s()[i]; got != want { t.Fatalf("/ints[%d] = %d, want %d", i, got, want) } } floats, ok := byPath["/floats"] if !ok { t.Fatal("/floats is missing") } if s := floats.Shape; len(s) != 1 || s[0] != 4 { t.Fatalf("/floats shape = %v, want [4]", s) } if floats.Values.Dtype() != core.Float { t.Fatalf("/floats dtype = %s, want float64", floats.Values.Dtype()) } for i, want := range []float64{1.5, 2.5, 3.5, 4.5} { if got := floats.Values.RawFloats()[i]; got != want { t.Fatalf("/floats[%d] = %v, want %v", i, got, want) } } f32, ok := byPath["/g/f32"] if !ok { t.Fatal("/g/f32 is missing") } if s := f32.Shape; len(s) != 2 || s[0] != 2 || s[1] != 2 { t.Fatalf("/g/f32 shape = %v, want [2 2]", s) } if f32.Values.Dtype() != core.Float32 { t.Fatalf("/g/f32 dtype = %s, want float32", f32.Values.Dtype()) } for i, want := range []float32{1, 2, 3, 4} { if got := f32.Values.RawFloat32s()[i]; got != want { t.Fatalf("/g/f32[%d] = %v, want %v", i, got, want) } } // The attributes: the root's title reaches every dataset, and the // group's units reach the dataset inside it, the nearest group // winning. if got := ints.Attrs["title"]; got != "h5 fixture" { t.Fatalf("/ints title = %q, want %q", got, "h5 fixture") } if _, ok := ints.Attrs["units"]; ok { t.Fatalf("/ints picked up a group attribute it should not have: %v", ints.Attrs) } if got := f32.Attrs["units"]; got != "K" { t.Fatalf("/g/f32 units = %q, want K", got) } if got := f32.Attrs["title"]; got != "h5 fixture" { t.Fatalf("/g/f32 title = %q, want the inherited one", got) } } // TestLoadHDF5Fletcher32 pins the checksum filter: the chunk carries a // fletcher32 sum that must verify before the chunk is used. func TestLoadHDF5Fletcher32(t *testing.T) { sets, err := LoadHDF5(h5Fixture(t, "fletcher.h5")) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 1 { t.Fatalf("datasets = %d, want 1", len(sets)) } d := sets[0] if s := d.Shape; len(s) != 1 || s[0] != 20 { t.Fatalf("shape = %v, want [20]", s) } for i := range 20 { if got := d.Values.RawFloats()[i]; got != float64(i) { t.Fatalf("value %d = %v, want %d", i, got, i) } } } // TestLoadHDF5Latest pins the "latest" file format against the // reference-written fixture: superblock version 3, object headers // version 2 with their lookup3 checksums, compact groups carrying link // messages, and contiguous datasets. func TestLoadHDF5Latest(t *testing.T) { sets, err := LoadHDF5(h5Fixture(t, "latest.h5")) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 2 { t.Fatalf("datasets = %d, want 2", len(sets)) } d, e := sets[0], sets[1] if d.Path != "/d" || e.Path != "/g/e" { t.Fatalf("paths = %q, %q, want /d and /g/e", d.Path, e.Path) } if s := d.Shape; len(s) != 1 || s[0] != 3 { t.Fatalf("/d shape = %v, want [3]", s) } for i, want := range []float64{1, 2, 3} { if got := d.Values.RawFloats()[i]; got != want { t.Fatalf("/d[%d] = %v, want %v", i, got, want) } } if s := e.Shape; len(s) != 1 || s[0] != 1 { t.Fatalf("/g/e shape = %v, want [1]", s) } if got := e.Values.RawFloats()[0]; got != 4 { t.Fatalf("/g/e[0] = %v, want 4", got) } } // TestHDF5Lookup3 pins the checksum against the sums the reference // library wrote into the latest fixture: the superblock's and two // object headers'. The literals are what the file stores, not what // this implementation computes. func TestHDF5Lookup3(t *testing.T) { raw, err := os.ReadFile(h5Fixture(t, "latest.h5")) if err != nil { t.Fatal(err) } for _, c := range []struct { name string want uint32 lo int hi int }{ {"superblock", 0x39ff1913, 0, 44}, {"root header", 0xb91c2db3, 48, 175}, {"dataset header", 0x8d125cb5, 179, 443}, } { if got := hdf5Lookup3(raw[c.lo:c.hi]); got != c.want { t.Errorf("%s: lookup3 = %#08x, want %#08x", c.name, got, c.want) } } } // TestLoadHDF5Refusals pins the errors: a file that is not HDF5 at // all, a truncated file, and corrupted latest-format checksums must // each be refused with a message that says so, never read halfway. func TestLoadHDF5Refusals(t *testing.T) { dir := t.TempDir() notHDF5 := filepath.Join(dir, "plain.bin") if err := os.WriteFile(notHDF5, []byte("this is not an HDF5 file at all, not even close"), 0o644); err != nil { t.Fatal(err) } if _, err := LoadHDF5(notHDF5); err == nil { t.Fatal("expected an error for a file without the HDF5 signature") } else if !strings.Contains(err.Error(), "signature") { t.Fatalf("error = %v, want a signature refusal", err) } // A truncated copy of a valid file: the reader may accept it only // when the structures it actually reads are complete, and it must // never hand back a partial array. Whatever the cut, the call either // errors or returns datasets whose element count matches their // shape. whole, err := os.ReadFile(h5Fixture(t, "fixture.h5")) if err != nil { t.Fatal(err) } for _, cut := range []int{8, 32, 100, 600, len(whole) / 2, len(whole) - 4} { path := filepath.Join(dir, "cut.h5") if err := os.WriteFile(path, whole[:cut], 0o644); err != nil { t.Fatal(err) } sets, err := LoadHDF5(path) if err != nil { continue // refused, which is the expected answer } for _, d := range sets { n := 1 for _, s := range d.Shape { n *= s } if d.Values.Len() != n { t.Fatalf("a file truncated to %d bytes gave %q %d values for shape %v", cut, d.Path, d.Values.Len(), d.Shape) } } } // The header itself must be refused: a superblock shorter than its // fixed part cannot be read at all. if _, err := LoadHDF5(writeCut(t, dir, whole, 40)); err == nil { t.Fatal("expected an error for a file truncated inside the superblock") } // The latest format verifies its checksums: a flipped byte in the // superblock and one in an object header must each refuse the file // instead of reading past the corruption. latest, err := os.ReadFile(h5Fixture(t, "latest.h5")) if err != nil { t.Fatal(err) } for _, c := range []struct { name string at int }{ // Byte 11 is the superblock's consistency flags, which the // reader would otherwise ignore: only the checksum sees it. {"superblock", 11}, {"object header", 60}, } { corrupt := slices.Clone(latest) corrupt[c.at] ^= 0xff path := filepath.Join(dir, "corrupt.h5") if err := os.WriteFile(path, corrupt, 0o644); err != nil { t.Fatal(err) } if _, err := LoadHDF5(path); err == nil { t.Fatalf("expected an error for a corrupted %s", c.name) } else if !strings.Contains(err.Error(), "checksum") { t.Fatalf("corrupted %s: error = %v, want a checksum refusal", c.name, err) } } } // writeCut writes the first n bytes of data to a temp file and returns // its path. func writeCut(t *testing.T, dir string, data []byte, n int) string { t.Helper() path := filepath.Join(dir, "cut40.h5") if err := os.WriteFile(path, data[:n], 0o644); err != nil { t.Fatal(err) } return path } // h5FixedType renders a version 1 fixed-point datatype message of the // given element width and signedness. The message carries the bit // offset and bit precision the HDF5 file format specification's // fixed-point property table defines behind the eight-byte header, // twelve bytes in total; the reader keys the landing on the header's // size and signed bit. func h5FixedType(size uint32, signed bool) []byte { m := make([]byte, 12) m[0] = 0x10 // version 1, class 0 (fixed-point) if signed { m[1] = 0x08 // class bit field: bit 3 marks two's complement } binary.LittleEndian.PutUint32(m[4:], size) binary.LittleEndian.PutUint16(m[8:], 0) // bit offset binary.LittleEndian.PutUint16(m[10:], uint16(8*size)) // bit precision return m } // h5EnumBoolType renders the boolean enumeration datatype message HDF5 // writers carry booleans in, following the HDF5 file format // specification's enumeration class layout: the member count in the // class bit field, the base type as a complete fixed-point message, // each member name NUL-terminated and padded from its own field start // to a multiple of eight bytes, and the packed member values behind // the names. func h5EnumBoolType(names []string, values []byte) []byte { // Version 1, class 8; member count; size 1; then the base type. m := []byte{0x18, byte(len(names)), 0, 0, 1, 0, 0, 0} m = append(m, h5FixedType(1, false)...) for _, n := range names { start := len(m) m = append(m, n...) m = append(m, 0) for (len(m)-start)%8 != 0 { m = append(m, 0) } } m = append(m, values...) return m } // h5AttrMessage renders a version 1 attribute message: the name and // every field boundary padded to the eight-byte grid the message // format defines, then the value bytes. func h5AttrMessage(name string, dtypeMsg []byte, dims []uint64, value []byte) []byte { space := h5Dataspace(dims...) nameSize := len(name) + 1 dtypeAt := alignUp(8+nameSize, 8) spaceAt := alignUp(dtypeAt+len(dtypeMsg), 8) b := make([]byte, spaceAt+len(space)+len(value)) b[0] = 1 binary.LittleEndian.PutUint16(b[2:], uint16(nameSize)) binary.LittleEndian.PutUint16(b[4:], uint16(len(dtypeMsg))) binary.LittleEndian.PutUint16(b[6:], uint16(len(space))) copy(b[8:], name) // the trailing NUL is the buffer's own zero copy(b[dtypeAt:], dtypeMsg) copy(b[spaceAt:], space) copy(b[spaceAt+len(space):], value) return b } // h5ChunkTreeWidth writes a one-entry leaf chunk B-tree for a dataset // of the given rank whose chunk elements are width bytes wide: the // key's element-size slot must agree with the datatype, which the // reader checks. func h5ChunkTreeWidth(f []byte, off, rank int, width uint64, size uint32, chunkAt uint64) { copy(f[off:], hdf5Tree) f[off+4] = 1 // chunk tree f[off+5] = 0 // leaf level binary.LittleEndian.PutUint16(f[off+6:], 1) p := off + 24 binary.LittleEndian.PutUint32(f[p:], size) // The filter mask stays zero; the chunk offsets stay zero. binary.LittleEndian.PutUint64(f[p+8+8*rank:], width) binary.LittleEndian.PutUint64(f[p+8+8*(rank+1):], chunkAt) } // TestLoadHDF5NativeFixedPoint pins the fixed-point landings of the // contiguous path: every stored width and signedness lands the core // dtype that holds it exactly, extremes included, and int64 stays int. func TestLoadHDF5NativeFixedPoint(t *testing.T) { cases := []struct { name string dtype []byte payload []byte want core.Dtype check func(t *testing.T, a *core.Array) }{ {"int8", h5FixedType(1, true), []byte{0x80, 0x00, 0x7f}, core.Int8, func(t *testing.T, a *core.Array) { if got, want := a.RawInt8s()[:3], []int8{-128, 0, 127}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"uint8", h5FixedType(1, false), []byte{0x00, 0x01, 0xff}, core.Uint8, func(t *testing.T, a *core.Array) { if got, want := a.RawUint8s()[:3], []uint8{0, 1, 255}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"int16", h5FixedType(2, true), []byte{0x00, 0x80, 0xff, 0xff, 0xff, 0x7f}, core.Int16, func(t *testing.T, a *core.Array) { if got, want := a.RawInt16s()[:3], []int16{-32768, -1, 32767}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"uint16", h5FixedType(2, false), []byte{0x00, 0x00, 0x00, 0x10, 0xff, 0xff}, core.Uint16, func(t *testing.T, a *core.Array) { if got, want := a.RawUint16s()[:3], []uint16{0, 4096, 65535}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"int32", h5FixedType(4, true), []byte{0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, core.Int32, func(t *testing.T, a *core.Array) { if got, want := a.RawInt32s()[:3], []int32{-2147483648, -1, 2147483647}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"uint32", h5FixedType(4, false), []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0xff, 0xff, 0xff, 0xff}, core.Uint32, func(t *testing.T, a *core.Array) { if got, want := a.RawUint32s()[:3], []uint32{0, 1 << 30, 4294967295}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"int64 stays int", h5FixedType(8, true), []byte{0xfb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01, 0, 0}, core.Int, func(t *testing.T, a *core.Array) { if got, want := a.RawInts()[:3], []int64{-5, 0, 1 << 40}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { f := h5HostileFile(512) h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(3)}, h5Msg{hdf5MsgDatatype, tc.dtype}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(448, uint64(len(tc.payload)))}, ) copy(f[448:], tc.payload) sets, err := LoadHDF5(writeHostile(t, "native.h5", f)) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 1 { t.Fatalf("datasets = %d, want 1", len(sets)) } d := sets[0] if d.Values.Dtype() != tc.want { t.Fatalf("dtype = %s, want %s", d.Values.Dtype(), tc.want) } if s := d.Shape; len(s) != 1 || s[0] != 3 { t.Fatalf("shape = %v, want [3]", s) } tc.check(t, d.Values) }) } } // TestLoadHDF5ChunkedNativeLandings pins the chunked dispatch: the // per-cell decode lands the same native dtypes the contiguous path // lands, through the chunk B-tree and the placement walk. func TestLoadHDF5ChunkedNativeLandings(t *testing.T) { cases := []struct { name string dtype []byte width uint64 payload []byte want core.Dtype check func(t *testing.T, a *core.Array) }{ {"uint8", h5FixedType(1, false), 1, []byte{0, 1, 255, 42}, core.Uint8, func(t *testing.T, a *core.Array) { if got, want := a.RawUint8s()[:4], []uint8{0, 1, 255, 42}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"int16", h5FixedType(2, true), 2, []byte{0xfd, 0xff, 0x00, 0x80, 0xff, 0x7f, 0x07, 0x00}, core.Int16, func(t *testing.T, a *core.Array) { if got, want := a.RawInt16s()[:4], []int16{-3, -32768, 32767, 7}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, {"bool", h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}), 1, []byte{1, 0, 1, 1}, core.Bool, func(t *testing.T, a *core.Array) { if got, want := a.RawBools()[:4], []bool{true, false, true, true}; !slices.Equal(got, want) { t.Fatalf("values = %v, want %v", got, want) } }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { const btree, chunkAt = 256, 320 f := h5HostileFile(512) end := h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(4)}, h5Msg{hdf5MsgDatatype, tc.dtype}, h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV3(btree, 4, uint32(tc.width))}, ) if end > btree { t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree) } h5ChunkTreeWidth(f, btree, 1, tc.width, uint32(len(tc.payload)), chunkAt) copy(f[chunkAt:], tc.payload) sets, err := LoadHDF5(writeHostile(t, "chunk-native.h5", f)) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 1 { t.Fatalf("datasets = %d, want 1", len(sets)) } d := sets[0] if d.Values.Dtype() != tc.want { t.Fatalf("dtype = %s, want %s", d.Values.Dtype(), tc.want) } tc.check(t, d.Values) }) } } // enumBoolLoad builds a one-dataset contiguous file around a datatype // message and payload, and returns the load error or the dataset. func enumBoolLoad(t *testing.T, dtypeMsg, payload []byte) ([]HDF5Dataset, error) { t.Helper() f := h5HostileFile(512) h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(uint64(len(payload)))}, h5Msg{hdf5MsgDatatype, dtypeMsg}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(448, uint64(len(payload)))}, ) copy(f[448:], payload) return LoadHDF5(writeHostile(t, "enum.h5", f)) } // TestLoadHDF5EnumBoolLandings pins the boolean enumeration landing: // a one-byte unsigned base whose member values are a subset of {0, 1} // lands core.Bool whatever the member names say, because the values, // not the names, carry the semantics. func TestLoadHDF5EnumBoolLandings(t *testing.T) { cases := []struct { name string dtype []byte values []byte want []bool }{ {"members TRUE and FALSE", h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}), []byte{1, 0, 1}, []bool{true, false, true}}, {"names are irrelevant to the values", h5EnumBoolType([]string{"present", "absent"}, []byte{0, 1}), []byte{0, 1, 1}, []bool{false, true, true}}, {"a single member of zero", h5EnumBoolType([]string{"off"}, []byte{0}), []byte{0, 0, 0}, []bool{false, false, false}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { sets, err := enumBoolLoad(t, tc.dtype, tc.values) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 1 { t.Fatalf("datasets = %d, want 1", len(sets)) } d := sets[0] if d.Values.Dtype() != core.Bool { t.Fatalf("dtype = %s, want bool", d.Values.Dtype()) } if got := d.Values.RawBools()[:len(tc.want)]; !slices.Equal(got, tc.want) { t.Fatalf("values = %v, want %v", got, tc.want) } }) } } // TestLoadHDF5EnumRefusals pins the loud refusals: every enumeration // outside the boolean convention, every bit field, and a boolean // payload cell outside the members are named errors, never a silent // guess. The variants mutate the spec-shaped message, which also pins // the field offsets the parser reads. func TestLoadHDF5EnumRefusals(t *testing.T) { signed := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[9] |= 0x08; return m } bigEndian := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[9] |= 0x01; return m } baseClass := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[8] = 0x11; return m } baseSize := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}) binary.LittleEndian.PutUint32(m[12:], 2) return m } valueSize := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}) binary.LittleEndian.PutUint32(m[4:], 2) return m } noMembers := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[1] = 0; return m } reserved := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[3] = 0x04; return m } shortNames := func() []byte { m := h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}); m[1] = 3; return m } bitField := func() []byte { m := h5FixedType(1, false); m[0] = 0x14; return m } cases := []struct { name string dtype []byte payload []byte want string }{ {"a member value outside {0, 1}", h5EnumBoolType([]string{"A", "B"}, []byte{0, 2}), []byte{0, 1}, "outside the boolean convention"}, {"a signed base type", signed(), []byte{1, 0}, "signed base type"}, {"a big-endian base type", bigEndian(), []byte{1, 0}, "big-endian"}, {"a non-fixed-point base type", baseClass(), []byte{1, 0}, "base type of class 1"}, {"a base type wider than one byte", baseSize(), []byte{1, 0}, "base type of 2 bytes"}, {"values wider than one byte", valueSize(), []byte{1, 0}, "2-byte values"}, {"no members at all", noMembers(), []byte{0}, "declares 0 members"}, {"reserved bit field bits", reserved(), []byte{1, 0}, "unknown bit field bits"}, {"more members than names", shortNames(), []byte{1, 0}, "ends inside"}, {"a bit field datatype", bitField(), []byte{0}, "bit field"}, {"a payload cell outside the members", h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}), []byte{1, 7, 0}, "outside the members 0 and 1"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { sets, err := enumBoolLoad(t, tc.dtype, tc.payload) if err == nil { t.Fatalf("LoadHDF5 accepted %s: %d datasets, %v", tc.name, len(sets), sets) } if !strings.Contains(err.Error(), tc.want) { t.Fatalf("error = %v, want it to carry %q", err, tc.want) } }) } // The same refusal on the chunked path, where the per-cell decode // runs inside the placement walk. t.Run("a chunked payload cell outside the members", func(t *testing.T) { const btree, chunkAt = 256, 320 f := h5HostileFile(512) end := h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(4)}, h5Msg{hdf5MsgDatatype, h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0})}, h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV3(btree, 4, 1)}, ) if end > btree { t.Fatalf("the test object header runs to %d, past the chunk B-tree at %d", end, btree) } h5ChunkTreeWidth(f, btree, 1, 1, 4, chunkAt) copy(f[chunkAt:], []byte{1, 9, 0, 1}) _, err := LoadHDF5(writeHostile(t, "enum-chunk.h5", f)) if err == nil || !strings.Contains(err.Error(), "outside the members 0 and 1") { t.Fatalf("chunked enum payload of 9: err = %v, want the member refusal", err) } }) } // TestLoadHDF5Uint64Refused pins the unsigned 64-bit refusal at both // sites that hold it: the dataset gate and the value decode, with the // same text at each. func TestLoadHDF5Uint64Refused(t *testing.T) { const want = "unsigned 64-bit integers have no exact core dtype" f := h5HostileFile(512) h5ObjectHeader(f, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(1)}, h5Msg{hdf5MsgDatatype, h5FixedType(8, false)}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(448, 8)}, ) copy(f[448:], []byte{1, 0, 0, 0, 0, 0, 0, 0}) _, err := LoadHDF5(writeHostile(t, "uint64.h5", f)) if err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("LoadHDF5 on an unsigned 64-bit dataset: err = %v, want it to carry %q", err, want) } // The decode site, called directly: the same text, no widening. if _, err := arrayFromRaw(make([]byte, 8), hdf5Type{class: 0, size: 8, width: 8}, []int{1}); err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("arrayFromRaw on unsigned 64-bit bytes: err = %v, want it to carry %q", err, want) } // The chunked dispatch, through a dataset fixture: the chunk // dimensions carry the element size in their last slot, matching // the datatype, and the chunk B-tree address points past the end // of the file, so the pin also records that the dataset gate // refuses the datatype before any storage or tree is read. cf := h5HostileFile(512) h5ObjectHeader(cf, 96, h5Msg{hdf5MsgDataspace, h5Dataspace(1)}, h5Msg{hdf5MsgDatatype, h5FixedType(8, false)}, h5Msg{hdf5MsgDataLayout, h5ChunkLayoutV3(1024, 1, 8)}, ) if _, err := LoadHDF5(writeHostile(t, "uint64-chunked.h5", cf)); err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("LoadHDF5 on a chunked unsigned 64-bit dataset: err = %v, want it to carry %q", err, want) } // The chunkedArray dispatch itself, called directly with width 8 // unsigned: the same refusal, reached before any chunk walk. var fh hdf5File if _, err := fh.chunkedArray("/u64", []int{1}, hdf5Type{class: 0, size: 8, width: 8}, hdf5Layout{class: 2, dims: []int{1}}, nil, 8); err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("chunkedArray on unsigned 64-bit bytes: err = %v, want it to carry %q", err, want) } } // TestLoadHDF5AttributeSignedRendering pins the attribute text of // numeric attributes: the datatype's own signed bit decides how its // stored bits read, the boolean enumeration renders 0 and 1, unsigned // values keep every digit, and a cell outside the boolean members // drops the attribute instead of guessing. func TestLoadHDF5AttributeSignedRendering(t *testing.T) { const datasetAt, dataAt = 768, 832 f := h5HostileFile(896) msgs := []h5Msg{{hdf5MsgLink, h5HardLink("d", datasetAt)}} // The dataspace carries the element count; the value bytes follow // it as count many datatype-width cells. add := func(name string, dtypeMsg, value []byte, elems uint64) { msgs = append(msgs, h5Msg{hdf5MsgAttribute, h5AttrMessage(name, dtypeMsg, []uint64{elems}, value)}) } add("s8", h5FixedType(1, true), []byte{0xff}, 1) add("u8", h5FixedType(1, false), []byte{0xff}, 1) add("s16", h5FixedType(2, true), []byte{0xfe, 0xff}, 1) add("u32", h5FixedType(4, false), []byte{0xff, 0xff, 0xff, 0xff}, 1) add("flag", h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}), []byte{1, 0}, 2) add("bad", h5EnumBoolType([]string{"TRUE", "FALSE"}, []byte{1, 0}), []byte{1, 7}, 2) end := h5ObjectHeader(f, 96, msgs...) if end > datasetAt { t.Fatalf("the root header runs to %d, past the dataset at %d", end, datasetAt) } h5ObjectHeader(f, datasetAt, h5Msg{hdf5MsgDataspace, h5Dataspace(1)}, h5Msg{hdf5MsgDatatype, h5FloatType(8)}, h5Msg{hdf5MsgDataLayout, h5ContiguousLayout(dataAt, 8)}, ) binary.LittleEndian.PutUint64(f[dataAt:], math.Float64bits(2.5)) sets, err := LoadHDF5(writeHostile(t, "attrs-signed.h5", f)) if err != nil { t.Fatalf("LoadHDF5: %v", err) } if len(sets) != 1 || sets[0].Path != "/d" { t.Fatalf("datasets = %v, want the linked /d", sets) } attrs := sets[0].Attrs for k, want := range map[string]string{ "s8": "-1", "u8": "255", "s16": "-2", "u32": "4294967295", "flag": "[1, 0]", } { if got := attrs[k]; got != want { t.Fatalf("attr %s = %q, want %q", k, got, want) } } if v, ok := attrs["bad"]; ok { t.Fatalf("the attribute with a cell outside the members was accepted as %q", v) } if got := sets[0].Values.FloatAt(0); got != 2.5 { t.Fatalf("value = %v, want 2.5", got) } }