635 lines
22 KiB
Go
635 lines
22 KiB
Go
|
// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
|
||
|
|
// SPDX-License-Identifier: MIT
|
||
|
|
|
||
|
|
package io
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/hex"
|
||
|
|
"math"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"slices"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"sourcedock.dev/petrbalvin/tensor/internal/core"
|
||
|
|
)
|
||
|
|
|
||
|
|
func ncTempPath(t *testing.T) string {
|
||
|
|
t.Helper()
|
||
|
|
return filepath.Join(t.TempDir(), "data.nc")
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFRoundTrip moves two variables of different dtypes through
|
||
|
|
// the file format and back, values, shapes, dimensions and attributes
|
||
|
|
// included.
|
||
|
|
func TestNetCDFRoundTrip(t *testing.T) {
|
||
|
|
dims := []NetCDFDim{{Name: "lat", Length: 3}, {Name: "lon", Length: 4}}
|
||
|
|
temp := mustFloats(t, []float64{
|
||
|
|
1.5, -2.25, 3.125, 4,
|
||
|
|
-5.5, 6.75, -7.875, 8,
|
||
|
|
9.25, -10.5, 11.125, -12,
|
||
|
|
}, 3, 4)
|
||
|
|
// A scalar carries one value and no dimensions.
|
||
|
|
scalar, err := core.FromFloats([]float64{42}, 1)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("FromFloats: %v", err)
|
||
|
|
}
|
||
|
|
vars := []NetCDFVar{
|
||
|
|
{
|
||
|
|
Name: "temp",
|
||
|
|
Dims: []string{"lat", "lon"},
|
||
|
|
Values: temp,
|
||
|
|
Attrs: map[string]string{"units": "degC", "long_name": "sea surface"},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
Name: "quality",
|
||
|
|
Values: scalar,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
path := ncTempPath(t)
|
||
|
|
attrs := map[string]string{"title": "round trip", "source": "tensor"}
|
||
|
|
if err := SaveNetCDF(path, dims, vars, attrs); err != nil {
|
||
|
|
t.Fatalf("SaveNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
gotDims, gotVars, gotAttrs, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("LoadNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
if len(gotDims) != 2 || gotDims[0] != dims[0] || gotDims[1] != dims[1] {
|
||
|
|
t.Fatalf("dims = %v, want %v", gotDims, dims)
|
||
|
|
}
|
||
|
|
for k, want := range attrs {
|
||
|
|
if gotAttrs[k] != want {
|
||
|
|
t.Fatalf("global attr %q = %q, want %q", k, gotAttrs[k], want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(gotVars) != 2 {
|
||
|
|
t.Fatalf("got %d variables, want 2", len(gotVars))
|
||
|
|
}
|
||
|
|
tv := gotVars[0]
|
||
|
|
if tv.Name != "temp" || len(tv.Dims) != 2 || tv.Dims[0] != "lat" || tv.Dims[1] != "lon" {
|
||
|
|
t.Fatalf("temp dims = %v", tv.Dims)
|
||
|
|
}
|
||
|
|
if tv.Values.NDim() != 2 || tv.Values.Shape()[0] != 3 || tv.Values.Shape()[1] != 4 {
|
||
|
|
t.Fatalf("temp shape = %v", tv.Values.Shape())
|
||
|
|
}
|
||
|
|
for i := range temp.Len() {
|
||
|
|
if tv.Values.FloatAt(i) != temp.FloatAt(i) {
|
||
|
|
t.Fatalf("temp[%d] = %g, want %g", i, tv.Values.FloatAt(i), temp.FloatAt(i))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for k, want := range map[string]string{"units": "degC", "long_name": "sea surface"} {
|
||
|
|
if tv.Attrs[k] != want {
|
||
|
|
t.Fatalf("temp attr %q = %q, want %q", k, tv.Attrs[k], want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
qv := gotVars[1]
|
||
|
|
if qv.Name != "quality" || len(qv.Dims) != 0 || qv.Values.FloatAt(0) != 42 {
|
||
|
|
t.Fatalf("quality = %v %v %g", qv.Name, qv.Dims, qv.Values.FloatAt(0))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFIntAndFloat32Ranges pins the integer and single-precision
|
||
|
|
// external types, extremes included: NC_INT is a signed 32-bit value
|
||
|
|
// that lands int32 on read, and the float32 round trip is exact
|
||
|
|
// through the float64 landing NC_FLOAT keeps.
|
||
|
|
func TestNetCDFIntAndFloat32Ranges(t *testing.T) {
|
||
|
|
ints, err := core.FromInts([]int64{-2147483648, -1, 0, 1, 2147483647}, 5)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("FromInts: %v", err)
|
||
|
|
}
|
||
|
|
f32, err := core.FromFloat32s([]float32{1.5, -2.25, 0, math.MaxFloat32, math.SmallestNonzeroFloat32}, 5)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("FromFloat32s: %v", err)
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
vars := []NetCDFVar{
|
||
|
|
{Name: "i", Dims: []string{"n"}, Values: ints},
|
||
|
|
{Name: "f", Dims: []string{"n"}, Values: f32},
|
||
|
|
}
|
||
|
|
if err := SaveNetCDF(path, []NetCDFDim{{Name: "n", Length: 5}}, vars, nil); err != nil {
|
||
|
|
t.Fatalf("SaveNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
_, got, _, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("LoadNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
if dt := got[0].Values.Dtype(); dt != core.Int32 {
|
||
|
|
t.Fatalf("int dtype = %s, want int32", dt)
|
||
|
|
}
|
||
|
|
wantInts := []int32{-2147483648, -1, 0, 1, 2147483647}
|
||
|
|
for i, want := range wantInts {
|
||
|
|
if got[0].Values.RawInt32s()[i] != want {
|
||
|
|
t.Fatalf("int[%d] = %d, want %d", i, got[0].Values.RawInt32s()[i], want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if dt := got[1].Values.Dtype(); dt != core.Float {
|
||
|
|
t.Fatalf("float32 dtype = %s, want the float64 landing NC_FLOAT keeps", dt)
|
||
|
|
}
|
||
|
|
for i := range 5 {
|
||
|
|
if got[1].Values.FloatAt(i) != float64(f32.RawFloat32s()[i]) {
|
||
|
|
t.Fatalf("f32[%d] = %g, want %g", i, got[1].Values.FloatAt(i), f32.RawFloat32s()[i])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFParsesKnownBytes pins the parser against the specification
|
||
|
|
// rather than against our own writer: this is the canonical tiny.nc
|
||
|
|
// from the NetCDF file format documentation, one SHORT variable "vx"
|
||
|
|
// of length 5 with values 3, 1, 4, 1, 5 and one fill-padded tail.
|
||
|
|
func TestNetCDFParsesKnownBytes(t *testing.T) {
|
||
|
|
const dump = "" +
|
||
|
|
"43444601" + // magic CDF\x01
|
||
|
|
"00000000" + // numrecs = 0
|
||
|
|
"0000000a" + // NC_DIMENSION
|
||
|
|
"00000001" + // one dimension
|
||
|
|
"0000000364696d00" + // name "dim" + pad
|
||
|
|
"00000005" + // length 5
|
||
|
|
"0000000000000000" + // gatt_list ABSENT
|
||
|
|
"0000000b" + // NC_VARIABLE
|
||
|
|
"00000001" + // one variable
|
||
|
|
"0000000276780000" + // name "vx" + pad
|
||
|
|
"00000001" + // rank 1
|
||
|
|
"00000000" + // dimid 0
|
||
|
|
"0000000000000000" + // vatt_list ABSENT
|
||
|
|
"00000003" + // NC_SHORT
|
||
|
|
"0000000c" + // vsize 12 (5 shorts + 1 fill pad)
|
||
|
|
"00000050" + // begin 80
|
||
|
|
"00030001000400010005" + // 3, 1, 4, 1, 5
|
||
|
|
"8001" // fill pad
|
||
|
|
raw, err := hex.DecodeString(dump)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
if len(raw) != 92 {
|
||
|
|
t.Fatalf("fixture is %d bytes, the specification says 92", len(raw))
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
dims, vars, attrs, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("LoadNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
if len(dims) != 1 || dims[0].Name != "dim" || dims[0].Length != 5 {
|
||
|
|
t.Fatalf("dims = %v", dims)
|
||
|
|
}
|
||
|
|
if len(vars) != 1 || vars[0].Name != "vx" || len(vars[0].Dims) != 1 || vars[0].Dims[0] != "dim" {
|
||
|
|
t.Fatalf("vars = %v", vars)
|
||
|
|
}
|
||
|
|
// NC_SHORT lands int16, the width the classic file stores.
|
||
|
|
if dt := vars[0].Values.Dtype(); dt != core.Int16 {
|
||
|
|
t.Fatalf("vx dtype = %s, want int16", dt)
|
||
|
|
}
|
||
|
|
if got := vars[0].Values.RawInt16s()[:5]; !slices.Equal(got, []int16{3, 1, 4, 1, 5}) {
|
||
|
|
t.Fatalf("vx = %v, want [3 1 4 1 5]", got)
|
||
|
|
}
|
||
|
|
want := []float64{3, 1, 4, 1, 5}
|
||
|
|
for i, w := range want {
|
||
|
|
if got := vars[0].Values.FloatAt(i); got != w {
|
||
|
|
t.Fatalf("vx[%d] = %g, want %g", i, got, w)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(attrs) != 0 {
|
||
|
|
t.Fatalf("attrs = %v, want none", attrs)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFErrors pins the refusal paths: a record dimension, an
|
||
|
|
// unknown version, a truncated file, bad names and a shape mismatch.
|
||
|
|
func TestNetCDFErrors(t *testing.T) {
|
||
|
|
path := ncTempPath(t)
|
||
|
|
a := mustFloats(t, []float64{1, 2, 3}, 3)
|
||
|
|
|
||
|
|
// The record dimension is supported, but only where the classic
|
||
|
|
// model allows it: leading the dimension list, leading each record
|
||
|
|
// variable's dimensions, and holding a whole number of records.
|
||
|
|
err := SaveNetCDF(path, []NetCDFDim{{Name: "x", Length: 2}, {Name: "t", Length: 0}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"x"}, Values: a}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "leads the list") {
|
||
|
|
t.Fatalf("record dimension not first: %v", err)
|
||
|
|
}
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "t", Length: 0}, {Name: "x", Length: 3}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"x", "t"}, Values: a}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "record axis leads") {
|
||
|
|
t.Fatalf("record axis not leading: %v", err)
|
||
|
|
}
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "t", Length: 0}, {Name: "x", Length: 2}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"t", "x"}, Values: a}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "whole number of records") {
|
||
|
|
t.Fatalf("partial record: %v", err)
|
||
|
|
}
|
||
|
|
two, terr := core.FromFloats([]float64{1, 2}, 2)
|
||
|
|
if terr != nil {
|
||
|
|
t.Fatalf("FromFloats: %v", terr)
|
||
|
|
}
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "t", Length: 0}, {Name: "x", Length: 1}},
|
||
|
|
[]NetCDFVar{
|
||
|
|
{Name: "u", Dims: []string{"t"}, Values: a},
|
||
|
|
{Name: "w", Dims: []string{"t", "x"}, Values: two},
|
||
|
|
}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "records, the others") {
|
||
|
|
t.Fatalf("record count disagreement: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// An int64 value outside int32 is refused, not truncated.
|
||
|
|
big, ferr := core.FromInts([]int64{1 << 40}, 1)
|
||
|
|
if ferr != nil {
|
||
|
|
t.Fatalf("FromInts: %v", ferr)
|
||
|
|
}
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "n", Length: 1}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"n"}, Values: big}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "int32 range") {
|
||
|
|
t.Fatalf("int64 out of range: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// A landed narrow variable is refused by the writer, whose classic
|
||
|
|
// model stores float64, float32 and int64 arrays only.
|
||
|
|
i8, ierr := core.FromInt8s([]int8{1, 2, 3}, 3)
|
||
|
|
if ierr != nil {
|
||
|
|
t.Fatalf("FromInt8s: %v", ierr)
|
||
|
|
}
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "n", Length: 3}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"n"}, Values: i8}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "SaveNetCDF") ||
|
||
|
|
!strings.Contains(err.Error(), "int8") ||
|
||
|
|
!strings.Contains(err.Error(), "the classic model stores float64, float32 and int arrays") {
|
||
|
|
t.Fatalf("int8 variable: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// The value count must match the dimensions.
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "n", Length: 4}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"n"}, Values: a}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "its dimensions hold 4") {
|
||
|
|
t.Fatalf("shape mismatch: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Names follow the traditional grammar.
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "bad name", Length: 1}},
|
||
|
|
[]NetCDFVar{{Name: "v", Dims: []string{"bad name"}, Values: mustFloats(t, []float64{1}, 1)}}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "forbids") {
|
||
|
|
t.Fatalf("bad name: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Duplicate variable names are refused.
|
||
|
|
err = SaveNetCDF(path, []NetCDFDim{{Name: "n", Length: 1}},
|
||
|
|
[]NetCDFVar{
|
||
|
|
{Name: "v", Dims: []string{"n"}, Values: mustFloats(t, []float64{1}, 1)},
|
||
|
|
{Name: "v", Dims: []string{"n"}, Values: mustFloats(t, []float64{2}, 1)},
|
||
|
|
}, nil)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "declared twice") {
|
||
|
|
t.Fatalf("duplicate variable: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Not a NetCDF file at all.
|
||
|
|
if err := os.WriteFile(path, []byte("not a netcdf file at all, sorry"), 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
if _, _, _, err := LoadNetCDF(path); err == nil || !strings.Contains(err.Error(), "not a NetCDF file") {
|
||
|
|
t.Fatalf("bad magic: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// A version this module does not speak.
|
||
|
|
if err := os.WriteFile(path, []byte{'C', 'D', 'F', 9}, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
if _, _, _, err := LoadNetCDF(path); err == nil || !strings.Contains(err.Error(), "unsupported NetCDF version") {
|
||
|
|
t.Fatalf("bad version: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// CDF-5, the 64-bit-count extension: refused by the same version
|
||
|
|
// gate rather than parsed as classic.
|
||
|
|
if err := os.WriteFile(path, []byte{'C', 'D', 'F', 5}, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
if _, _, _, err := LoadNetCDF(path); err == nil || !strings.Contains(err.Error(), "unsupported NetCDF version") {
|
||
|
|
t.Fatalf("CDF-5 version: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// A truncated header.
|
||
|
|
if err := os.WriteFile(path, []byte{'C', 'D', 'F', 1, 0, 0}, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
if _, _, _, err := LoadNetCDF(path); err == nil || !strings.Contains(err.Error(), "ends inside the header") {
|
||
|
|
t.Fatalf("truncated: %v", err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFRecordDimensionRead pins the record dimension at load
|
||
|
|
// time: a length of 0 in the leading dimension is the unlimited
|
||
|
|
// dimension, so the header parses and the dimension comes back with
|
||
|
|
// length 0 (its extent lives on each record variable's first axis).
|
||
|
|
func TestNetCDFRecordDimensionRead(t *testing.T) {
|
||
|
|
const dump = "" +
|
||
|
|
"43444601" + // magic
|
||
|
|
"00000001" + // numrecs = 1
|
||
|
|
"0000000a00000001" + // NC_DIMENSION, one dimension
|
||
|
|
"0000000174000000" + // name "t" + pad
|
||
|
|
"00000000" + // length 0: the record dimension
|
||
|
|
"0000000000000000" + // gatt ABSENT
|
||
|
|
"0000000b00000000" // NC_VARIABLE, zero variables
|
||
|
|
raw, err := hex.DecodeString(dump)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
dims, vars, _, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("record dimension: %v", err)
|
||
|
|
}
|
||
|
|
if len(dims) != 1 || dims[0].Name != "t" || dims[0].Length != 0 {
|
||
|
|
t.Fatalf("dims = %+v, want one record dimension named t with length 0", dims)
|
||
|
|
}
|
||
|
|
if len(vars) != 0 {
|
||
|
|
t.Fatalf("vars = %+v, want none", vars)
|
||
|
|
}
|
||
|
|
// A second zero-length dimension is refused: only one unlimited
|
||
|
|
// dimension exists, and it leads the list.
|
||
|
|
const twoRecords = "" +
|
||
|
|
"43444601" + // magic
|
||
|
|
"00000001" + // numrecs = 1
|
||
|
|
"0000000a00000002" + // NC_DIMENSION, two dimensions
|
||
|
|
"0000000174000000" + // name "t" + pad
|
||
|
|
"00000000" + // length 0: the record dimension
|
||
|
|
"0000000178000000" + // name "x" + pad
|
||
|
|
"00000000" + // length 0 again: refused
|
||
|
|
"0000000000000000" + // gatt ABSENT
|
||
|
|
"0000000b00000000" // NC_VARIABLE, zero variables
|
||
|
|
raw, err = hex.DecodeString(twoRecords)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
if _, _, _, err := LoadNetCDF(path); err == nil {
|
||
|
|
t.Fatal("expected an error for a second zero-length dimension")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFClassicNativeLandings pins the classic type codes onto the
|
||
|
|
// core dtypes they land, against a hand-built CDF-1 file rather than
|
||
|
|
// the package's own writer: NC_BYTE as int8, NC_SHORT as int16,
|
||
|
|
// NC_INT as int32, NC_CHAR as uint8 raw bytes, and NC_FLOAT and
|
||
|
|
// NC_DOUBLE keeping their float64 landing. The values carry the
|
||
|
|
// extremes of every width, so a widened or unsigned reading of any of
|
||
|
|
// them fails the pin.
|
||
|
|
func TestNetCDFClassicNativeLandings(t *testing.T) {
|
||
|
|
const dump = "" +
|
||
|
|
"43444601" + // magic CDF\x01
|
||
|
|
"00000000" + // numrecs = 0
|
||
|
|
"0000000a00000001" + // NC_DIMENSION, one dimension
|
||
|
|
"000000016e000000" + // name "n" + pad
|
||
|
|
"00000004" + // length 4
|
||
|
|
"0000000000000000" + // gatt_list ABSENT
|
||
|
|
"0000000b00000006" + // NC_VARIABLE, six variables
|
||
|
|
"000000016200000000000001000000000000000000000000" + // "b": rank 1, dim 0, no attrs
|
||
|
|
"00000001" + // NC_BYTE
|
||
|
|
"00000004" + // vsize 4
|
||
|
|
"00000104" + // begin 260
|
||
|
|
"000000017300000000000001000000000000000000000000" + // "s"
|
||
|
|
"00000003" + // NC_SHORT
|
||
|
|
"00000008" +
|
||
|
|
"00000108" + // begin 264
|
||
|
|
"000000016900000000000001000000000000000000000000" + // "i"
|
||
|
|
"00000004" + // NC_INT
|
||
|
|
"00000010" +
|
||
|
|
"00000110" + // begin 272
|
||
|
|
"000000016300000000000001000000000000000000000000" + // "c"
|
||
|
|
"00000002" + // NC_CHAR
|
||
|
|
"00000004" +
|
||
|
|
"00000120" + // begin 288
|
||
|
|
"000000016600000000000001000000000000000000000000" + // "f"
|
||
|
|
"00000005" + // NC_FLOAT
|
||
|
|
"00000010" +
|
||
|
|
"00000124" + // begin 292
|
||
|
|
"000000016400000000000001000000000000000000000000" + // "d"
|
||
|
|
"00000006" + // NC_DOUBLE
|
||
|
|
"00000020" +
|
||
|
|
"00000134" + // begin 308
|
||
|
|
"80ff007f" + // b: -128, -1, 0, 127
|
||
|
|
"8000ffff00017fff" + // s: -32768, -1, 1, 32767
|
||
|
|
"80000000ffffffff000000017fffffff" + // i: int32 extremes
|
||
|
|
"0041c8ff" + // c: raw bytes 0, 65, 200, 255
|
||
|
|
"3fc00000c02000000000000040500000" + // f: 1.5, -2.5, 0, 3.25
|
||
|
|
"3ff8000000000000c004000000000000" + // d: 1.5, -2.5,
|
||
|
|
"00000000000000004010000000000000" // 0, 4
|
||
|
|
raw, err := hex.DecodeString(dump)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
dims, vars, attrs, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("LoadNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
if len(dims) != 1 || dims[0].Name != "n" || dims[0].Length != 4 {
|
||
|
|
t.Fatalf("dims = %v", dims)
|
||
|
|
}
|
||
|
|
if len(attrs) != 0 {
|
||
|
|
t.Fatalf("attrs = %v, want none", attrs)
|
||
|
|
}
|
||
|
|
if len(vars) != 6 {
|
||
|
|
t.Fatalf("vars = %d, want 6", len(vars))
|
||
|
|
}
|
||
|
|
byName := map[string]NetCDFVar{}
|
||
|
|
for _, v := range vars {
|
||
|
|
byName[v.Name] = v
|
||
|
|
if s := v.Values.Shape(); len(s) != 1 || s[0] != 4 {
|
||
|
|
t.Fatalf("%s shape = %v, want [4]", v.Name, s)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if dt := byName["b"].Values.Dtype(); dt != core.Int8 {
|
||
|
|
t.Fatalf("NC_BYTE dtype = %s, want int8", dt)
|
||
|
|
}
|
||
|
|
if got, want := byName["b"].Values.RawInt8s()[:4], []int8{-128, -1, 0, 127}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("NC_BYTE values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
if dt := byName["s"].Values.Dtype(); dt != core.Int16 {
|
||
|
|
t.Fatalf("NC_SHORT dtype = %s, want int16", dt)
|
||
|
|
}
|
||
|
|
if got, want := byName["s"].Values.RawInt16s()[:4], []int16{-32768, -1, 1, 32767}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("NC_SHORT values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
if dt := byName["i"].Values.Dtype(); dt != core.Int32 {
|
||
|
|
t.Fatalf("NC_INT dtype = %s, want int32", dt)
|
||
|
|
}
|
||
|
|
if got, want := byName["i"].Values.RawInt32s()[:4], []int32{-2147483648, -1, 1, 2147483647}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("NC_INT values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
// CHAR carries bytes, not text: the array level lands uint8 and
|
||
|
|
// what the bytes spell is the caller's question.
|
||
|
|
if dt := byName["c"].Values.Dtype(); dt != core.Uint8 {
|
||
|
|
t.Fatalf("NC_CHAR dtype = %s, want uint8", dt)
|
||
|
|
}
|
||
|
|
if got, want := byName["c"].Values.RawUint8s()[:4], []uint8{0, 65, 200, 255}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("NC_CHAR values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
if dt := byName["f"].Values.Dtype(); dt != core.Float {
|
||
|
|
t.Fatalf("NC_FLOAT dtype = %s, want float", dt)
|
||
|
|
}
|
||
|
|
for i, want := range []float64{1.5, -2.5, 0, 3.25} {
|
||
|
|
if got := byName["f"].Values.FloatAt(i); got != want {
|
||
|
|
t.Fatalf("f[%d] = %v, want %v", i, got, want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if dt := byName["d"].Values.Dtype(); dt != core.Float {
|
||
|
|
t.Fatalf("NC_DOUBLE dtype = %s, want float", dt)
|
||
|
|
}
|
||
|
|
for i, want := range []float64{1.5, -2.5, 0, 4} {
|
||
|
|
if got := byName["d"].Values.FloatAt(i); got != want {
|
||
|
|
t.Fatalf("d[%d] = %v, want %v", i, got, want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFRecordNarrowLandings pins the record path on a hand-built
|
||
|
|
// CDF-1 file: an NC_BYTE record variable's slabs land int8 in place
|
||
|
|
// and an NC_SHORT record variable's slabs land int16, including the
|
||
|
|
// slab padding the classic layout writes between records, and a fixed
|
||
|
|
// NC_SHORT variable lands int16 beside them.
|
||
|
|
func TestNetCDFRecordNarrowLandings(t *testing.T) {
|
||
|
|
const dump = "" +
|
||
|
|
"43444601" + // magic
|
||
|
|
"00000002" + // numrecs = 2
|
||
|
|
"0000000a00000002" + // NC_DIMENSION, two dimensions
|
||
|
|
"0000000372656300" + // name "rec" + pad
|
||
|
|
"00000000" + // length 0: the record dimension
|
||
|
|
"0000000178000000" + // name "x" + pad
|
||
|
|
"00000003" + // length 3
|
||
|
|
"0000000000000000" + // gatt_list ABSENT
|
||
|
|
"0000000b00000003" + // NC_VARIABLE, three variables
|
||
|
|
"0000000272620000" + // name "rb" + pad
|
||
|
|
"00000001" + // rank 1
|
||
|
|
"00000000" + // dimid 0: the record dimension
|
||
|
|
"0000000000000000" + // vatt ABSENT
|
||
|
|
"00000001" + // NC_BYTE
|
||
|
|
"00000004" + // vsize 4: one byte padded to four
|
||
|
|
"000000ac" + // begin 172: its slab inside the first record
|
||
|
|
"0000000273730000" + // name "ss" + pad
|
||
|
|
"00000001" + // rank 1
|
||
|
|
"00000001" + // dimid 1: x, fixed
|
||
|
|
"0000000000000000" + // vatt ABSENT
|
||
|
|
"00000003" + // NC_SHORT
|
||
|
|
"00000008" + // vsize 8: six bytes padded to eight
|
||
|
|
"000000a4" + // begin 164
|
||
|
|
"0000000272730000" + // name "rs" + pad
|
||
|
|
"00000001" + // rank 1
|
||
|
|
"00000000" + // dimid 0: the record dimension
|
||
|
|
"0000000000000000" + // vatt ABSENT
|
||
|
|
"00000003" + // NC_SHORT
|
||
|
|
"00000004" + // vsize 4: one short padded to four
|
||
|
|
"000000b0" + // begin 176: its slab inside the first record
|
||
|
|
"0005fffa0007" + // ss: 5, -6, 7
|
||
|
|
"0000" + // slab padding
|
||
|
|
"07000000" + // rb record 0: 7
|
||
|
|
"12340000" + // rs record 0: 0x1234
|
||
|
|
"c8000000" + // rb record 1: 200 as int8 is -56
|
||
|
|
"fff00000" // rs record 1: -16
|
||
|
|
raw, err := hex.DecodeString(dump)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
dims, vars, _, err := LoadNetCDF(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("LoadNetCDF: %v", err)
|
||
|
|
}
|
||
|
|
if len(dims) != 2 || dims[0].Name != "rec" || dims[0].Length != 0 || dims[1].Length != 3 {
|
||
|
|
t.Fatalf("dims = %v", dims)
|
||
|
|
}
|
||
|
|
byName := map[string]NetCDFVar{}
|
||
|
|
for _, v := range vars {
|
||
|
|
byName[v.Name] = v
|
||
|
|
}
|
||
|
|
rb := byName["rb"].Values
|
||
|
|
if rb.Dtype() != core.Int8 {
|
||
|
|
t.Fatalf("rb dtype = %s, want int8", rb.Dtype())
|
||
|
|
}
|
||
|
|
if s := rb.Shape(); len(s) != 1 || s[0] != 2 {
|
||
|
|
t.Fatalf("rb shape = %v, want [2]: the record axis carries the record count", s)
|
||
|
|
}
|
||
|
|
if got, want := rb.RawInt8s()[:2], []int8{7, -56}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("rb values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
rs := byName["rs"].Values
|
||
|
|
if rs.Dtype() != core.Int16 {
|
||
|
|
t.Fatalf("rs dtype = %s, want int16", rs.Dtype())
|
||
|
|
}
|
||
|
|
if s := rs.Shape(); len(s) != 1 || s[0] != 2 {
|
||
|
|
t.Fatalf("rs shape = %v, want [2]: the record axis carries the record count", s)
|
||
|
|
}
|
||
|
|
if got, want := rs.RawInt16s()[:2], []int16{0x1234, -16}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("rs values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
ss := byName["ss"].Values
|
||
|
|
if ss.Dtype() != core.Int16 {
|
||
|
|
t.Fatalf("ss dtype = %s, want int16", ss.Dtype())
|
||
|
|
}
|
||
|
|
if got, want := ss.RawInt16s()[:3], []int16{5, -6, 7}; !slices.Equal(got, want) {
|
||
|
|
t.Fatalf("ss values = %v, want %v", got, want)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFUnknownTypeRefused pins the refusal of every type code
|
||
|
|
// beyond the classic six: the unsigned and 64-bit codes exist only in
|
||
|
|
// formats this module does not speak, and a header carrying one is
|
||
|
|
// refused by name rather than decoded into some other dtype.
|
||
|
|
func TestNetCDFUnknownTypeRefused(t *testing.T) {
|
||
|
|
const template = "" +
|
||
|
|
"43444601" + // magic
|
||
|
|
"00000000" + // numrecs = 0
|
||
|
|
"0000000000000000" + // dim_list ABSENT
|
||
|
|
"0000000000000000" + // gatt_list ABSENT
|
||
|
|
"0000000b" + // NC_VARIABLE
|
||
|
|
"00000001" + // one variable
|
||
|
|
"0000000176000000" + // name "v" + pad
|
||
|
|
"00000000" + // rank 0
|
||
|
|
"0000000000000000" + // vatt ABSENT
|
||
|
|
"00000007" + // type code, replaced per case
|
||
|
|
"00000000" + // vsize
|
||
|
|
"00000000" // begin
|
||
|
|
for _, tc := range []struct{ code, word string }{
|
||
|
|
{"7", "00000007"}, {"8", "00000008"}, {"9", "00000009"},
|
||
|
|
{"10", "0000000a"}, {"11", "0000000b"},
|
||
|
|
} {
|
||
|
|
raw, err := hex.DecodeString(strings.Replace(template, "00000007", tc.word, 1))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("hex: %v", err)
|
||
|
|
}
|
||
|
|
path := ncTempPath(t)
|
||
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||
|
|
t.Fatalf("write: %v", err)
|
||
|
|
}
|
||
|
|
_, _, _, err = LoadNetCDF(path)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "unknown type "+tc.code) {
|
||
|
|
t.Fatalf("type code %s: err = %v, want the unknown-type refusal", tc.code, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestNetCDFDecodeCellsRefused pins the explicit refusal inside
|
||
|
|
// decodeCells itself: an ncType without a case is a named error, never
|
||
|
|
// a buffer left silently untouched.
|
||
|
|
func TestNetCDFDecodeCellsRefused(t *testing.T) {
|
||
|
|
arr := core.New(core.Float, 2)
|
||
|
|
for _, code := range []uint32{0, 7, 8, 9, 10, 11, 12, 99} {
|
||
|
|
err := decodeCells(arr, make([]byte, 16), code, 0, 2)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "unknown type") {
|
||
|
|
t.Fatalf("decodeCells with type %d: err = %v, want the unknown-type refusal", code, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|