diff --git a/CHANGELOG.md b/CHANGELOG.md index 496dd53..9d0646c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (10000 levels, which no hand-written document approaches): a document that nests arrays or inline tables deeper used to run the stack out and is now rejected with a `SyntaxError` naming the limit. +- `OrderedMap`, the string-keyed table that remembers the order its keys were + set in: decoding into one fills it in the order the document wrote the + keys, and `Marshal` writes one back in that order, where a map carries no + order on decode and sorts on encode. It works as a decode target on its + own, in a struct field, and as the element of an array of tables; its + values are untyped, so a nested table stays a `map[string]any`. - The `toml` tag gained the `required` option: a field tagged `toml:"host,required"` makes the decode fail with `missing required key "host"` when the document carries no key that diff --git a/decode.go b/decode.go index 512b9cd..1630c73 100644 --- a/decode.go +++ b/decode.go @@ -18,9 +18,13 @@ import ( // decoder maps a parsed TOML tree onto Go values via reflection. ctx is the // context a cancellable entry point handed in, and reaches an // UnmarshalerContext destination; entry points without one leave it nil. +// nodes is the document's node index, present only when a destination can +// reach an OrderedMap and the parse built the tree its key order is read +// from. type decoder struct { disallowUnknown bool ctx context.Context + nodes nodeIndex } func newDecoder() *decoder { return &decoder{} } @@ -273,6 +277,9 @@ func textUnmarshalerOf(dst reflect.Value) (encoding.TextUnmarshaler, bool) { } func (d *decoder) assignTable(tbl map[string]any, dst reflect.Value) error { + if dst.Type() == orderedMapType { + return d.fillOrderedMap(tbl, dst) + } switch dst.Kind() { case reflect.Struct: return d.assignStruct(tbl, dst) diff --git a/docs/API.md b/docs/API.md index d6fe2be..5ee78ad 100644 --- a/docs/API.md +++ b/docs/API.md @@ -196,6 +196,8 @@ table with `Number`, which keeps the literal; see - `*struct`, matched per the field rules below - `*map[string]any` or `*map[string]T`, keys become map keys and values decode into `T` recursively +- `*OrderedMap`, the keys fill in the order the document wrote them; see + [Ordered tables](#ordered-tables) - `*any`, receives the whole parsed tree unchanged Anything else returns `interpres: decode target must be a non-nil pointer`. @@ -366,6 +368,26 @@ and `from_text = "1h30m"` decode to the same duration. Text that `time.ParseDuration` rejects, `d = "90"` among it, fails with `interpres: invalid duration "90"`. +### Ordered tables + +`OrderedMap` is a string-keyed table that remembers the order its keys were +set in, the shape a `map[string]any` cannot carry. Decoding into one fills it +in the order the document wrote the keys, and `Marshal` writes one back in +that order, where a map destination carries no order and a map source sorts +its keys. The type is a decode target on its own, in a struct field, and as +the element of an array of tables. + +```go +var cfg OrderedMap +err := interpres.Unmarshal(data, &cfg) +out, err := interpres.Marshal(&cfg) // the keys come back in written order +``` + +The values are untyped, the shape the parser produces, so a nested table +inside an `OrderedMap` is a plain `map[string]any`; the order is kept at the +level the `OrderedMap` sits at. Inside a value array an `OrderedMap` renders +as an ordinary inline table, whose keys are sorted. + ### Strict decoding By default unknown keys are dropped silently. A `Decoder` built with @@ -782,6 +804,11 @@ The literal a number was written with, what `UseNumber` decodes into and what `Marshal` writes back as it is. See [Numbers as literals](#numbers-as-literals). +### `type OrderedMap` + +The string-keyed table that keeps its key order on both the encode and the +decode side. See [Ordered tables](#ordered-tables). + ### Date-time wrappers ```go diff --git a/document.go b/document.go index 31e8cb5..d3182c6 100644 --- a/document.go +++ b/document.go @@ -20,8 +20,13 @@ type Document struct { footer []string } -// Root returns the document's root table. -func (d *Document) Root() *Table { return d.root } +// Root returns the document's root table. A nil document has no root. +func (d *Document) Root() *Table { + if d == nil { + return nil + } + return d.root +} // Map returns the value tree, the shape ParseMap gives. It is the tree the // document was parsed into, not a copy. diff --git a/encode.go b/encode.go index 0414664..fa6e82d 100644 --- a/encode.go +++ b/encode.go @@ -184,6 +184,13 @@ func (e *encoder) encode(v any) error { return fmt.Errorf("interpres: cannot marshal a Document; marshal its Map() to write the values") case Document: return fmt.Errorf("interpres: cannot marshal a Document; marshal its Map() to write the values") + case OrderedMap: + return e.encodeOrderedMap(&x) + case *OrderedMap: + if x == nil { + return fmt.Errorf("interpres: cannot marshal nil value") + } + return e.encodeOrderedMap(x) } rv := reflect.ValueOf(v) if !rv.IsValid() { @@ -212,6 +219,63 @@ func (e *encoder) encode(v any) error { return e.emitDoc(doc, nil) } +// encodeOrderedMap emits an OrderedMap as a document, its keys in the order +// they were set, which is the reason the type exists. +func (e *encoder) encodeOrderedMap(om *OrderedMap) error { + if err := e.checkCtx(); err != nil { + return err + } + doc := &tomlDoc{ctx: e.ctx, opts: e.opts} + if err := buildOrderedDoc(om, doc, encPath{}); err != nil { + return err + } + return e.emitDoc(doc, nil) +} + +// buildOrderedDoc adds the entries of om to doc in the order its keys were +// set. Nested OrderedMaps recurse; every other value goes through addField, +// which resolves Marshaler, text and the ordinary kinds the same way it does +// for a struct field. +func buildOrderedDoc(om *OrderedMap, doc *tomlDoc, path encPath) error { + if cap(doc.entries) == 0 { + doc.entries = make([]entry, 0, om.Len()) + } + for _, key := range om.Keys() { + if err := doc.checkCtx(); err != nil { + return err + } + val := om.values[key] + switch x := val.(type) { + case OrderedMap: + sub := &tomlDoc{ctx: doc.ctx, opts: doc.opts} + if err := buildOrderedDoc(&x, sub, path.key(key)); err != nil { + return err + } + doc.addTable(key, sub) + continue + case *OrderedMap: + if x == nil { + continue + } + sub := &tomlDoc{ctx: doc.ctx, opts: doc.opts} + if err := buildOrderedDoc(x, sub, path.key(key)); err != nil { + return err + } + doc.addTable(key, sub) + continue + } + rv := reflect.ValueOf(val) + if !rv.IsValid() { + // A nil value has no TOML form, the rule nil pointer fields follow. + continue + } + if err := addField(doc, key, rv, path); err != nil { + return err + } + } + return nil +} + // --- intermediate representation ----------------------------------------- // entryKind discriminates the three forms an entry in a tomlDoc may take. @@ -518,6 +582,15 @@ func addField(doc *tomlDoc, name string, v reflect.Value, path encPath) error { } v = v.Elem() } + if v.Type() == orderedMapType { + om := v.Interface().(OrderedMap) + sub := &tomlDoc{ctx: doc.ctx, opts: doc.opts} + if err := buildOrderedDoc(&om, sub, path.key(name)); err != nil { + return err + } + doc.addTable(name, sub) + return nil + } switch v.Kind() { case reflect.Struct: if isScalarStruct(v.Type()) { @@ -620,15 +693,20 @@ func addArrayValue(doc *tomlDoc, name string, v reflect.Value, path encPath) err } } sub := &tomlDoc{ctx: doc.ctx, opts: doc.opts} - switch ev.Kind() { - case reflect.Struct: + switch { + case ev.Type() == orderedMapType: + om := ev.Interface().(OrderedMap) + if err := buildOrderedDoc(&om, sub, apath.elem(i)); err != nil { + return err + } + case ev.Kind() == reflect.Struct: if isScalarStruct(ev.Type()) { return &EncodeError{Path: apath.elem(i).String(), Err: errors.New("heterogeneous array contains scalar")} } if err := buildStructDoc(ev, sub, apath.elem(i)); err != nil { return err } - case reflect.Map: + case ev.Kind() == reflect.Map: if err := buildMapDoc(ev, sub, apath.elem(i)); err != nil { return err } @@ -717,6 +795,9 @@ func resolveElement(v reflect.Value, path encPath) (reflect.Value, error) { // nested-array representations the emitter understands. Slices and arrays are // recursively normalised so that nested arrays (e.g. [][]int) work. func normaliseValue(v reflect.Value) (any, error) { + if !v.IsValid() { + return nil, fmt.Errorf("cannot encode nil value") + } // Map and slice elements arrive wrapped in interface{}; look through them. for v.Kind() == reflect.Interface && !v.IsNil() { v = v.Elem() @@ -755,6 +836,20 @@ func normaliseValue(v reflect.Value) (any, error) { if v.Type() == numberType { return Number(v.String()), nil } + // An OrderedMap in a value array has no header form, so it renders as an + // ordinary inline table, whose keys come out sorted. + if v.Type() == orderedMapType { + om := v.Interface().(OrderedMap) + out := make(map[string]any, om.Len()) + for _, k := range om.Keys() { + val, err := normaliseValue(reflect.ValueOf(om.values[k])) + if err != nil { + return nil, fmt.Errorf("[%s]: %w", k, err) + } + out[k] = val + } + return out, nil + } // A type that renders itself as text becomes a TOML string, scalar kinds // and structs alike. s, isText, err := textValue(v) diff --git a/interpres.go b/interpres.go index 7db463d..def72b9 100644 --- a/interpres.go +++ b/interpres.go @@ -26,6 +26,7 @@ import ( "errors" "fmt" "os" + "reflect" "slices" "strings" ) @@ -237,12 +238,15 @@ func Unmarshal(data []byte, v any) error { // UnmarshalContext is the cancellable variant of Unmarshal. func UnmarshalContext(ctx context.Context, data []byte, v any) error { - tree, err := ParseMapContext(ctx, data) + // Only a destination that can reach an OrderedMap needs the node tree the + // written key order is read from; every other decode skips building it. + tree, doc, err := parseWithOptions(ctx, data, parseOptions{}, typeWantsOrder(reflect.TypeOf(v))) if err != nil { return err } dec := newDecoder() dec.ctx = ctx + dec.nodes = indexNodes(doc.Root()) return dec.decode(tree, v) } @@ -307,17 +311,19 @@ func (d *Decoder) Decode(data []byte, v any) error { // DecodeContext is the cancellable variant of Decode. func (d *Decoder) DecodeContext(ctx context.Context, data []byte, v any) error { - tree, _, err := parseWithOptions(ctx, data, parseOptions{ + opts := parseOptions{ maxDepth: d.maxDepth, maxInputSize: d.maxInputSize, useNumber: d.useNumber, - }, false) + } + tree, doc, err := parseWithOptions(ctx, data, opts, typeWantsOrder(reflect.TypeOf(v))) if err != nil { return err } dec := newDecoder() dec.disallowUnknown = d.disallowUnknown dec.ctx = ctx + dec.nodes = indexNodes(doc.Root()) return dec.decode(tree, v) } diff --git a/orderedmap.go b/orderedmap.go new file mode 100644 index 0000000..6be2a59 --- /dev/null +++ b/orderedmap.go @@ -0,0 +1,199 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: MIT + +package interpres + +import ( + "fmt" + "maps" + "reflect" + "slices" + "sync" +) + +// An OrderedMap is a string-keyed table that remembers the order its keys +// were set in, the shape a map[string]any cannot carry. Marshal writes a +// table of its own kind in that order, and decoding a document into one +// fills it in the order the document wrote the keys, where a map +// destination carries no order at all. The values are untyped, the shape +// the parser produces, so a nested table inside an OrderedMap is a plain +// map[string]any; the order is kept at the level the OrderedMap sits at. +// +// The zero value is an empty table ready for use. +type OrderedMap struct { + keys []string + values map[string]any +} + +var orderedMapType = reflect.TypeFor[OrderedMap]() + +// NewOrderedMap returns an empty OrderedMap. +func NewOrderedMap() *OrderedMap { return &OrderedMap{} } + +// Set stores value under key. A key the table already has keeps its position +// and takes the new value; a new one joins the end. +func (m *OrderedMap) Set(key string, value any) { + if m.values == nil { + m.values = make(map[string]any, 4) + } + if _, ok := m.values[key]; !ok { + m.keys = append(m.keys, key) + } + m.values[key] = value +} + +// Get returns the value under key, and whether the table has one. +func (m *OrderedMap) Get(key string) (any, bool) { + v, ok := m.values[key] + return v, ok +} + +// Delete removes key. A later Set of the same key appends it to the end +// again. +func (m *OrderedMap) Delete(key string) { + if _, ok := m.values[key]; !ok { + return + } + delete(m.values, key) + m.keys = slices.DeleteFunc(m.keys, func(k string) bool { return k == key }) +} + +// Keys returns the keys in the order they were set. +func (m *OrderedMap) Keys() []string { return m.keys } + +// Len returns the number of keys. +func (m *OrderedMap) Len() int { return len(m.keys) } + +// Range calls f for every key in order, stopping when f returns false. +func (m *OrderedMap) Range(f func(key string, value any) bool) { + for _, k := range m.keys { + if !f(k, m.values[k]) { + return + } + } +} + +// Map returns the values as a plain map, which carries no order. It is the +// view Marshal's Document-free callers need. +func (m *OrderedMap) Map() map[string]any { return m.values } + +// --- decode: the order the document wrote ---------------------------------- + +// wantsOrderCache holds whether a destination type mentions OrderedMap +// anywhere a decode can reach. One computed answer per type, the same +// trade-off structSchemaCache makes. +var wantsOrderCache sync.Map // reflect.Type -> bool + +// typeWantsOrder reports whether decoding into t can reach an OrderedMap, in +// which case the parse has to build the node tree the key order is read +// from. Structs walk their exported fields, and pointers, slices, arrays and +// maps walk their element; anything else holds no OrderedMap. +func typeWantsOrder(t reflect.Type) bool { + if t == nil { + return false + } + if v, ok := wantsOrderCache.Load(t); ok { + return v.(bool) + } + r := scanWantsOrder(t, make(map[reflect.Type]bool)) + v, _ := wantsOrderCache.LoadOrStore(t, r) + return v.(bool) +} + +func scanWantsOrder(t reflect.Type, seen map[reflect.Type]bool) bool { + for { + if t == orderedMapType { + return true + } + if seen[t] { + return false + } + seen[t] = true + switch t.Kind() { + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map: + t = t.Elem() + case reflect.Struct: + for f := range t.Fields() { + if f.PkgPath != "" { + continue + } + if scanWantsOrder(f.Type, seen) { + return true + } + } + return false + default: + return false + } + } +} + +// nodes maps a table's value map to its node, the index the decoder reads +// the written key order from. The key is the map header's runtime pointer, +// the one identity a map value offers; the nodes share their maps with the +// value tree, so one lookup per table is exact. +type nodeIndex map[uintptr]*Table + +// indexNodeIndex walks a document's node tree into an index. A nil tree +// gives a nil index, which every lookup answers with nil. +func indexNodes(t *Table) nodeIndex { + if t == nil { + return nil + } + idx := nodeIndex{} + var walk func(t *Table) + walk = func(t *Table) { + idx[reflect.ValueOf(t.values).Pointer()] = t + for _, e := range t.entries { + if e.child != nil { + walk(e.child) + } + // The elements of a value array carry a node only where an element + // is an inline table; the rest are nil. + for _, el := range e.elements { + if el != nil { + walk(el) + } + } + } + } + walk(t) + return idx +} + +// nodeOf returns the node a value table was parsed into, or nil when the +// parse built no node tree, which is the ordinary decode's shape. A tree +// built by hand carries no nodes either. +func (d *decoder) nodeOf(tbl map[string]any) *Table { + return d.nodes[reflect.ValueOf(tbl).Pointer()] +} + +// fillOrderedMap decodes a parsed table into an OrderedMap destination, +// taking the keys in the order the document wrote them. A table with no +// node, which is what a hand-built tree or a ParseMap result offers, fills +// in sorted key order, the deterministic order a map can offer. +func (d *decoder) fillOrderedMap(tbl map[string]any, dst reflect.Value) error { + if !dst.CanAddr() { + return fmt.Errorf("interpres: cannot decode into an OrderedMap that is not addressable") + } + om := dst.Addr().Interface().(*OrderedMap) + if om.values == nil { + om.values = make(map[string]any, len(tbl)) + } + keys := slices.Sorted(maps.Keys(tbl)) + if node := d.nodeOf(tbl); node != nil { + keys = node.Keys() + } + for _, key := range keys { + val, ok := tbl[key] + if !ok { + continue + } + elem := reflect.New(reflect.TypeFor[any]()).Elem() + if err := d.assign(val, elem); err != nil { + return newDecodeError(key, err) + } + om.Set(key, elem.Interface()) + } + return nil +} diff --git a/orderedmap_test.go b/orderedmap_test.go new file mode 100644 index 0000000..e6e4ae9 --- /dev/null +++ b/orderedmap_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: MIT + +package interpres + +import ( + "context" + "slices" + "testing" +) + +func TestOrderedMapBasics(t *testing.T) { + m := NewOrderedMap() + if m.Len() != 0 { + t.Fatalf("fresh map holds %d keys", m.Len()) + } + m.Set("b", 1) + m.Set("a", 2) + m.Set("c", 3) + if got := m.Keys(); !slices.Equal(got, []string{"b", "a", "c"}) { + t.Errorf("keys = %v, want [b a c]", got) + } + if v, ok := m.Get("a"); !ok || v != 2 { + t.Errorf("a = %v, %v", v, ok) + } + m.Set("a", 9) + if got := m.Keys(); !slices.Equal(got, []string{"b", "a", "c"}) { + t.Errorf("keys after replace = %v, want the position kept", got) + } + if v, _ := m.Get("a"); v != 9 { + t.Errorf("a = %v, want 9", v) + } + seen := "" + m.Range(func(key string, value any) bool { + seen += key + return key != "a" + }) + if seen != "ba" { + t.Errorf("range visited %q, want \"ba\"", seen) + } + m.Delete("b") + m.Delete("missing") + if got := m.Keys(); !slices.Equal(got, []string{"a", "c"}) { + t.Errorf("keys after delete = %v, want [a c]", got) + } + m.Delete("c") + m.Set("c", 3) + if got := m.Keys(); !slices.Equal(got, []string{"a", "c"}) { + t.Errorf("re-set key = %v, want it appended as [a c]", got) + } +} + +func TestMarshalOrderedMap(t *testing.T) { + t.Run("top level keeps the order", func(t *testing.T) { + m := NewOrderedMap() + m.Set("zebra", int64(1)) + m.Set("alpha", "x") + out, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + want := "zebra = 1\nalpha = \"x\"\n" + if string(out) != want { + t.Errorf("output:\n%q\nwant:\n%q", out, want) + } + }) + t.Run("a pointer top level does the same", func(t *testing.T) { + m := &OrderedMap{} + m.Set("second", true) + m.Set("first", int64(2)) + out, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + if string(out) != "second = true\nfirst = 2\n" { + t.Errorf("output %q", out) + } + }) + t.Run("a struct field keeps the order as a table", func(t *testing.T) { + type Cfg struct { + Title string `toml:"title"` + Extra *OrderedMap `toml:"extra"` + } + m := &OrderedMap{} + m.Set("late", int64(1)) + m.Set("early", int64(2)) + out, err := Marshal(Cfg{Title: "t", Extra: m}) + if err != nil { + t.Fatal(err) + } + want := "title = \"t\"\n\n[extra]\nlate = 1\nearly = 2\n" + if string(out) != want { + t.Errorf("output:\n%q\nwant:\n%q", out, want) + } + }) + t.Run("inline form keeps the order too", func(t *testing.T) { + m := NewOrderedMap() + m.Set("zebra", int64(1)) + m.Set("alpha", int64(2)) + out, err := NewEncoder().InlineTables(60).Marshal(map[string]any{"t": m}) + if err != nil { + t.Fatal(err) + } + if string(out) != "t = {zebra = 1, alpha = 2}\n" { + t.Errorf("output %q", out) + } + }) + t.Run("an array of tables keeps each element's order", func(t *testing.T) { + type Cfg struct { + Items []*OrderedMap `toml:"items"` + } + a, b := NewOrderedMap(), NewOrderedMap() + a.Set("y", int64(1)) + a.Set("x", int64(2)) + b.Set("n", int64(3)) + out, err := Marshal(Cfg{Items: []*OrderedMap{a, b}}) + if err != nil { + t.Fatal(err) + } + want := "[[items]]\ny = 1\nx = 2\n\n[[items]]\nn = 3\n" + if string(out) != want { + t.Errorf("output:\n%q\nwant:\n%q", out, want) + } + }) + t.Run("a nil value is skipped", func(t *testing.T) { + m := NewOrderedMap() + m.Set("gone", nil) + m.Set("here", int64(1)) + out, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + if string(out) != "here = 1\n" { + t.Errorf("output %q", out) + } + }) +} + +func TestDecodeOrderedMap(t *testing.T) { + t.Run("keys come back in written order", func(t *testing.T) { + doc := []byte("zebra = 1\nmiddle = \"m\"\nalpha = true\n") + var m OrderedMap + if err := Unmarshal(doc, &m); err != nil { + t.Fatal(err) + } + if got := m.Keys(); !slices.Equal(got, []string{"zebra", "middle", "alpha"}) { + t.Fatalf("keys = %v", got) + } + if v, _ := m.Get("middle"); v != "m" { + t.Errorf("middle = %#v", v) + } + }) + t.Run("a nested table keeps the table order", func(t *testing.T) { + type Cfg struct { + Ports []int `toml:"ports"` + DB *OrderedMap `toml:"db"` + } + doc := []byte("ports = [1, 2]\n\n[db]\nslow = 1\nfast = 2\n") + var cfg Cfg + if err := Unmarshal(doc, &cfg); err != nil { + t.Fatal(err) + } + if got := cfg.DB.Keys(); !slices.Equal(got, []string{"slow", "fast"}) { + t.Errorf("db keys = %v", got) + } + }) + t.Run("an array of tables fills in order", func(t *testing.T) { + var m OrderedMap + doc := []byte("b = 1\n[[items]]\nname = \"x\"\n[[items]]\nname = \"y\"\na = 2\n") + if err := Unmarshal(doc, &m); err != nil { + t.Fatal(err) + } + if got := m.Keys(); !slices.Equal(got, []string{"b", "items"}) { + t.Errorf("keys = %v, want [b items]", got) + } + elems, ok := m.values["items"].([]map[string]any) + if !ok || len(elems) != 2 { + t.Fatalf("items = %#v", m.values["items"]) + } + if elems[1]["name"] != "y" { + t.Errorf("second element = %#v", elems[1]) + } + }) + t.Run("the sorted fallback needs a tree without nodes", func(t *testing.T) { + // Unmarshal and Decode build the node tree whenever the destination can + // reach an OrderedMap, so the sorted fallback is only reachable from a + // tree that never had one. + tree, _, err := parseWithOptions(context.Background(), []byte("b = 1\na = 2\n"), parseOptions{}, false) + if err != nil { + t.Fatal(err) + } + var m OrderedMap + if err := newDecoder().decode(tree, &m); err != nil { + t.Fatal(err) + } + if got := m.Keys(); !slices.Equal(got, []string{"a", "b"}) { + t.Errorf("keys = %v, want the sorted [a b]", got) + } + }) + t.Run("the order survives a round trip", func(t *testing.T) { + doc := []byte("z = 1\na = 2\nm = 3\n") + var m OrderedMap + if err := Unmarshal(doc, &m); err != nil { + t.Fatal(err) + } + out, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + if string(out) != "z = 1\na = 2\nm = 3\n" { + t.Errorf("output:\n%q", out) + } + }) +}