diff --git a/CHANGELOG.md b/CHANGELOG.md index 206a15c..904bdc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 at most `threshold` bytes is written as an inline table instead of a header section, which shortens a document of small tables. An array of tables keeps its header form, because its inline form would re-parse as a value array. +- `Document` and `ParseMap`: `Parse` now returns a `*Document`, which holds the + values together with the key order, whether a table was written as an inline + table or under a header, and the comments, with `Keys`, `Entries`, `Get`, + `Comments` and `SetComments` to read and write them. `ParseMap` returns the + plain `map[string]any` tree, the shape `Parse` used to give. `Marshal` does + not accept a `Document`; it writes values, so `doc.Map()` is the way through. - `OffsetDateTime`, the Go type of the offset date-time kind, so that all four TOML date-time kinds have one of their own. `Parse` and `Unmarshal` hand it back where they produced a bare `time.Time` before, and `Marshal` accepts it. diff --git a/README.md b/README.md index 67cce0c..d4a2818 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ the entire official [toml-test](https://github.com/toml-lang/toml-test) suite: user types with text methods need no configuration. - **Cancellation**: every entry point has a `*Context` sibling that honours a `context.Context`. +- **Ordered documents**: `Parse` gives a `*Document` that keeps the key order, +tells an inline table from a header one, and carries the comments; `ParseMap` +gives the plain `map[string]any` tree. - **Configurable emission**: `Encoder` options for declaration-order output, omitting empty arrays, literal multiline strings, and inlining small sub-tables. @@ -67,8 +70,9 @@ err := interpres.Unmarshal(data, &cfg) ``` Fields match by the `toml:"name"` tag, or by the lower-cased field name when no -tag is present; `toml:"-"` skips a field. `Parse` returns the untyped -`map[string]any` tree instead, and `UnmarshalContext` accepts a context. +tag is present; `toml:"-"` skips a field. `Parse` returns a `*Document` that +also carries the key order and the comments, `ParseMap` returns the plain +`map[string]any` tree, and `UnmarshalContext` accepts a context. ### Encode from a struct diff --git a/bench_test.go b/bench_test.go index 0504c37..804004f 100644 --- a/bench_test.go +++ b/bench_test.go @@ -87,14 +87,14 @@ func BenchmarkParse(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(len(benchDoc))) for b.Loop() { - if _, err := Parse(benchDoc); err != nil { + if _, err := ParseMap(benchDoc); err != nil { b.Fatal(err) } } } func BenchmarkMarshal(b *testing.B) { - tree, err := Parse(benchDoc) + tree, err := ParseMap(benchDoc) if err != nil { b.Fatal(err) } @@ -122,7 +122,7 @@ func BenchmarkParseLong(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(len(longDoc))) for b.Loop() { - if _, err := Parse(longDoc); err != nil { + if _, err := ParseMap(longDoc); err != nil { b.Fatal(err) } } diff --git a/cmd/interpres-decode/main.go b/cmd/interpres-decode/main.go index e938458..3a2e882 100644 --- a/cmd/interpres-decode/main.go +++ b/cmd/interpres-decode/main.go @@ -67,7 +67,7 @@ func Run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "read stdin:", err) return 2 } - tree, err := interpres.Parse(data) + tree, err := interpres.ParseMap(data) if err != nil { fmt.Fprintln(stderr, err) return 1 @@ -108,7 +108,7 @@ func validatePaths(paths []string, stdin io.Reader, stderr io.Writer) int { fmt.Fprintf(stderr, "interpres-decode: %s: %v\n", name, err) return 2 } - if _, err := interpres.Parse(data); err != nil { + if _, err := interpres.ParseMap(data); err != nil { fmt.Fprintf(stderr, "%s: %v\n", name, err) valid = false } @@ -262,7 +262,7 @@ func decodeTagged(typ, val string) (any, error) { // parser and requiring the result to hold exactly that one statement, so a // value carrying a newline or a comment cannot smuggle a second one in. func parseAtom(val string) (any, error) { - tree, err := interpres.Parse([]byte("v = " + val + "\n")) + tree, err := interpres.ParseMap([]byte("v = " + val + "\n")) if err != nil { return nil, err } diff --git a/cmd/interpres-decode/main_test.go b/cmd/interpres-decode/main_test.go index 1cc2a83..16e2502 100644 --- a/cmd/interpres-decode/main_test.go +++ b/cmd/interpres-decode/main_test.go @@ -434,11 +434,11 @@ n = "a" if code := Run([]string{"-encode"}, bytes.NewReader(tagged.Bytes()), &out, &stderr); code != 0 { t.Fatalf("encode returned %d, stderr = %q", code, stderr.String()) } - want, err := interpres.Parse([]byte(doc)) + want, err := interpres.ParseMap([]byte(doc)) if err != nil { t.Fatalf("parse of the original: %v", err) } - got, err := interpres.Parse(out.Bytes()) + got, err := interpres.ParseMap(out.Bytes()) if err != nil { t.Fatalf("parse of the encoder output (%q): %v", out.String(), err) } diff --git a/decode_test.go b/decode_test.go index c90dd8c..7934efd 100644 --- a/decode_test.go +++ b/decode_test.go @@ -24,7 +24,7 @@ func TestSyntaxErrorMessage(t *testing.T) { } func TestParseRejectsInvalidUTF8(t *testing.T) { - _, err := Parse([]byte("v = \"\xff\"\n")) + _, err := ParseMap([]byte("v = \"\xff\"\n")) if err == nil { t.Fatal("expected a UTF-8 validation error") } @@ -130,7 +130,7 @@ func TestUnmarshalIntoNilAny(t *testing.T) { func TestParseContextHonoursCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := ParseContext(ctx, []byte("a = 1\n")); !errors.Is(err, context.Canceled) { + if _, err := ParseMapContext(ctx, []byte("a = 1\n")); !errors.Is(err, context.Canceled) { t.Fatalf("ParseContext returned %v, want context.Canceled", err) } } @@ -160,7 +160,7 @@ func TestContextRoundTrip(t *testing.T) { in := []byte(`title = "x" count = 3 `) - if _, err := ParseContext(context.Background(), in); err != nil { + if _, err := ParseMapContext(context.Background(), in); err != nil { t.Fatalf("ParseContext: %v", err) } var out struct { @@ -987,7 +987,7 @@ func TestDecoderMaxInputSize(t *testing.T) { t.Errorf("err = %v, want it to name the limit", err) } // Parse carries the nesting default and no size limit. - if _, err := Parse(doc); err != nil { + if _, err := ParseMap(doc); err != nil { t.Fatalf("parse: %v", err) } } diff --git a/docs/API.md b/docs/API.md index 14c05ff..f3c8f8c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -13,23 +13,95 @@ and trailing commas. The encoder emits TOML 1.1. ## Functions -### `func Parse(data []byte) (map[string]any, error)` +### `func Parse(data []byte) (*Document, error)` -Decodes a TOML document into an untyped tree, using the value mapping in the -[Decoding](#decoding) section below. Returns `*SyntaxError` on a malformed -document. Input that is not valid UTF-8 is rejected before the parser runs. -Equivalent to `ParseContext(context.Background(), data)`. +Decodes a TOML document into a [Document](#documents): the values, the order the +keys were written in, whether a table was written inline, and the comments. +The values follow the mapping in the [Decoding](#decoding) section below. +Returns `*SyntaxError` on a malformed document. Input that is not valid UTF-8 +is rejected before the parser runs. Equivalent to +`ParseContext(context.Background(), data)`. ```go -tree, err := interpres.Parse([]byte("title = \"x\"\nport = 8080\n")) +doc, err := interpres.Parse([]byte("title = \"x\"\nport = 8080\n")) +tree := doc.Map() ``` -### `func ParseContext(ctx context.Context, data []byte) (map[string]any, error)` +### `func ParseContext(ctx context.Context, data []byte) (*Document, error)` The cancellable variant of `Parse`. An already-cancelled context returns `ctx.Err()` before any work. During parsing the context is checked every 64 top-level statements, so a long document aborts without running to completion. +### `func ParseMap(data []byte) (map[string]any, error)` + +Decodes a TOML document into an untyped tree, the shape this package parsed +into before [Document](#documents) existed: the order of the keys and the +comments are not part of a map, so they are dropped. Use it when only the +values matter, or when the extra bookkeeping of a document is not wanted. +Equivalent to `ParseMapContext(context.Background(), data)`. + +```go +tree, err := interpres.ParseMap([]byte("title = \"x\"\nport = 8080\n")) +``` + +### `func ParseMapContext(ctx context.Context, data []byte) (map[string]any, error)` + +The cancellable variant of `ParseMap`. + +## Documents + +`Parse` returns a `Document`: the value tree together with what a map cannot +carry, which is the order the keys were written in, whether a table was written +as an inline table or under a header, and the comments. `ParseMap` gives the +plain tree when none of that is wanted. + +```go +doc, err := interpres.Parse(data) +if err != nil { + return err +} +root := doc.Root() +for _, key := range root.Keys() { // written order, not sorted + entry, _ := root.Get(key) + fmt.Println(key, entry.Value()) +} +``` + +The values are shared with the tree `ParseMap` returns, so a value read from a +document and from `doc.Map()` is the same value. + +| Type | Meaning | +|---|---| +| `Document` | the parsed document: `Root()` for the top-level table, `Map()` for the value tree, `Footer()` for a comment block at the end | +| `Table` | one TOML table: `Keys()` and `Entries()` in written order, `Get(key)`, `Values()` for its part of the value tree, `Inline()` | +| `Entry` | one key: `Value()`, `Inline()`, `Table()` when the value is a table, `Elements()` for the tables of an array value | + +`Elements()` holds one node per element of an array value: the tables of an +array of tables, and the inline tables inside a value array, with `nil` for the +elements that are not tables. + +### Comments + +A comment belongs to the line it precedes or follows, and to the node that line +introduced: + +| Written | Carried by | +|---|---| +| lines above a key | that key's `Entry`, through `Comments()` | +| a comment beside a key | that key's `Entry`, through `Trailing()` | +| lines above a `[header]` or `[[header]]` | that `Table`, through `Comments()` | +| a comment beside a header | that `Table`, through `Trailing()` | +| a comment block after the last statement | the `Document`, through `Footer()` | + +`SetComments` and `SetTrailing` replace them. A line carries no leading `#` +and no surrounding space, so `# note` is stored as `note` and a bare `#` as +`""`. + +A `Document` is not a value to marshal: `Marshal` writes values, so it refuses +one and points at `doc.Map()`. Writing a document back, with its order and its +comments, belongs with the editing API. + ### `func Unmarshal(data []byte, v any) error` Parses `data` and stores the result in the value pointed to by `v`, typically a @@ -602,6 +674,11 @@ A configured `Encoder` holds no per-call state; each `Marshal` or `MarshalContext` call copies the options and is safe for concurrent use, as long as no setter races with a call. +### `type Document`, `type Table`, `type Entry` + +See [Documents](#documents). A `Document` is what `Parse` returns, and it is +not a value `Marshal` accepts. + ### `type Marshaler interface{ MarshalTOML() (any, error) }` See [Custom encoding](#custom-encoding-marshaler). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 855cc22..a0d3d37 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -43,7 +43,8 @@ Inside the library package, one file owns one concern: | File | Responsibility | |---|---| -| `parser.go` | The recursive-descent parser. Produces the `map[string]any` tree and enforces the structural rules of TOML 1.1 (table redefinitions, dotted keys, arrays of tables, multi-line inline tables). Reports a 1-based line on failure. | +| `parser.go` | The recursive-descent parser. Produces the `map[string]any` tree, records the nodes a [Document](API.md#documents) is built from, and enforces the structural rules of TOML 1.1 (table redefinitions, dotted keys, arrays of tables, multi-line inline tables). Reports a 1-based line on failure. | +| `document.go` | The parsed-document types: `Document`, `Table` and `Entry`, which carry the key order, whether a table was written inline, and the comments. The values they expose are the parser's own tree, not a copy. | | `number.go` | Strict numeric tokens: integers in the four radixes with `_` separators, and floats including `inf` and `nan`. Rejects leading zeros, misplaced underscores and malformed fractions. | | `datetime.go` | The three local date-time wrapper types and `parseDateTime`, which classifies a token into the four date-time kinds under the strict TOML grammar. | | `decode.go` | Maps the parsed tree onto Go values by reflection: struct fields, maps, slices, scalar conversion with overflow checks, `Unmarshaler` dispatch. | diff --git a/document.go b/document.go new file mode 100644 index 0000000..31e8cb5 --- /dev/null +++ b/document.go @@ -0,0 +1,196 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: MIT + +package interpres + +// A Document is a parsed TOML document: the values, plus what the map shape +// cannot carry, which is the order the keys were written in, whether a table +// was written inline or under a header, and the comments. +// +// The values are the tree ParseMap returns, shared rather than copied, so a +// value read from a Document and from that map is the same value. A comment +// belongs to the statement it precedes: the lines above a key belong to the +// key, the lines above a header belong to the header's table, and a comment +// block after the last statement belongs to the Document. +// +// Comments inside array and inline-table values are not carried yet; the +// parser skips them as it always has. +type Document struct { + root *Table + footer []string +} + +// Root returns the document's root table. +func (d *Document) Root() *Table { return d.root } + +// Map returns the value tree, the shape ParseMap gives. It is the tree the +// document was parsed into, not a copy. +func (d *Document) Map() map[string]any { return d.root.values } + +// Footer returns the comment lines that follow the last statement, and every +// line of a document that holds no statement at all. +func (d *Document) Footer() []string { return d.footer } + +// SetFooter replaces those lines. +func (d *Document) SetFooter(lines []string) { d.footer = lines } + +// A Table is one TOML table: its values, its keys in written order, and the +// comments around the header or the key that introduced it. +type Table struct { + values map[string]any + entries []*Entry + index map[string]*Entry + + // inline records that the table was written as an inline table, `{…}`, + // rather than under a header or as a dotted key. + inline bool + + // comments are the lines above the table's header, trailing is the comment + // on the header's own line. Both are empty for a table a dotted key + // introduced, which has no line of its own. + comments []string + trailing string +} + +func newTable(values map[string]any) *Table { + return &Table{values: values, index: map[string]*Entry{}} +} + +// Keys returns the table's keys in the order they were written. +func (t *Table) Keys() []string { + keys := make([]string, len(t.entries)) + for i, e := range t.entries { + keys[i] = e.key + } + return keys +} + +// Values returns the table's values, which is the map the value tree holds for +// it. +func (t *Table) Values() map[string]any { return t.values } + +// Entries returns the table's entries in written order. +func (t *Table) Entries() []*Entry { return t.entries } + +// Get returns the entry for key, and whether the table has one. +func (t *Table) Get(key string) (*Entry, bool) { + e, ok := t.index[key] + return e, ok +} + +// Inline reports whether the table was written as an inline table, `{…}`, +// rather than under a header or introduced by a dotted key. +func (t *Table) Inline() bool { return t.inline } + +// Comments returns the comment lines above the table's header, or above the +// key that introduced it. Lines carry no leading '#' and no surrounding space. +func (t *Table) Comments() []string { return t.comments } + +// SetComments replaces those lines. Each line is written back with a "# " in +// front of it, so a line should not carry one. +func (t *Table) SetComments(lines []string) { t.comments = lines } + +// Trailing returns the comment on the header's own line, without the '#'. +func (t *Table) Trailing() string { return t.trailing } + +// SetTrailing replaces that comment. +func (t *Table) SetTrailing(line string) { t.trailing = line } + +// addValue records a key of the table, in written order. +func (t *Table) addValue(key string, val any, inline bool) *Entry { + e := &Entry{table: t, key: key, inline: inline} + t.entries = append(t.entries, e) + t.index[key] = e + if node, ok := val.(map[string]any); ok { + e.child = newTable(node) + } + return e +} + +// addTable records a key whose value is a table introduced by a header or a +// dotted key, and returns the table's node. +func (t *Table) addTable(key string, values map[string]any) *Table { + if e, ok := t.index[key]; ok { + // The key was seen before, as the leaf of an earlier dotted key. + if e.child == nil { + e.child = newTable(values) + } + return e.child + } + e := t.addValue(key, values, false) + e.child = newTable(values) + return e.child +} + +// addElement records one element of an array of tables, and returns its node. +func (t *Table) addElement(key string, values map[string]any) *Table { + e, ok := t.index[key] + if !ok { + e = t.addValue(key, nil, false) + e.elements = []*Table{} + } + el := newTable(values) + e.elements = append(e.elements, el) + return el +} + +// child returns the node of a table-valued key, or nil. +func (t *Table) child(key string) *Table { + if e, ok := t.index[key]; ok { + return e.child + } + return nil +} + +// lastElement returns the node of the newest element of an array of tables. +func (t *Table) lastElement(key string) *Table { + if e, ok := t.index[key]; ok && len(e.elements) > 0 { + return e.elements[len(e.elements)-1] + } + return nil +} + +// An Entry is one key of a table: the value and the comments around the key. +type Entry struct { + table *Table + key string + inline bool + comments []string + trailing string + + // child is the table the value is, and elements are the tables of an array + // of tables; one of them is set only when the value has that shape. + child *Table + elements []*Table +} + +// Key returns the key as it was written. +func (e *Entry) Key() string { return e.key } + +// Value returns the value the key holds. It is read from the table's map, so +// it stays current if that map is changed. +func (e *Entry) Value() any { return e.table.values[e.key] } + +// Inline reports whether the value was written as an inline table, `{…}`. +func (e *Entry) Inline() bool { return e.inline } + +// Table returns the table the value is, or nil when it is not a table. +func (e *Entry) Table() *Table { return e.child } + +// Elements returns the tables of an array of tables, or nil when the value is +// not one. +func (e *Entry) Elements() []*Table { return e.elements } + +// Comments returns the comment lines above the key. Lines carry no leading '#' +// and no surrounding space. +func (e *Entry) Comments() []string { return e.comments } + +// SetComments replaces those lines. Each line is written back with a "# " in +// front of it, so a line should not carry one. +func (e *Entry) SetComments(lines []string) { e.comments = lines } + +// Trailing returns the comment on the key's own line, without the '#'. +func (e *Entry) Trailing() string { return e.trailing } + +// SetTrailing replaces that comment. +func (e *Entry) SetTrailing(line string) { e.trailing = line } diff --git a/document_test.go b/document_test.go new file mode 100644 index 0000000..7c463ef --- /dev/null +++ b/document_test.go @@ -0,0 +1,314 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: MIT + +package interpres + +import ( + "slices" + "strings" + "testing" +) + +// mustEntry returns the entry a table must have, and fails the test when it +// does not. +func mustEntry(t *testing.T, tbl *Table, key string) *Entry { + t.Helper() + e, ok := tbl.Get(key) + if !ok { + t.Fatalf("%q is missing from the table", key) + } + return e +} + +func TestDocumentKeepsKeyOrder(t *testing.T) { + doc, err := Parse([]byte(` +b = 1 +a = 2 +inline = {y = 1, x = 2} + +[table] +z = 3 +m = 4 +`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + + // The root's keys come back in written order, not sorted. + if got := doc.Root().Keys(); !slices.Equal(got, []string{"b", "a", "inline", "table"}) { + t.Errorf("root keys = %v, want [b a inline table]", got) + } + + // So do the keys of an inline table, which the map shape loses. + inline, ok := doc.Root().Get("inline") + if !ok { + t.Fatal("inline is missing from the root") + } + if !inline.Inline() { + t.Error("inline is not marked inline") + } + if got := inline.Table().Keys(); !slices.Equal(got, []string{"y", "x"}) { + t.Errorf("inline keys = %v, want [y x]", got) + } + + // And the keys of a table written under a header, which is not inline. + tbl, ok := doc.Root().Get("table") + if !ok { + t.Fatal("table is missing from the root") + } + if tbl.Inline() { + t.Error("table is marked inline") + } + if got := tbl.Table().Keys(); !slices.Equal(got, []string{"z", "m"}) { + t.Errorf("table keys = %v, want [z m]", got) + } +} + +func TestDocumentValuesAreTheTree(t *testing.T) { + doc, err := Parse([]byte("n = 7\ns = \"x\"\n\n[t]\nk = true\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := mustEntry(t, doc.Root(), "n").Value(); got != int64(7) { + t.Errorf("n = %#v, want int64(7)", got) + } + tbl := mustEntry(t, doc.Root(), "t").Table() + if got := mustEntry(t, tbl, "k").Value(); got != true { + t.Errorf("t.k = %#v, want true", got) + } + // Map gives the tree ParseMap would have returned, the same values. + tree := doc.Map() + if tree["n"] != int64(7) || tree["s"] != "x" { + t.Errorf("Map = %#v", tree) + } + if tree["t"].(map[string]any)["k"] != true { + t.Errorf("Map[t] = %#v", tree["t"]) + } + if tbl.Values()["k"] != true { + t.Errorf("t.Values() = %#v", tbl.Values()) + } +} + +func TestDocumentComments(t *testing.T) { + doc, err := Parse([]byte(`# above b +b = 1 # trailing b + +# above the table +[table] # trailing table +# above m +m = 2 + +# footer +`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + + b, ok := doc.Root().Get("b") + if !ok { + t.Fatal("b is missing") + } + if got := b.Comments(); !slices.Equal(got, []string{"above b"}) { + t.Errorf("b comments = %q, want [above b]", got) + } + if got := b.Trailing(); got != "trailing b" { + t.Errorf("b trailing = %q, want \"trailing b\"", got) + } + + tbl, ok := doc.Root().Get("table") + if !ok { + t.Fatal("table is missing") + } + // A [header] line introduces the table, so the comments around it belong + // to the table node; the entry that names it stays bare. + if got := tbl.Table().Comments(); !slices.Equal(got, []string{"above the table"}) { + t.Errorf("table comments = %q, want [above the table]", got) + } + if got := tbl.Table().Trailing(); got != "trailing table" { + t.Errorf("table trailing = %q, want \"trailing table\"", got) + } + if got := tbl.Comments(); got != nil { + t.Errorf("entry comments = %q, want none", got) + } + m, ok := tbl.Table().Get("m") + if !ok { + t.Fatal("table.m is missing") + } + if got := m.Comments(); !slices.Equal(got, []string{"above m"}) { + t.Errorf("m comments = %q, want [above m]", got) + } + + if got := doc.Footer(); !slices.Equal(got, []string{"footer"}) { + t.Errorf("footer = %q, want [footer]", got) + } +} + +func TestDocumentCommentsAreWritable(t *testing.T) { + doc, err := Parse([]byte("# above\nk = 1\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + entry, ok := doc.Root().Get("k") + if !ok { + t.Fatal("k is missing") + } + entry.SetComments([]string{"first", "second"}) + entry.SetTrailing("beside") + if got := entry.Comments(); !slices.Equal(got, []string{"first", "second"}) { + t.Errorf("comments = %q", got) + } + if got := entry.Trailing(); got != "beside" { + t.Errorf("trailing = %q", got) + } + + tbl := doc.Root() + tbl.SetComments([]string{"above the root"}) + if got := tbl.Comments(); !slices.Equal(got, []string{"above the root"}) { + t.Errorf("root comments = %q", got) + } + doc.SetFooter([]string{"end"}) + if got := doc.Footer(); !slices.Equal(got, []string{"end"}) { + t.Errorf("footer = %q", got) + } +} + +func TestDocumentArrayOfTables(t *testing.T) { + doc, err := Parse([]byte(`# first element +[[item]] +a = 1 + +[[item]] +b = 2 # beside b +`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + entry, ok := doc.Root().Get("item") + if !ok { + t.Fatal("item is missing") + } + elems := entry.Elements() + if len(elems) != 2 { + t.Fatalf("elements = %d, want 2", len(elems)) + } + if got := elems[0].Keys(); !slices.Equal(got, []string{"a"}) { + t.Errorf("first element keys = %v, want [a]", got) + } + if got := elems[0].Comments(); !slices.Equal(got, []string{"first element"}) { + t.Errorf("first element comments = %q", got) + } + if got := elems[1].Keys(); !slices.Equal(got, []string{"b"}) { + t.Errorf("second element keys = %v, want [b]", got) + } + if got := mustEntry(t, elems[1], "b").Trailing(); got != "beside b" { + t.Errorf("b trailing = %q, want \"beside b\"", got) + } + // The value keeps the map shape the decoder reads. + if _, ok := entry.Value().([]map[string]any); !ok { + t.Errorf("item value = %#v, want []map[string]any", entry.Value()) + } +} + +func TestDocumentDottedKeysAndValueArrays(t *testing.T) { + doc, err := Parse([]byte("a.b.c = 1\narr = [1, {x = 1}]\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + + // A dotted key builds tables, and they are not inline ones. + a, ok := doc.Root().Get("a") + if !ok { + t.Fatal("a is missing") + } + if a.Inline() { + t.Error("a is marked inline") + } + b, ok := a.Table().Get("b") + if !ok { + t.Fatal("a.b is missing") + } + if b.Inline() { + t.Error("a.b is marked inline") + } + if got := b.Table().Keys(); !slices.Equal(got, []string{"c"}) { + t.Errorf("a.b keys = %v, want [c]", got) + } + + // An inline table inside a value array keeps its node in the elements + // slice; the scalar before it has none. + arr, ok := doc.Root().Get("arr") + if !ok { + t.Fatal("arr is missing") + } + elems := arr.Elements() + if len(elems) != 2 { + t.Fatalf("elements = %d, want 2", len(elems)) + } + if elems[0] != nil { + t.Errorf("elements[0] = %#v, want nil for a scalar", elems[0]) + } + if got := elems[1].Keys(); !slices.Equal(got, []string{"x"}) { + t.Errorf("elements[1] keys = %v, want [x]", got) + } +} + +func TestDocumentWithoutStatements(t *testing.T) { + doc, err := Parse([]byte("# only a comment\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := doc.Root().Keys(); len(got) != 0 { + t.Errorf("keys = %v, want none", got) + } + if got := doc.Footer(); !slices.Equal(got, []string{"only a comment"}) { + t.Errorf("footer = %q, want [only a comment]", got) + } + + empty, err := Parse(nil) + if err != nil { + t.Fatalf("parse of nothing: %v", err) + } + if len(empty.Root().Keys()) != 0 || len(empty.Footer()) != 0 { + t.Errorf("empty document = %v / %q", empty.Root().Keys(), empty.Footer()) + } +} + +func TestParseMapIsTheValueTree(t *testing.T) { + // ParseMap is the path that does not build a document, and it gives the + // tree the decoder reads. + tree, err := ParseMap([]byte("a = 1\n\n[t]\nb = \"x\"\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if tree["a"] != int64(1) { + t.Errorf("a = %#v", tree["a"]) + } + if tree["t"].(map[string]any)["b"] != "x" { + t.Errorf("t = %#v", tree["t"]) + } +} + +func TestMarshalRejectsDocument(t *testing.T) { + // A Document is not a value to marshal: its order and comments would be + // dropped, and a struct walk would silently write nothing at all. + doc, err := Parse([]byte("a = 1\n")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := Marshal(doc); err == nil { + t.Fatal("expected an error for a Document") + } else if !strings.Contains(err.Error(), "Map()") { + t.Errorf("err = %v, want it to point at Map()", err) + } + if _, err := Marshal(*doc); err == nil { + t.Fatal("expected an error for a Document value") + } + // The tree marshals, which is the way through. + out, err := Marshal(doc.Map()) + if err != nil { + t.Fatalf("marshal of the tree: %v", err) + } + if want := "a = 1\n"; string(out) != want { + t.Errorf("output = %q, want %q", out, want) + } +} diff --git a/encode.go b/encode.go index c9560ae..e5dde33 100644 --- a/encode.go +++ b/encode.go @@ -86,6 +86,15 @@ func (e *encoder) encode(v any) error { if err := e.checkCtx(); err != nil { return err } + switch x := v.(type) { + case *Document: + if x == nil { + return fmt.Errorf("interpres: cannot marshal nil value") + } + return fmt.Errorf("interpres: cannot marshal a Document; marshal its Map() to write the values") + case Document: + return fmt.Errorf("interpres: cannot marshal a Document; marshal its Map() to write the values") + } rv := reflect.ValueOf(v) if !rv.IsValid() { return fmt.Errorf("interpres: cannot marshal nil value") diff --git a/encode_test.go b/encode_test.go index e44293c..4016308 100644 --- a/encode_test.go +++ b/encode_test.go @@ -289,7 +289,7 @@ func TestEncoderLiteralMultilineFallsBackWhenUnsafe(t *testing.T) { if !bytes.HasPrefix(out, []byte("s = \"")) { t.Errorf("%s: expected the basic quoted form, got:\n%s", c.name, out) } - re, err := Parse(out) + re, err := ParseMap(out) if err != nil { t.Errorf("%s: re-parse: %v\ndoc:\n%s", c.name, err, out) continue @@ -448,7 +448,7 @@ func TestMarshalDuplicateKeyResolvesToOneField(t *testing.T) { if want := "name = \"outer\"\n"; string(out) != want { t.Errorf("output mismatch:\ngot: %q\nwant: %q", out, want) } - if _, err := Parse(out); err != nil { + if _, err := ParseMap(out); err != nil { t.Errorf("re-parse: %v\ndoc:\n%s", err, out) } } @@ -610,7 +610,7 @@ func TestMarshalMixedArrayWithInlineTable(t *testing.T) { // Parse accepts a mixed array (TOML allows any value kinds in one array), // so Marshal of the parsed tree must re-emit it. The table element has no // header form inside a value array and renders inline. - tree, err := Parse([]byte("arr = [1, {a = 2}, \"x\"]\n")) + tree, err := ParseMap([]byte("arr = [1, {a = 2}, \"x\"]\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -622,7 +622,7 @@ func TestMarshalMixedArrayWithInlineTable(t *testing.T) { if string(out) != want { t.Fatalf("output mismatch:\ngot: %q\nwant: %q", out, want) } - re, err := Parse(out) + re, err := ParseMap(out) if err != nil { t.Fatalf("re-parse: %v", err) } @@ -640,7 +640,7 @@ func TestMarshalValueArrayOfTablesStaysInline(t *testing.T) { "a = [{x = 1}, {x = 2}]\n", "b = [{x = 1}, 2, \"three\"]\n", } { - tree, err := Parse([]byte(doc)) + tree, err := ParseMap([]byte(doc)) if err != nil { t.Fatalf("%s: parse: %v", doc, err) } @@ -651,7 +651,7 @@ func TestMarshalValueArrayOfTablesStaysInline(t *testing.T) { if bytes.HasPrefix(out, []byte("[[")) { t.Errorf("%s: emitted the [[header]] form for a value array:\n%s", doc, out) } - re, err := Parse(out) + re, err := ParseMap(out) if err != nil { t.Fatalf("%s: re-parse: %v\ndoc:\n%s", doc, err, out) } @@ -695,7 +695,7 @@ func TestMarshalInlineTableWithDatetime(t *testing.T) { } func TestMarshalArrayOfTablesStaysHeaderForm(t *testing.T) { - tree, err := Parse([]byte("[[items]]\nname = \"a\"\n\n[[items]]\nname = \"b\"\n")) + tree, err := ParseMap([]byte("[[items]]\nname = \"a\"\n\n[[items]]\nname = \"b\"\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -722,7 +722,7 @@ func TestMarshalFloatExponentNoLeadingZero(t *testing.T) { } // Parse to check the output is valid TOML (for a strict parser that // rejects leading zeros in exponents). - if _, err := Parse(out); err != nil { + if _, err := ParseMap(out); err != nil { t.Fatalf("marshalled output is not valid TOML:\n%s\nerror: %v", out, err) } if string(out) != "large = 1e+6\nsmall = 1e-5\n" { @@ -1161,7 +1161,7 @@ func TestMarshalThenParseRoundTrip(t *testing.T) { if err != nil { t.Fatalf("marshal: %v", err) } - tree1, err := Parse(out) + tree1, err := ParseMap(out) if err != nil { t.Fatalf("parse of marshalled: %v\noutput:\n%s", err, out) } @@ -1199,7 +1199,7 @@ created = 2026-06-26T10:00:00Z mixed = [1, {n = 1, name = "a value long enough to push this line well past the one hundred column limit"}] `) - tree1, err := Parse(src) + tree1, err := ParseMap(src) if err != nil { t.Fatalf("parse src: %v", err) } @@ -1207,7 +1207,7 @@ mixed = [1, {n = 1, name = "a value long enough to push this line well past the if err != nil { t.Fatalf("marshal: %v", err) } - tree2, err := Parse(out) + tree2, err := ParseMap(out) if err != nil { t.Fatalf("re-parse marshalled: %v\noutput:\n%s", err, out) } @@ -1649,7 +1649,7 @@ func TestMarshalInlineTableBreaksWhenLong(t *testing.T) { } // The broken form parses back to the same tree. - tree, err := Parse(out) + tree, err := ParseMap(out) if err != nil { t.Fatalf("parse of the encoder output: %v", err) } @@ -1758,11 +1758,11 @@ func TestEncoderInlineTablesOrderAndRoundTrip(t *testing.T) { t.Errorf("output mismatch:\ngot: %q\nwant: %q", compact, want) } - got, err := Parse(compact) + got, err := ParseMap(compact) if err != nil { t.Fatalf("parse of the compact output: %v", err) } - ref, err := Parse(headerForm) + ref, err := ParseMap(headerForm) if err != nil { t.Fatalf("parse of the header output: %v", err) } @@ -1793,7 +1793,7 @@ func TestEncoderInlineTablesKeepsArraysOfTables(t *testing.T) { if string(out) != want { t.Errorf("output mismatch:\ngot: %q\nwant: %q", out, want) } - tree, err := Parse(out) + tree, err := ParseMap(out) if err != nil { t.Fatalf("parse: %v", err) } diff --git a/fuzz_test.go b/fuzz_test.go index 46e0b4b..3e01afc 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -41,7 +41,7 @@ func FuzzParse(f *testing.F) { f.Add([]byte(s)) } f.Fuzz(func(t *testing.T, data []byte) { - tree, err := Parse(data) + tree, err := ParseMap(data) if err != nil { return } @@ -49,7 +49,7 @@ func FuzzParse(f *testing.F) { if err != nil { t.Fatalf("marshal of a parsed tree failed: %v\ntree: %#v", err, tree) } - re, err := Parse(out) + re, err := ParseMap(out) if err != nil { t.Fatalf("re-parse of the emitted document failed: %v\ndoc:\n%s", err, out) } diff --git a/interpres.go b/interpres.go index 89720e2..f45da58 100644 --- a/interpres.go +++ b/interpres.go @@ -11,9 +11,10 @@ // // out, err := interpres.Marshal(cfg) // -// or, for an untyped tree: +// or, for the document with its key order and comments: // -// tree, err := interpres.Parse(data) +// doc, err := interpres.Parse(data) +// tree := doc.Map() // // A Decoder allows strict decoding that rejects keys without a matching // struct field, mirroring (*json.Decoder).DisallowUnknownFields. @@ -89,22 +90,41 @@ func (e *EncodeError) Error() string { return "interpres: " + e.Path + ": " + e. // Unwrap returns the failure the path points at. func (e *EncodeError) Unwrap() error { return e.Err } -// Parse decodes a TOML document into a nested map[string]any. +// Parse decodes a TOML document into a Document: the values, the order the +// keys were written in, whether a table was written inline, and the comments. +// ParseMap gives the plain value tree instead. // // Values are mapped to Go types as follows: strings to string, integers to -// int64, floats to float64, booleans to bool, date-times to time.Time, arrays -// to []any, and tables (including inline tables) to map[string]any. +// int64, floats to float64, booleans to bool, offset date-times to +// OffsetDateTime, the local date-time kinds to their wrappers, arrays to +// []any, and tables (including inline tables) to map[string]any. // // Parse is equivalent to ParseContext with context.Background. -func Parse(data []byte) (map[string]any, error) { +func Parse(data []byte) (*Document, error) { return ParseContext(context.Background(), data) } -// ParseContext decodes a TOML document into a nested map[string]any, obeying -// ctx. The context is checked between top-level statements so cancellation is -// honoured before the parser has done substantial work. -func ParseContext(ctx context.Context, data []byte) (map[string]any, error) { - return parseWithOptions(ctx, data, parseOptions{}) +// ParseContext decodes a TOML document into a Document, obeying ctx. The +// context is checked between top-level statements so cancellation is honoured +// before the parser has done substantial work. +func ParseContext(ctx context.Context, data []byte) (*Document, error) { + _, doc, err := parseWithOptions(ctx, data, parseOptions{}, true) + return doc, err +} + +// ParseMap decodes a TOML document into a nested map[string]any, the value +// tree without the order and the comments a Document carries. It is the shape +// this package parsed into before [Document] existed. +// +// ParseMap is equivalent to ParseMapContext with context.Background. +func ParseMap(data []byte) (map[string]any, error) { + return ParseMapContext(context.Background(), data) +} + +// ParseMapContext is the cancellable variant of ParseMap. +func ParseMapContext(ctx context.Context, data []byte) (map[string]any, error) { + tree, _, err := parseWithOptions(ctx, data, parseOptions{}, false) + return tree, err } // parseOptions bound the work one parse may do. A zero field takes the @@ -114,15 +134,17 @@ type parseOptions struct { maxInputSize int } -func parseWithOptions(ctx context.Context, data []byte, opts parseOptions) (map[string]any, error) { +// parseWithOptions parses data, building the node tree of a Document when +// wantDoc asks for it, and returns both the value tree and that document. +func parseWithOptions(ctx context.Context, data []byte, opts parseOptions, wantDoc bool) (map[string]any, *Document, error) { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } if opts.maxInputSize > 0 && len(data) > opts.maxInputSize { - return nil, fmt.Errorf("interpres: input is %d bytes, over the limit of %d", len(data), opts.maxInputSize) + return nil, nil, fmt.Errorf("interpres: input is %d bytes, over the limit of %d", len(data), opts.maxInputSize) } if !utf8.Valid(data) { - return nil, &SyntaxError{Line: 1, Msg: "input is not valid UTF-8"} + return nil, nil, &SyntaxError{Line: 1, Msg: "input is not valid UTF-8"} } maxDepth := opts.maxDepth if maxDepth <= 0 { @@ -130,8 +152,15 @@ func parseWithOptions(ctx context.Context, data []byte, opts parseOptions) (map[ } // The parser scans data in place; it only reads the buffer, and every // string it stores in the tree is copied out of it. - p := &parser{src: data, line: 1, ctx: ctx, maxDepth: maxDepth} - return p.parse() + p := &parser{src: data, line: 1, ctx: ctx, maxDepth: maxDepth, wantDoc: wantDoc} + tree, err := p.parse() + if err != nil { + return nil, nil, err + } + if !wantDoc { + return tree, nil, nil + } + return tree, &Document{root: p.doc, footer: p.footer}, nil } // Unmarshal parses a TOML document and stores the result in the value pointed @@ -153,7 +182,7 @@ func Unmarshal(data []byte, v any) error { // UnmarshalContext is the cancellable variant of Unmarshal. func UnmarshalContext(ctx context.Context, data []byte, v any) error { - tree, err := ParseContext(ctx, data) + tree, err := ParseMapContext(ctx, data) if err != nil { return err } @@ -210,10 +239,10 @@ func (d *Decoder) Decode(data []byte, v any) error { // DecodeContext is the cancellable variant of Decode. func (d *Decoder) DecodeContext(ctx context.Context, data []byte, v any) error { - tree, err := parseWithOptions(ctx, data, parseOptions{ + tree, _, err := parseWithOptions(ctx, data, parseOptions{ maxDepth: d.maxDepth, maxInputSize: d.maxInputSize, - }) + }, false) if err != nil { return err } diff --git a/interpres_test.go b/interpres_test.go index e0668fa..c1d0093 100644 --- a/interpres_test.go +++ b/interpres_test.go @@ -12,7 +12,7 @@ import ( ) func TestParseScalars(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` title = "interpres" count = 42 ratio = 3.14 @@ -50,7 +50,7 @@ expv = 1e3 } func TestParseInfNan(t *testing.T) { - tree, err := Parse([]byte("pos = inf\nneg = -inf\nbad = nan\n")) + tree, err := ParseMap([]byte("pos = inf\nneg = -inf\nbad = nan\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -66,7 +66,7 @@ func TestParseInfNan(t *testing.T) { } func TestParseStrings(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` basic = "a\tb\nc" literal = 'C:\path\no\escape' quote = "say \"hi\"" @@ -90,7 +90,7 @@ unicode = "\u00e9" } func TestParseMultilineString(t *testing.T) { - tree, err := Parse([]byte("text = \"\"\"\nfirst\nsecond\"\"\"\n")) + tree, err := ParseMap([]byte("text = \"\"\"\nfirst\nsecond\"\"\"\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -100,7 +100,7 @@ func TestParseMultilineString(t *testing.T) { } func TestParseMultilineLineEndingBackslash(t *testing.T) { - tree, err := Parse([]byte("text = \"\"\"\\\n one \\\n two\"\"\"\n")) + tree, err := ParseMap([]byte("text = \"\"\"\\\n one \\\n two\"\"\"\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -110,7 +110,7 @@ func TestParseMultilineLineEndingBackslash(t *testing.T) { } func TestParseTablesAndDottedKeys(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` owner.name = "Petr" [server] @@ -138,7 +138,7 @@ enabled = true } func TestParseArrayOfTables(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` [[forms]] name = "contact" @@ -158,7 +158,7 @@ name = "feedback" } func TestParseArraysAndInlineTables(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` ports = [80, 443] mixed = [ "a", @@ -184,7 +184,7 @@ point = { x = 1, y = 2 } } func TestParseDateTime(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` offset = 1979-05-27T07:32:00Z local = 1979-05-27T07:32:00 day = 1979-05-27 @@ -208,7 +208,7 @@ clock = 07:32:00 } func TestDateTimeFormats(t *testing.T) { - tree, err := Parse([]byte("a = 1987-07-05 17:45:00Z\nb = 1987-07-05t17:45:00z\nc = 1977-12-21T10:32:00.555\n")) + tree, err := ParseMap([]byte("a = 1987-07-05 17:45:00Z\nb = 1987-07-05t17:45:00z\nc = 1977-12-21T10:32:00.555\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -353,7 +353,7 @@ func TestSkippedFieldTag(t *testing.T) { } func TestSyntaxErrorReportsLine(t *testing.T) { - _, err := Parse([]byte("a = 1\nb = \nc = 3\n")) + _, err := ParseMap([]byte("a = 1\nb = \nc = 3\n")) if err == nil { t.Fatal("expected a syntax error") } @@ -367,7 +367,7 @@ func TestSyntaxErrorReportsLine(t *testing.T) { } func TestComments(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` # a leading comment key = "value" # trailing comment # another @@ -381,7 +381,7 @@ key = "value" # trailing comment } func TestDuplicateKeyRejected(t *testing.T) { - _, err := Parse([]byte("a = 1\na = 2\n")) + _, err := ParseMap([]byte("a = 1\na = 2\n")) if err == nil { t.Fatal("expected duplicate key error") } @@ -396,7 +396,7 @@ func TestRejectsInvalidNumbers(t *testing.T) { "0x", "0o", "0b", "0b2", "0o8", "0xG", "+0x1", } { - if _, err := Parse([]byte("v = " + tok + "\n")); err == nil { + if _, err := ParseMap([]byte("v = " + tok + "\n")); err == nil { t.Errorf("%q: expected an error, got none", tok) } } @@ -409,14 +409,14 @@ func TestParseRejectsOffsetOutOfRange(t *testing.T) { "1979-05-27T07:32:00+24:00", "1979-05-27T07:32:00+99:99", } { - if _, err := Parse([]byte("v = " + tok + "\n")); err == nil { + if _, err := ParseMap([]byte("v = " + tok + "\n")); err == nil { t.Errorf("%q: expected an error, got none", tok) } } } func TestParseAcceptsOffsetBounds(t *testing.T) { - tree, err := Parse([]byte("a = 1979-05-27T07:32:00+23:59\nb = 1979-05-27T07:32:00-23:59\n")) + tree, err := ParseMap([]byte("a = 1979-05-27T07:32:00+23:59\nb = 1979-05-27T07:32:00-23:59\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -450,7 +450,7 @@ func TestAcceptsNumberEdgeCases(t *testing.T) { "-2.5E-3": -2.5e-3, } for tok, want := range cases { - tree, err := Parse([]byte("v = " + tok + "\n")) + tree, err := ParseMap([]byte("v = " + tok + "\n")) if err != nil { t.Errorf("%q: %v", tok, err) continue @@ -462,14 +462,14 @@ func TestAcceptsNumberEdgeCases(t *testing.T) { } func TestRejectsTableRedefinition(t *testing.T) { - _, err := Parse([]byte("[a]\nx = 1\n\n[a]\ny = 2\n")) + _, err := ParseMap([]byte("[a]\nx = 1\n\n[a]\ny = 2\n")) if err == nil { t.Fatal("expected a table-redefinition error") } } func TestAllowsImplicitThenExplicitTable(t *testing.T) { - tree, err := Parse([]byte("[a.b]\nx = 1\n\n[a]\ny = 2\n")) + tree, err := ParseMap([]byte("[a.b]\nx = 1\n\n[a]\ny = 2\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -483,13 +483,13 @@ func TestAllowsImplicitThenExplicitTable(t *testing.T) { } func TestRejectsControlCharInString(t *testing.T) { - if _, err := Parse([]byte("v = \"a\x01b\"\n")); err == nil { + if _, err := ParseMap([]byte("v = \"a\x01b\"\n")); err == nil { t.Fatal("expected a control-character error") } } func TestAllowsEscapedControlChar(t *testing.T) { - tree, err := Parse([]byte(`v = "\u0000"`)) + tree, err := ParseMap([]byte(`v = "\u0000"`)) if err != nil { t.Fatalf("parse: %v", err) } @@ -499,7 +499,7 @@ func TestAllowsEscapedControlChar(t *testing.T) { } func TestMultilineQuotesAtDelimiter(t *testing.T) { - tree, err := Parse([]byte("a = '''''two quotes'''''\n")) + tree, err := ParseMap([]byte("a = '''''two quotes'''''\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -518,7 +518,7 @@ func TestRejectsInlineTableExtension(t *testing.T) { "by nested array header": "a = { b = {} }\n[[a.b.c]]\nx = 2\n", } for name, doc := range cases { - if _, err := Parse([]byte(doc)); err == nil { + if _, err := ParseMap([]byte(doc)); err == nil { t.Errorf("%s: expected an inline-table extension error", name) } } @@ -533,7 +533,7 @@ func TestArrayOfTablesFreshScopePerElement(t *testing.T) { "dotted key": "[[a]]\nb.c = 1\n[[a]]\n[a.b]\nd = 2\n", } for name, doc := range cases { - tree, err := Parse([]byte(doc)) + tree, err := ParseMap([]byte(doc)) if err != nil { t.Errorf("%s: %v", name, err) continue @@ -548,7 +548,7 @@ func TestArrayOfTablesFreshScopePerElement(t *testing.T) { "header over dotted in one element": "[[a]]\nb.c = 1\n[a.b]\nd = 2\n", "table over nested array": "[[a]]\n[[a.b]]\n[a.b]\nx = 1\n", } { - if _, err := Parse([]byte(doc)); err == nil { + if _, err := ParseMap([]byte(doc)); err == nil { t.Errorf("%s: expected an error, got none", name) } } @@ -566,14 +566,14 @@ func TestRejectsSpecInvalid(t *testing.T) { // makes the seconds optional. } for name, doc := range cases { - if _, err := Parse([]byte(doc)); err == nil { + if _, err := ParseMap([]byte(doc)); err == nil { t.Errorf("%s: expected an error", name) } } } func TestArrayOfTablesPerElementSubtable(t *testing.T) { - tree, err := Parse([]byte(` + tree, err := ParseMap([]byte(` [[forms]] name = "a" @@ -604,7 +604,7 @@ host = "h2" // --- TOML 1.1 -------------------------------------------------------------- func TestParseAcceptsNoSecondsDatetimes(t *testing.T) { - tree, err := Parse([]byte(`t = 13:37 + tree, err := ParseMap([]byte(`t = 13:37 dt = 1979-05-27T07:32 odt1 = 1979-05-27 07:32Z odt2 = 1979-05-27 07:32-07:00 @@ -627,13 +627,13 @@ odt2 = 1979-05-27 07:32-07:00 t.Errorf("odt2 = %q", got) } // The fraction still requires the seconds it belongs to. - if _, err := Parse([]byte("a = 07:32.5\n")); err == nil { + if _, err := ParseMap([]byte("a = 07:32.5\n")); err == nil { t.Error("07:32.5: expected an error, got none") } } func TestParseAcceptsEscapeAndHexEscapes(t *testing.T) { - tree, err := Parse([]byte(`esc = "\e" + tree, err := ParseMap([]byte(`esc = "\e" hex = "\x20\x7f\xf8" nul = "\x00" multi = """\x68\x65""" @@ -660,14 +660,14 @@ lit = '\x20' } // Two digits exactly; a short or non-hex escape is an error. for _, doc := range []string{`a = "\x4"`, `a = "\x"`, `a = "\xgg"`} { - if _, err := Parse([]byte(doc)); err == nil { + if _, err := ParseMap([]byte(doc)); err == nil { t.Errorf("%s: expected an error, got none", doc) } } } func TestParseAcceptsMultilineInlineTables(t *testing.T) { - tree, err := Parse([]byte("tbl = {\n\thello = \"world\",\n\tarr = [1,\n\t\t2,\n\t],\n\tsub = {\n\t\tk = 1,\n\t},\n\tbare = 2}\n")) + tree, err := ParseMap([]byte("tbl = {\n\thello = \"world\",\n\tarr = [1,\n\t\t2,\n\t],\n\tsub = {\n\t\tk = 1,\n\t},\n\tbare = 2}\n")) if err != nil { t.Fatalf("parse: %v", err) } @@ -682,7 +682,7 @@ func TestParseAcceptsMultilineInlineTables(t *testing.T) { t.Errorf("sub = %#v", tbl["sub"]) } // Comments inside the table, and a trailing comma at both depths. - tree, err = Parse([]byte("m = { # one\n\t# two\n\ta = 1, # three\n\t# four\n}\n")) + tree, err = ParseMap([]byte("m = { # one\n\t# two\n\ta = 1, # three\n\t# four\n}\n")) if err != nil { t.Fatalf("parse with comments: %v", err) } @@ -690,7 +690,7 @@ func TestParseAcceptsMultilineInlineTables(t *testing.T) { t.Errorf("m = %#v", m) } // The old single-line shapes keep working, with and without the comma. - if _, err := Parse([]byte("a = { b = 1, c = 2 }\n")); err != nil { + if _, err := ParseMap([]byte("a = { b = 1, c = 2 }\n")); err != nil { t.Errorf("single line: %v", err) } // Still rejected: two commas, a missing value, and an unclosed table. @@ -699,7 +699,7 @@ func TestParseAcceptsMultilineInlineTables(t *testing.T) { "missing value": "a = {\n\tb =\n}\n", "unterminated": "a = { b = 1,\n", } { - if _, err := Parse([]byte(doc)); err == nil { + if _, err := ParseMap([]byte(doc)); err == nil { t.Errorf("%s: expected an error, got none", name) } } @@ -711,10 +711,10 @@ func TestParseNestingLimit(t *testing.T) { deep := func(n int) []byte { return []byte("v = " + strings.Repeat("[", n) + strings.Repeat("]", n) + "\n") } - if _, err := Parse(deep(100)); err != nil { + if _, err := ParseMap(deep(100)); err != nil { t.Fatalf("a document well inside the limit: %v", err) } - _, err := Parse(deep(maxNestingDepth + 1)) + _, err := ParseMap(deep(maxNestingDepth + 1)) if err == nil { t.Fatal("expected a nesting error") } diff --git a/parser.go b/parser.go index 2e8d54c..4122a85 100644 --- a/parser.go +++ b/parser.go @@ -44,6 +44,27 @@ type parser struct { arrays map[string]bool currentPath []string + + // wantDoc asks for the node tree the Document is built from; doc is that + // tree, and it stays nil when only the value tree is wanted. currentNode + // is the node of p.current; pending collects the comment lines since the + // last statement and trailing the comment on the statement's own line; + // lastEntry and lastTable name the statement those comments belong to; + // lastInline and lastArrayElems carry the nodes of the value parseValue has + // just produced. + wantDoc bool + doc *Table + currentNode *Table + pending []string + trailing string + lastEntry *Entry + lastTable *Table + lastInline *Table + lastArrayElems []*Table + + // footer holds the comment lines that follow the last statement, which + // belong to the document rather than to any table or key. + footer []string } // maxNestingDepth bounds how deeply arrays and inline tables may nest when no @@ -71,6 +92,10 @@ func (p *parser) parse() (map[string]any, error) { p.dotted = map[string]bool{} p.arrays = map[string]bool{} p.currentPath = nil + if p.wantDoc { + p.doc = newTable(p.root) + p.currentNode = p.doc + } for i := 0; ; i++ { if i%ctxCheckInterval == 0 { @@ -84,6 +109,7 @@ func (p *parser) parse() (map[string]any, error) { if p.eof() { break } + p.lastEntry, p.lastTable = nil, nil c := p.peek() switch { case c == '[': @@ -98,10 +124,38 @@ func (p *parser) parse() (map[string]any, error) { if err := p.expectLineEnd(); err != nil { return nil, err } + p.attachComments() } + p.attachFooter() return p.root, nil } +// attachComments hands the collected comments to the statement just parsed: +// the lines above it to its entry or table, the comment on its own line as the +// trailing one. +func (p *parser) attachComments() { + if p.doc != nil { + switch { + case p.lastEntry != nil: + p.lastEntry.comments = p.pending + p.lastEntry.trailing = p.trailing + case p.lastTable != nil: + p.lastTable.comments = p.pending + p.lastTable.trailing = p.trailing + } + } + p.pending, p.trailing = nil, "" +} + +// attachFooter hands the comment lines that follow the last statement to the +// document, which is where a comment block at the end of a file belongs. +func (p *parser) attachFooter() { + if p.doc != nil && len(p.pending) > 0 { + p.footer = p.pending + } + p.pending = nil +} + // checkCtx returns ctx.Err() when the context has been cancelled, nil // otherwise. The call is a no-op when ctx is nil or the zero Background // context, both of which never cancel. @@ -140,7 +194,7 @@ func (p *parser) parseTableHeader() error { } if array { - tbl, err := p.appendArrayTable(key) + tbl, elem, err := p.appendArrayTable(key) if err != nil { return err } @@ -150,6 +204,8 @@ func (p *parser) parseTableHeader() error { p.arrays[pathKey(key)] = true p.current = tbl p.currentPath = key + p.currentNode = elem + p.lastTable = elem return nil } @@ -159,69 +215,91 @@ func (p *parser) parseTableHeader() error { } p.headers[pk] = true - tbl, err := p.tableAt(key) + tbl, node, err := p.tableAt(key) if err != nil { return err } p.current = tbl p.currentPath = key + p.currentNode = node + p.lastTable = node return nil } // tableAt walks (creating intermediate tables) to the table named by key, // relative to the document root, rejecting any step into a frozen inline table. -func (p *parser) tableAt(key []string) (map[string]any, error) { +func (p *parser) tableAt(key []string) (map[string]any, *Table, error) { cur := p.root + node := p.doc path := make([]string, 0, len(key)) for _, k := range key { path = append(path, k) if p.frozen[pathKey(path)] { - return nil, p.errf("cannot extend inline table %q", strings.Join(path, ".")) + return nil, nil, p.errf("cannot extend inline table %q", strings.Join(path, ".")) } existing, ok := cur[k] if !ok { next := map[string]any{} cur[k] = next cur = next + if node != nil { + node = node.addTable(k, next) + } continue } switch v := existing.(type) { case map[string]any: cur = v + if node != nil { + node = node.addTable(k, v) + } case []map[string]any: if len(v) == 0 { - return nil, p.errf("key %q is an empty array of tables", k) + return nil, nil, p.errf("key %q is an empty array of tables", k) } cur = v[len(v)-1] + if node != nil { + node = node.lastElement(k) + } default: - return nil, p.errf("key %q is not a table", k) + return nil, nil, p.errf("key %q is not a table", k) } } - return cur, nil + return cur, node, nil } -func (p *parser) appendArrayTable(key []string) (map[string]any, error) { +func (p *parser) appendArrayTable(key []string) (map[string]any, *Table, error) { parent := p.root + node := p.doc path := make([]string, 0, len(key)) for _, k := range key[:len(key)-1] { path = append(path, k) if p.frozen[pathKey(path)] { - return nil, p.errf("cannot extend inline table %q", strings.Join(path, ".")) + return nil, nil, p.errf("cannot extend inline table %q", strings.Join(path, ".")) } existing, ok := parent[k] if !ok { next := map[string]any{} parent[k] = next parent = next + if node != nil { + node = node.addTable(k, next) + } continue } switch v := existing.(type) { case map[string]any: parent = v + if node != nil { + node = node.addTable(k, v) + } case []map[string]any: parent = v[len(v)-1] + if node != nil { + node = node.lastElement(k) + } default: - return nil, p.errf("key %q is not a table", k) + return nil, nil, p.errf("key %q is not a table", k) } } @@ -233,9 +311,13 @@ func (p *parser) appendArrayTable(key []string) (map[string]any, error) { case []map[string]any: parent[leaf] = append(existing, tbl) default: - return nil, p.errf("key %q is not an array of tables", leaf) + return nil, nil, p.errf("key %q is not an array of tables", leaf) } - return tbl, nil + var elem *Table + if node != nil { + elem = node.addElement(leaf, tbl) + } + return tbl, elem, nil } // --- key/value ------------------------------------------------------------- @@ -262,6 +344,9 @@ func (p *parser) parseKeyValue() error { // top-level statement reuses it for the leaf. abs := make([]string, 0, len(p.currentPath)+len(key)) abs = append(abs, p.currentPath...) + // dests collects the map each dotted key descended into, which the node + // tree needs to build the matching tables around the value. + var dests []map[string]any for _, k := range key[:len(key)-1] { abs = append(abs, k) if p.frozen[pathKey(abs)] { @@ -276,6 +361,7 @@ func (p *parser) parseKeyValue() error { next := map[string]any{} dest[k] = next dest = next + dests = append(dests, next) continue } m, ok := existing.(map[string]any) @@ -283,6 +369,7 @@ func (p *parser) parseKeyValue() error { return p.errf("key %q is not a table", k) } dest = m + dests = append(dests, m) } leaf := key[len(key)-1] abs = append(abs, leaf) @@ -290,10 +377,47 @@ func (p *parser) parseKeyValue() error { return p.errf("duplicate key %q", leaf) } dest[leaf] = val + if p.doc != nil { + node := p.currentNode + for i, k := range key[:len(key)-1] { + node = node.addTable(k, dests[i]) + } + _, inline := val.(map[string]any) + entry := node.addValue(leaf, val, inline) + if inline { + entry.child = p.takeInline(val) + } + if nodes := p.takeArrayElems(val); nodes != nil { + entry.elements = nodes + } + p.lastEntry = entry + } p.freezeInline(abs, val) return nil } +// takeInline returns the node of the inline table just parsed, when v is that +// table's value, and clears it so a later value cannot pick it up. +func (p *parser) takeInline(v any) *Table { + node := p.lastInline + p.lastInline = nil + if _, ok := v.(map[string]any); !ok { + return nil + } + return node +} + +// takeArrayElems returns the element nodes of the array just parsed, when v is +// that array's value, and clears them. +func (p *parser) takeArrayElems(v any) []*Table { + nodes := p.lastArrayElems + p.lastArrayElems = nil + if _, ok := v.([]any); !ok { + return nil + } + return nodes +} + // freezeInline marks the path of an inline table (and any nested inline tables) // as immutable, so a later header or dotted key cannot extend it. func (p *parser) freezeInline(path []string, val any) { @@ -383,6 +507,9 @@ func (p *parser) parseValue() (any, error) { if p.eof() { return nil, p.errf("expected a value") } + // A container value leaves its node behind for the caller to pick up; a + // value that follows must not find the previous one. + p.lastInline, p.lastArrayElems = nil, nil switch c := p.peek(); { case c == '"': return p.parseBasicString() @@ -698,13 +825,23 @@ func (p *parser) readUnicode(n int) (rune, error) { // --- arrays and inline tables --------------------------------------------- -func (p *parser) parseArray() (any, error) { +func (p *parser) parseArray() (val any, err error) { if err := p.enterNesting(); err != nil { return nil, err } defer p.leaveNesting() p.pos++ // '[' arr := []any{} + // elems carries the node of each element that is an inline table, so the + // caller can keep its key order; the entries are nil for other values. + var elems []*Table + if p.doc != nil { + defer func() { + if err == nil { + p.lastArrayElems = elems + } + }() + } for { if err := p.skipNestedSpace(); err != nil { return nil, err @@ -720,6 +857,9 @@ func (p *parser) parseArray() (any, error) { if err != nil { return nil, err } + if p.doc != nil { + elems = append(elems, p.takeInline(v)) + } arr = append(arr, v) if err := p.skipNestedSpace(); err != nil { return nil, err @@ -739,7 +879,7 @@ func (p *parser) parseArray() (any, error) { } } -func (p *parser) parseInlineTable() (any, error) { +func (p *parser) parseInlineTable() (val any, err error) { if err := p.enterNesting(); err != nil { return nil, err } @@ -747,6 +887,18 @@ func (p *parser) parseInlineTable() (any, error) { p.pos++ // '{' tbl := map[string]any{} assigned := map[string]bool{} + // The inline table is a node of its own, so the keys keep their order; the + // caller picks the node up when the table parses. + var node *Table + if p.doc != nil { + node = newTable(tbl) + node.inline = true + defer func() { + if err == nil { + p.lastInline = node + } + }() + } // TOML 1.1 lets an inline table span lines: interior whitespace includes // newlines and comments, and a trailing comma is allowed before the // closing brace. @@ -778,6 +930,7 @@ func (p *parser) parseInlineTable() (any, error) { dest := tbl path := make([]string, 0, len(key)) + var dests []map[string]any for _, k := range key[:len(key)-1] { path = append(path, k) if assigned[pathKey(path)] { @@ -788,6 +941,7 @@ func (p *parser) parseInlineTable() (any, error) { m := map[string]any{} dest[k] = m dest = m + dests = append(dests, m) continue } m, isMap := existing.(map[string]any) @@ -795,6 +949,7 @@ func (p *parser) parseInlineTable() (any, error) { return nil, p.errf("key %q is already defined", k) } dest = m + dests = append(dests, m) } leaf := key[len(key)-1] path = append(path, leaf) @@ -803,6 +958,20 @@ func (p *parser) parseInlineTable() (any, error) { } dest[leaf] = val assigned[pathKey(path)] = true + if node != nil { + child := node + for i, k := range key[:len(key)-1] { + child = child.addTable(k, dests[i]) + } + _, inline := val.(map[string]any) + entry := child.addValue(leaf, val, inline) + if inline { + entry.child = p.takeInline(val) + } + if nodes := p.takeArrayElems(val); nodes != nil { + entry.elements = nodes + } + } if err := p.skipNestedSpace(); err != nil { return nil, err @@ -897,7 +1066,9 @@ func (p *parser) skipNestedSpace() error { p.line++ p.pos++ case '#': - if err := p.skipComment(); err != nil { + // A comment between values inside an array or an inline table is + // skipped; the Document does not carry those yet. + if _, err := p.skipComment(); err != nil { return err } default: @@ -921,9 +1092,11 @@ func (p *parser) skipBlank() error { p.line++ p.pos++ case '#': - if err := p.skipComment(); err != nil { + line, err := p.skipComment() + if err != nil { return err } + p.pending = append(p.pending, line) default: return nil } @@ -931,27 +1104,34 @@ func (p *parser) skipBlank() error { return nil } -func (p *parser) skipComment() error { +func (p *parser) skipComment() (string, error) { p.pos++ // consume '#' + start := p.pos for !p.eof() { c := p.peek() switch { case c == '\n': - return nil + return commentText(string(p.src[start:p.pos])), nil case c == '\r': if p.pos+1 < len(p.src) && p.src[p.pos+1] == '\n' { - return nil + return commentText(string(p.src[start:p.pos])), nil } - return p.errf("bare carriage return is not allowed") + return "", p.errf("bare carriage return is not allowed") case c == '\t': p.pos++ case c < 0x20 || c == 0x7f: - return p.errf("control character U+%04X is not allowed in a comment", c) + return "", p.errf("control character U+%04X is not allowed in a comment", c) default: p.pos++ } } - return nil + return commentText(string(p.src[start:p.pos])), nil +} + +// commentText drops the one space that usually follows the '#', so a line +// stored in a Document reads as the comment itself. +func commentText(s string) string { + return strings.TrimPrefix(s, " ") } // expectCRLF consumes a carriage return that must be immediately followed by a @@ -972,9 +1152,13 @@ func (p *parser) expectLineEnd() error { return nil } if p.peek() == '#' { - if err := p.skipComment(); err != nil { + line, err := p.skipComment() + if err != nil { return err } + if p.doc != nil { + p.trailing = line + } } if p.eof() { return nil