scriptorium/internal/markdown/parse.go

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: MIT

package markdown

import "bytes"

const (
	tabStop    = 4
	codeIndent = 4
)

// Parse parses Markdown source into a tree of block nodes. The source is
// trusted input: no sanitisation is applied, because whether the output may
// reach an audience is the consumer's policy.
func Parse(source []byte) *Node {
	p := &parser{doc: &Node{kind: kindDocument, refs: map[string]reference{}}}
	p.tip = p.doc
	for _, line := range splitLines(normalise(source)) {
		p.processLine(line)
	}
	p.closeUnmatched(p.doc)
	p.finalise(p.doc)
	return p.doc
}

// parser holds the block parsing state. The offset, column, indent and
// blank fields describe the current line from the current offset onward,
// which moves as container prefixes are consumed; a tab may end up
// partially consumed, in which case offset rests on the tab and column
// counts only the consumed part of it.
type parser struct {
	doc *Node
	tip *Node

	line   []byte
	lineNo int

	offset int
	column int

	firstNonspace       int
	firstNonspaceColumn int
	indent              int
	blank               bool

	partiallyConsumedTab bool

	// suppressBlankMark keeps the blank-line bookkeeping away from a line
	// the parser consumed entirely, such as a closing code fence.
	suppressBlankMark bool
}

func (p *parser) processLine(line []byte) {
	p.line = line
	p.lineNo++
	p.offset = 0
	p.column = 0
	p.partiallyConsumedTab = false
	p.findFirstNonspace()

	lastMatched := p.checkOpenBlocks()
	container, opened := p.openNewBlocks(lastMatched)
	p.addText(container, opened)

	switch p.tip.kind {
	case kindHeading, kindThematicBreak:
		p.finalise(p.tip)
	}
}

// checkOpenBlocks matches the line against the open block chain, from the
// document down to the tip, consuming the prefix of every block that
// continues. It returns the deepest matched block; the blocks below it stay
// open until addText closes or lazily continues them.
func (p *parser) checkOpenBlocks() *Node {
	var chain []*Node
	for n := p.tip; n != nil; n = n.parent {
		chain = append(chain, n)
	}
	for i := len(chain) - 2; i >= 0; i-- {
		p.findFirstNonspace()
		if !p.continueBlock(chain[i]) {
			return chain[i+1]
		}
	}
	return chain[0]
}

// continueBlock reports whether the open block continues on the current
// line, consuming its prefix when it does.
func (p *parser) continueBlock(n *Node) bool {
	switch n.kind {
	case kindBlockquote:
		if p.blank || p.indent > 3 {
			return false
		}
		if p.line[p.firstNonspace] != '>' {
			return false
		}
		p.advanceOffset(p.firstNonspace+1-p.offset, false)
		if p.offset < len(p.line) && isSpaceTab(p.line[p.offset]) {
			p.advanceOffset(1, true)
		}
		return true
	case kindListItem:
		if p.blank {
			// A blank line ends an item that never took content.
			if len(n.children) == 0 {
				return false
			}
			p.advanceOffset(p.firstNonspace-p.offset, false)
			return true
		}
		if p.indent >= n.markerOffset+n.padding {
			p.advanceOffset(n.markerOffset+n.padding, true)
			return true
		}
		return false
	case kindCodeBlock:
		if !n.fenced {
			switch {
			case p.indent >= codeIndent:
				p.advanceOffset(codeIndent, true)
				return true
			case p.blank:
				p.advanceOffset(p.firstNonspace-p.offset, false)
				return true
			}
			return false
		}
		if !p.blank && p.indent <= 3 && p.line[p.firstNonspace] == n.fenceChar {
			length := fenceRun(p.line[p.firstNonspace:], n.fenceChar)
			if length >= n.fenceLength && allSpaceTab(p.line[p.firstNonspace+length:]) {
				// A closing fence ends the block; the rest of the line is
				// nothing but whitespace.
				p.advanceOffset(len(p.line)-p.offset, false)
				p.finalise(n)
				p.suppressBlankMark = true
				return false
			}
		}
		// A content line gives up to the opening fence's indentation.
		for i := n.fenceOffset; i > 0 && p.offset < len(p.line) && isSpaceTab(p.line[p.offset]); i-- {
			p.advanceOffset(1, true)
		}
		return true
	case kindHTMLBlock:
		// The tag-based kinds end at a blank line; the raw kinds run to
		// their closing condition.
		if n.htmlType == 6 || n.htmlType == 7 {
			return !p.blank
		}
		return true
	case kindParagraph:
		return !p.blank
	case kindTable:
		return !p.blank
	case kindFootnoteDef:
		if p.blank {
			p.advanceOffset(p.firstNonspace-p.offset, false)
			return true
		}
		if p.indent >= codeIndent {
			p.advanceOffset(codeIndent, true)
			return true
		}
		return false
	case kindDefItem:
		if p.blank {
			p.advanceOffset(p.firstNonspace-p.offset, false)
			return true
		}
		if p.indent >= n.markerOffset+n.padding {
			p.advanceOffset(n.markerOffset+n.padding, true)
			return true
		}
		return false
	}
	return true
}

// openNewBlocks starts new blocks on the line, beginning at the last
// matched container and opening containers until a leaf takes over. It
// returns the container the remaining text belongs to and whether anything
// was opened or changed on the line.
func (p *parser) openNewBlocks(container *Node) (*Node, bool) {
	opened := false
	for {
		switch container.kind {
		case kindCodeBlock, kindHTMLBlock:
			return container, opened
		}
		p.findFirstNonspace()
		if p.blank {
			return container, opened
		}
		indented := p.indent >= codeIndent

		if !indented && p.line[p.firstNonspace] == '>' {
			p.advanceOffset(p.firstNonspace+1-p.offset, false)
			if p.offset < len(p.line) && isSpaceTab(p.line[p.offset]) {
				p.advanceOffset(1, true)
			}
			container = p.addChild(container, kindBlockquote)
			opened = true
			continue
		}
		if !indented {
			if level, ok := scanATX(p.line[p.firstNonspace:]); ok {
				h := p.addChild(container, kindHeading)
				h.level = level
				p.advanceOffset(p.firstNonspace+level-p.offset, false)
				return h, true
			}
		}
		if !indented {
			if char, length, ok := scanOpenFence(p.line[p.firstNonspace:]); ok {
				code := p.addChild(container, kindCodeBlock)
				code.fenced = true
				code.fenceChar = char
				code.fenceLength = length
				code.fenceOffset = p.indent
				p.advanceOffset(p.firstNonspace+length-p.offset, false)
				return code, true
			}
		}
		if !indented {
			if t := scanHTMLBlockStart(p.line[p.firstNonspace:], container.kind == kindParagraph); t > 0 {
				h := p.addChild(container, kindHTMLBlock)
				h.htmlType = t
				return h, true
			}
		}
		if !indented && container.kind == kindParagraph {
			if aligns, ok := scanTableDelimiter(p.line[p.firstNonspace:]); ok {
				if table := p.tryOpenTable(container, aligns); table != nil {
					p.advanceOffset(len(p.line)-p.offset, false)
					return table, true
				}
			}
		}
		if !indented {
			if label, markerLen, ok := scanFootnoteDefStart(p.line[p.firstNonspace:]); ok {
				def := p.addChild(container, kindFootnoteDef)
				def.label = normaliseLabel(label)
				def.padding = codeIndent
				p.advanceOffset(p.firstNonspace+markerLen-p.offset, false)
				container = def
				opened = true
				continue
			}
		}
		if !indented {
			if container.kind == kindParagraph || container.kind == kindDefList {
				if scanDefMarker(p.line[p.firstNonspace:]) {
					if item := p.tryOpenDefItem(container); item != nil {
						container = item
						opened = true
						continue
					}
				}
			}
		}
		if !indented && container.kind == kindParagraph {
			if level, ok := scanSetext(p.line[p.firstNonspace:]); ok {
				// Reference definitions leave the paragraph first; the
				// heading forms only over what remains of it, and a
				// paragraph the definitions emptied turns the underline
				// back into plain text.
				p.extractReferences(container)
				if len(container.content) == 0 {
					parent := container.parent
					parent.children = parent.children[:len(parent.children)-1]
					p.tip = parent
					return parent, true
				}
				container.kind = kindHeading
				container.level = level
				p.advanceOffset(len(p.line)-p.offset, false)
				return container, true
			}
		}
		if !indented && isThematicBreak(p.line[p.firstNonspace:]) {
			p.addChild(container, kindThematicBreak)
			p.advanceOffset(len(p.line)-p.offset, false)
			return p.tip, true
		}
		if !indented {
			if data, markerLen, ok := p.parseListMarker(container.kind == kindParagraph); ok {
				if container.kind != kindList || !listsMatch(container, data) {
					l := p.addChild(container, kindList)
					l.listKind = data.listKind
					l.bulletChar = data.bulletChar
					l.delimiter = data.delimiter
					l.start = data.start
					container = l
				}
				item := p.addChild(container, kindListItem)
				item.markerOffset = p.indent
				p.advanceOffset(p.firstNonspace+markerLen-p.offset, false)
				saveOffset, saveColumn, saveTab := p.offset, p.column, p.partiallyConsumedTab
				for p.offset < len(p.line) && isSpaceTab(p.line[p.offset]) {
					p.advanceOffset(1, true)
				}
				cols := p.column - saveColumn
				blankItem := p.offset >= len(p.line)
				padding := markerLen + cols
				if blankItem || cols >= 5 || cols < 1 {
					padding = markerLen + 1
				}
				p.offset, p.column, p.partiallyConsumedTab = saveOffset, saveColumn, saveTab
				item.padding = padding
				p.advanceOffset(padding-markerLen, true)
				container = item
				opened = true
				continue
			}
		}
		return container, opened
	}
}

// listData carries what a list marker says about the list it belongs to.
type listData struct {
	listKind   listKind
	bulletChar byte
	delimiter  byte
	start      int
}

// listsMatch reports whether a new marker continues the given list.
func listsMatch(l *Node, d listData) bool {
	if l.listKind != d.listKind {
		return false
	}
	if d.listKind == bulletList {
		return l.bulletChar == d.bulletChar
	}
	return l.delimiter == d.delimiter
}

// parseListMarker recognises a list marker at the first non-space
// character. A marker interrupting a paragraph must carry content, and an
// ordered one must number 1.
func (p *parser) parseListMarker(interrupts bool) (listData, int, bool) {
	s := p.line[p.firstNonspace:]
	if len(s) == 0 {
		return listData{}, 0, false
	}
	var d listData
	i := 0
	switch c := s[0]; {
	case c == '-' || c == '+' || c == '*':
		d.listKind = bulletList
		d.bulletChar = c
		i = 1
	case c >= '0' && c <= '9':
		start := 0
		for i < len(s) && i < 9 && s[i] >= '0' && s[i] <= '9' {
			start = start*10 + int(s[i]-'0')
			i++
		}
		if i >= len(s) || (s[i] != '.' && s[i] != ')') {
			return listData{}, 0, false
		}
		if interrupts && start != 1 {
			return listData{}, 0, false
		}
		d.listKind = orderedList
		d.delimiter = s[i]
		d.start = start
		i++
	default:
		return listData{}, 0, false
	}
	if i < len(s) && !isSpaceTab(s[i]) {
		return listData{}, 0, false
	}
	if interrupts {
		j := i
		for j < len(s) && isSpaceTab(s[j]) {
			j++
		}
		if j >= len(s) {
			return listData{}, 0, false
		}
	}
	return d, i, true
}

// tryOpenTable turns the last line of an open paragraph into the header of
// a table whose delimiter row is on the current line. The paragraph keeps
// its earlier lines, or disappears when the header was all of it. The
// table opens only when the header and the delimiter row agree on the
// number of columns.
func (p *parser) tryOpenTable(para *Node, aligns []uint8) *Node {
	content := para.content
	if len(content) == 0 || content[len(content)-1] != '\n' {
		return nil
	}
	lastNewline := bytes.LastIndexByte(content[:len(content)-1], '\n') + 1
	headerLine := content[lastNewline : len(content)-1]
	cells := splitTableRow(headerLine)
	if len(cells) != len(aligns) {
		return nil
	}
	parent := para.parent
	if lastNewline > 0 {
		para.content = content[:lastNewline]
		p.finalise(para)
	} else {
		parent.children = parent.children[:len(parent.children)-1]
		p.tip = parent
	}
	table := p.addChild(parent, kindTable)
	table.align = aligns
	table.header = cells
	return table
}

// tryOpenDefItem turns the line's definition marker into a definition
// inside a definition list. When the container is a paragraph, the
// paragraph's lines become the terms of a new entry; when it is a
// definition list, the marker adds another definition to the entry. The
// returned item is the container for the definition's content, with the
// position placed at the content.
func (p *parser) tryOpenDefItem(container *Node) *Node {
	var dl *Node
	if container.kind == kindParagraph {
		dl = p.defListFromTerms(container)
		if dl == nil {
			return nil
		}
	} else {
		dl = container
	}
	item := p.addChild(dl, kindDefItem)
	item.markerOffset = p.indent
	p.advanceOffset(p.firstNonspace+1-p.offset, false)
	saveOffset, saveColumn, saveTab := p.offset, p.column, p.partiallyConsumedTab
	for p.offset < len(p.line) && isSpaceTab(p.line[p.offset]) {
		p.advanceOffset(1, true)
	}
	cols := p.column - saveColumn
	blankItem := p.offset >= len(p.line)
	padding := 1 + cols
	if blankItem || cols >= 5 || cols < 1 {
		padding = 2
	}
	p.offset, p.column, p.partiallyConsumedTab = saveOffset, saveColumn, saveTab
	item.padding = padding
	p.advanceOffset(padding-1, true)
	return item
}

// defListFromTerms turns a paragraph of terms into definition terms of a
// definition list, continuing the list when one is already open beside the
// paragraph.
func (p *parser) defListFromTerms(para *Node) *Node {
	if len(para.content) == 0 || para.content[len(para.content)-1] != '\n' {
		return nil
	}
	lines := splitLines(para.content)
	if len(lines) == 0 {
		return nil
	}
	parent := para.parent
	parent.children = parent.children[:len(parent.children)-1]
	p.tip = parent
	var dl *Node
	switch {
	case parent.kind == kindDefList:
		dl = parent
	case len(parent.children) > 0 && parent.children[len(parent.children)-1].kind == kindDefList:
		dl = parent.children[len(parent.children)-1]
	default:
		dl = p.addChild(parent, kindDefList)
	}
	for _, line := range lines {
		dt := p.addChild(dl, kindDefTerm)
		dt.content = line
	}
	return dl
}

// addText places the remaining text of the line: into the open paragraph
// when it continues, lazily or not, or into the container that accepts
// lines, or into a new block otherwise. It also records which blocks the
// blank line terminates, which decides list tightness.
func (p *parser) addText(container *Node, opened bool) {
	p.findFirstNonspace()

	if p.suppressBlankMark {
		p.suppressBlankMark = false
		return
	}

	if p.blank && len(container.children) > 0 {
		container.children[len(container.children)-1].lastLineBlank = true
	}
	lastBlank := p.blank &&
		container.kind != kindBlockquote &&
		container.kind != kindHeading &&
		container.kind != kindThematicBreak &&
		!(container.kind == kindCodeBlock && container.fenced) &&
		!(container.kind == kindListItem && len(container.children) == 0 && container.startLine == p.lineNo)
	container.lastLineBlank = lastBlank
	for n := container.parent; n != nil; n = n.parent {
		n.lastLineBlank = false
	}

	maybeLazy := p.tip.kind == kindParagraph
	if maybeLazy && !opened && !p.blank {
		p.advanceOffset(p.firstNonspace-p.offset, false)
		p.appendLine(p.tip, true)
		return
	}

	p.closeUnmatched(container)

	switch {
	case container.kind == kindCodeBlock:
		p.appendLine(container, true)
	case container.kind == kindHTMLBlock:
		p.appendLine(container, true)
		if htmlBlockEnds(container.htmlType, p.line[p.offset:]) {
			p.finalise(container)
		}
	case p.blank:
		// A blank line adds no content.
	case container.kind == kindTable:
		row := splitTableRow(p.line[p.offset:])
		for len(row) < len(container.header) {
			row = append(row, nil)
		}
		container.rows = append(container.rows, row[:len(container.header)])
	case container.kind == kindParagraph:
		p.advanceOffset(p.firstNonspace-p.offset, false)
		p.appendLine(container, true)
	case container.kind == kindHeading:
		container.content = append(container.content, chopClosingHashes(p.line[p.firstNonspace:])...)
	default:
		if p.indent >= codeIndent && !maybeLazy {
			code := p.addChild(container, kindCodeBlock)
			p.advanceOffset(codeIndent, true)
			p.appendLine(code, true)
		} else {
			if container.kind == kindListItem && len(container.children) == 0 && container.startLine == p.lineNo {
				if checked, n, ok := scanTaskMarker(p.line[p.firstNonspace:]); ok {
					p.advanceOffset(p.firstNonspace+n-p.offset, false)
					container.task = true
					container.taskDone = checked
					p.findFirstNonspace()
				}
			}
			para := p.addChild(container, kindParagraph)
			p.advanceOffset(p.firstNonspace-p.offset, false)
			p.appendLine(para, true)
		}
	}
}

// addChild attaches a new block below parent, closing open blocks that
// cannot contain it, and makes it the tip.
func (p *parser) addChild(parent *Node, kind nodeKind) *Node {
	for !canContain(parent.kind, kind) {
		p.finalise(parent)
		parent = parent.parent
	}
	n := &Node{kind: kind, parent: parent, startLine: p.lineNo}
	parent.children = append(parent.children, n)
	p.tip = n
	return n
}

// closeUnmatched closes every open block below the given container.
func (p *parser) closeUnmatched(container *Node) {
	for p.tip != container {
		p.finalise(p.tip)
	}
}

// finalise closes a block and its children, trimming content and computing
// derived data such as list tightness.
func (p *parser) finalise(n *Node) {
	if n.finalised {
		return
	}
	n.finalised = true
	for _, c := range n.children {
		p.finalise(c)
	}
	switch n.kind {
	case kindParagraph:
		p.extractReferences(n)
		if len(n.content) == 0 && n.parent != nil {
			n.parent.children = n.parent.children[:len(n.parent.children)-1]
		}
	case kindHeading:
		if bytes.HasSuffix(n.content, []byte("\n")) {
			n.content = n.content[:len(n.content)-1]
		}
	case kindCodeBlock:
		if n.fenced {
			if i := bytes.IndexByte(n.content, '\n'); i >= 0 {
				n.info = unescapeText(string(bytes.TrimSpace(n.content[:i])))
				n.content = n.content[i+1:]
			} else {
				n.info = unescapeText(string(bytes.TrimSpace(n.content)))
				n.content = nil
			}
		} else {
			n.content = trimTrailingBlankLines(n.content)
		}
	case kindList, kindDefList:
		if n.kind == kindDefList {
			p.liftLeakedTerms(n)
		}
		n.tight = !blocksAreLoose(n.children)
	case kindDocument:
		p.gatherFootnotes(n)
	}
	if p.tip == n {
		p.tip = n.parent
	}
}

// liftLeakedTerms moves the paragraphs that never became terms out of a
// definition list. A paragraph may sit directly inside the list as a
// term waiting for its definition line; when that line never arrives the
// paragraph is prose after the list, and the list ends before it, since
// a dl element carries nothing but terms and definitions. Terms and
// definitions after the leak form a list of their own.
func (p *parser) liftLeakedTerms(dl *Node) {
	leak := -1
	for i, c := range dl.children {
		if c.kind == kindParagraph {
			leak = i
			break
		}
	}
	if leak < 0 {
		return
	}
	parent := dl.parent
	base := 0
	for i, c := range parent.children {
		if c == dl {
			base = i + 1
			break
		}
	}
	tail := dl.children[leak:]
	dl.children = dl.children[:leak]
	insert := func(n *Node) {
		parent.children = append(parent.children, nil)
		copy(parent.children[base+1:], parent.children[base:])
		parent.children[base] = n
		base++
		n.parent = parent
	}
	var regroup *Node
	for _, c := range tail {
		switch c.kind {
		case kindParagraph:
			insert(c)
			regroup = nil
		case kindDefTerm, kindDefItem:
			if regroup == nil {
				regroup = &Node{kind: kindDefList, parent: parent}
				insert(regroup)
			}
			c.parent = regroup
			regroup.children = append(regroup.children, c)
		}
	}
	if regroup != nil {
		regroup.tight = !blocksAreLoose(regroup.children)
	}
}

// trimTrailingBlankLines removes the trailing blank lines of an indented
// code block. Every content line carries its newline, so the result of a
// non-empty block ends with exactly one.
func trimTrailingBlankLines(c []byte) []byte {
	for len(c) > 0 {
		end := len(c) - 1 // the final newline
		start := bytes.LastIndexByte(c[:end], '\n') + 1
		if !allSpaceTab(c[start:end]) {
			return c
		}
		c = c[:start]
	}
	return c
}

// blocksAreLoose reports whether any two sibling blocks, or any two blocks
// of one container-like block, are separated by a blank line.
func blocksAreLoose(nodes []*Node) bool {
	for i, n := range nodes {
		if endsWithBlank(n) && i+1 < len(nodes) {
			return true
		}
		switch n.kind {
		case kindListItem, kindDefItem:
			for j, child := range n.children {
				lastItem := i+1 == len(nodes)
				lastChild := j+1 == len(n.children)
				if endsWithBlank(child) && (!lastItem || !lastChild) {
					return true
				}
			}
		}
	}
	return false
}

// endsWithBlank reports whether the block, or the last block inside a
// container chain, was followed by a blank line.
func endsWithBlank(n *Node) bool {
	if n.lastLineChecked {
		return n.lastLineBlank
	}
	n.lastLineChecked = true
	switch n.kind {
	case kindList, kindListItem, kindDefList, kindDefItem, kindFootnoteDef:
		if len(n.children) > 0 {
			return endsWithBlank(n.children[len(n.children)-1])
		}
	}
	return n.lastLineBlank
}

// gatherFootnotes lifts every footnote definition out of the tree, keeping
// the first definition of a label.
func (p *parser) gatherFootnotes(doc *Node) {
	seen := map[string]bool{}
	var defs []*Node
	var walk func(n *Node)
	walk = func(n *Node) {
		keep := n.children[:0]
		for _, c := range n.children {
			if c.kind == kindFootnoteDef {
				if !seen[c.label] {
					seen[c.label] = true
					defs = append(defs, c)
				}
				continue
			}
			walk(c)
			keep = append(keep, c)
		}
		n.children = keep
	}
	walk(doc)
	doc.footnotes = defs
}

// appendLine appends the remaining line to the block's content. A tab
// partially consumed while skipping indentation becomes the spaces it
// stood for.
func (p *parser) appendLine(n *Node, newline bool) {
	if p.partiallyConsumedTab {
		p.offset++
		for i := tabStop - p.column%tabStop; i > 0; i-- {
			n.content = append(n.content, ' ')
		}
	}
	n.content = append(n.content, p.line[p.offset:]...)
	if newline {
		n.content = append(n.content, '\n')
	}
}

// advanceOffset moves into the line by count bytes, or columns when
// columns is set, expanding tabs to tab stops and leaving a tab partially
// consumed when the count stops inside it.
func (p *parser) advanceOffset(count int, columns bool) {
	for count > 0 && p.offset < len(p.line) {
		c := p.line[p.offset]
		if c != '\t' {
			p.partiallyConsumedTab = false
			p.offset++
			p.column++
			count--
			continue
		}
		toTab := tabStop - p.column%tabStop
		if columns {
			p.partiallyConsumedTab = toTab > count
			steps := min(count, toTab)
			p.column += steps
			if !p.partiallyConsumedTab {
				p.offset++
			}
			count -= steps
		} else {
			p.partiallyConsumedTab = false
			p.offset++
			p.column += toTab
			count--
		}
	}
}

// findFirstNonspace locates the first non-space character from the offset
// onward, computing the column of that character and the indentation of
// the line relative to the offset. The line is blank when the first
// non-space character does not exist.
func (p *parser) findFirstNonspace() {
	toTab := tabStop - p.column%tabStop
	p.firstNonspace = p.offset
	p.firstNonspaceColumn = p.column
	for p.firstNonspace < len(p.line) {
		c := p.line[p.firstNonspace]
		switch c {
		case ' ':
			p.firstNonspace++
			p.firstNonspaceColumn++
			toTab--
			if toTab == 0 {
				toTab = tabStop
			}
		case '\t':
			p.firstNonspace++
			p.firstNonspaceColumn += toTab
			toTab = tabStop
		default:
			p.indent = p.firstNonspaceColumn - p.column
			p.blank = false
			return
		}
	}
	p.indent = p.firstNonspaceColumn - p.column
	p.blank = true
}

// normalise prepares source for parsing: line endings become newlines and
// a null byte becomes the replacement character.
func normalise(source []byte) []byte {
	out := make([]byte, 0, len(source))
	for i := 0; i < len(source); i++ {
		switch c := source[i]; c {
		case '\r':
			if i+1 < len(source) && source[i+1] == '\n' {
				i++
			}
			out = append(out, '\n')
		case 0:
			out = append(out, '\xef', '\xbf', '\xbd')
		default:
			out = append(out, c)
		}
	}
	return out
}

// splitLines splits normalised source into lines without their newlines.
// A trailing newline produces no empty final line.
func splitLines(source []byte) [][]byte {
	var lines [][]byte
	start := 0
	for i, c := range source {
		if c == '\n' {
			lines = append(lines, source[start:i])
			start = i + 1
		}
	}
	if start < len(source) {
		lines = append(lines, source[start:])
	}
	return lines
}