Merge branch 'worktree-agent-a085d9a07ef787c18'
This commit is contained in:
121 files changed
+5737
-1
No files matched your search
@@ -0,0 +1,305 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BlockKind is the kind of a Markdown block.
|
||||
type BlockKind uint8
|
||||
|
||||
const (
|
||||
BlockParagraph BlockKind = iota
|
||||
BlockHeading
|
||||
BlockList
|
||||
BlockQuote
|
||||
BlockCode
|
||||
BlockTable
|
||||
BlockRule
|
||||
)
|
||||
|
||||
// MdSpan is a run of inline text with one style.
|
||||
type MdSpan struct {
|
||||
Text string
|
||||
// URL is set for a link.
|
||||
URL string
|
||||
Strong, Em, Code, Strike bool
|
||||
}
|
||||
|
||||
// MdItem is one list item.
|
||||
type MdItem struct {
|
||||
// Number is the ordinal of an ordered item, 0 for a bullet.
|
||||
Number int
|
||||
Depth int
|
||||
Spans []MdSpan
|
||||
}
|
||||
|
||||
// TableAlign is a table column's alignment.
|
||||
type TableAlign uint8
|
||||
|
||||
const (
|
||||
AlignLeft TableAlign = iota
|
||||
AlignCenter
|
||||
AlignRight
|
||||
)
|
||||
|
||||
// MdBlock is one parsed block.
|
||||
type MdBlock struct {
|
||||
Kind BlockKind
|
||||
// Level is a heading's level, 1 to 6.
|
||||
Level int
|
||||
// Spans hold a paragraph's, heading's or quote's text.
|
||||
Spans []MdSpan
|
||||
Items []MdItem
|
||||
// Lang and Code hold a fenced code block; Open is true until its fence closes.
|
||||
Lang, Code string
|
||||
Open bool
|
||||
Header []string
|
||||
Align []TableAlign
|
||||
Rows [][]string
|
||||
}
|
||||
|
||||
// Markdown is a document parsed incrementally as its source grows.
|
||||
type Markdown struct {
|
||||
src string
|
||||
Blocks []MdBlock
|
||||
starts []int
|
||||
}
|
||||
|
||||
// Update parses src and returns the index of the first block parsed anew.
|
||||
// When src extends the last source, blocks before the last stay as they are.
|
||||
func (m *Markdown) Update(src string) int {
|
||||
if src == m.src && m.starts != nil {
|
||||
return len(m.Blocks)
|
||||
}
|
||||
from := 0
|
||||
if len(m.Blocks) > 0 && strings.HasPrefix(src, m.src) {
|
||||
from = len(m.Blocks) - 1
|
||||
}
|
||||
off := 0
|
||||
if from > 0 {
|
||||
off = m.starts[from]
|
||||
}
|
||||
m.Blocks, m.starts = m.Blocks[:from], m.starts[:from]
|
||||
m.src = src
|
||||
parseBlocks(m, src, off)
|
||||
return from
|
||||
}
|
||||
|
||||
// mdLine is one source line and its byte offset.
|
||||
type mdLine struct {
|
||||
text string
|
||||
off int
|
||||
}
|
||||
|
||||
func parseBlocks(m *Markdown, src string, off int) {
|
||||
var lines []mdLine
|
||||
for off < len(src) {
|
||||
end := strings.IndexByte(src[off:], '\n')
|
||||
if end < 0 {
|
||||
lines = append(lines, mdLine{src[off:], off})
|
||||
break
|
||||
}
|
||||
lines = append(lines, mdLine{strings.TrimRight(src[off:off+end], "\r"), off})
|
||||
off += end + 1
|
||||
}
|
||||
for i := 0; i < len(lines); {
|
||||
l := lines[i].text
|
||||
if strings.TrimSpace(l) == "" {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
start := lines[i].off
|
||||
var b MdBlock
|
||||
switch {
|
||||
case mdFence(l) != "":
|
||||
b, i = parseFenced(lines, i)
|
||||
case mdHeadingLevel(l) > 0:
|
||||
lv := mdHeadingLevel(l)
|
||||
b = MdBlock{Kind: BlockHeading, Level: lv, Spans: ParseInline(strings.TrimSpace(strings.TrimRight(l[lv:], "# ")))}
|
||||
i++
|
||||
case mdIsRule(l):
|
||||
b = MdBlock{Kind: BlockRule}
|
||||
i++
|
||||
case strings.HasPrefix(strings.TrimLeft(l, " "), ">"):
|
||||
b, i = parseQuote(lines, i)
|
||||
case mdListMarker(l) != nil:
|
||||
b, i = parseList(lines, i)
|
||||
case i+1 < len(lines) && strings.Contains(l, "|") && mdDelimiter(lines[i+1].text) != nil:
|
||||
b, i = parseTable(lines, i)
|
||||
default:
|
||||
b, i = parseParagraph(lines, i)
|
||||
}
|
||||
m.Blocks = append(m.Blocks, b)
|
||||
m.starts = append(m.starts, start)
|
||||
}
|
||||
}
|
||||
|
||||
// mdStarts reports whether line l begins a block other than a paragraph.
|
||||
func mdStarts(lines []mdLine, i int) bool {
|
||||
l := lines[i].text
|
||||
return strings.TrimSpace(l) == "" || mdFence(l) != "" || mdHeadingLevel(l) > 0 || mdIsRule(l) ||
|
||||
strings.HasPrefix(strings.TrimLeft(l, " "), ">") || mdListMarker(l) != nil ||
|
||||
i+1 < len(lines) && strings.Contains(l, "|") && mdDelimiter(lines[i+1].text) != nil
|
||||
}
|
||||
|
||||
func parseParagraph(lines []mdLine, i int) (MdBlock, int) {
|
||||
var parts []string
|
||||
for ; i < len(lines) && (len(parts) == 0 || !mdStarts(lines, i)); i++ {
|
||||
parts = append(parts, strings.TrimSpace(lines[i].text))
|
||||
}
|
||||
return MdBlock{Kind: BlockParagraph, Spans: ParseInline(strings.Join(parts, " "))}, i
|
||||
}
|
||||
|
||||
// mdFence returns the fence a line opens, "```" or "~~~", or "".
|
||||
func mdFence(l string) string {
|
||||
t := strings.TrimLeft(l, " ")
|
||||
for _, f := range []string{"```", "~~~"} {
|
||||
if strings.HasPrefix(t, f) {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseFenced(lines []mdLine, i int) (MdBlock, int) {
|
||||
fence := mdFence(lines[i].text)
|
||||
b := MdBlock{Kind: BlockCode, Open: true, Lang: strings.ToLower(strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(lines[i].text), fence[:1])))}
|
||||
var code []string
|
||||
for i++; i < len(lines); i++ {
|
||||
if strings.HasPrefix(strings.TrimSpace(lines[i].text), fence) && strings.Trim(strings.TrimSpace(lines[i].text), fence[:1]) == "" {
|
||||
b.Open = false
|
||||
i++
|
||||
break
|
||||
}
|
||||
code = append(code, lines[i].text)
|
||||
}
|
||||
b.Code = strings.Join(code, "\n")
|
||||
return b, i
|
||||
}
|
||||
|
||||
func mdHeadingLevel(l string) int {
|
||||
n := 0
|
||||
for n < len(l) && l[n] == '#' {
|
||||
n++
|
||||
}
|
||||
if n == 0 || n > 6 || n < len(l) && l[n] != ' ' {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func mdIsRule(l string) bool {
|
||||
t := strings.ReplaceAll(strings.TrimSpace(l), " ", "")
|
||||
if len(t) < 3 {
|
||||
return false
|
||||
}
|
||||
return strings.Count(t, t[:1]) == len(t) && strings.ContainsAny(t[:1], "-*_")
|
||||
}
|
||||
|
||||
func parseQuote(lines []mdLine, i int) (MdBlock, int) {
|
||||
var parts []string
|
||||
for ; i < len(lines); i++ {
|
||||
t := strings.TrimLeft(lines[i].text, " ")
|
||||
if !strings.HasPrefix(t, ">") {
|
||||
break
|
||||
}
|
||||
parts = append(parts, strings.TrimSpace(t[1:]))
|
||||
}
|
||||
return MdBlock{Kind: BlockQuote, Spans: ParseInline(strings.Join(parts, " "))}, i
|
||||
}
|
||||
|
||||
// mdMarker is a list item's marker.
|
||||
type mdMarker struct {
|
||||
number, depth, width int
|
||||
}
|
||||
|
||||
func mdListMarker(l string) *mdMarker {
|
||||
indent := len(l) - len(strings.TrimLeft(l, " "))
|
||||
t := l[indent:]
|
||||
if len(t) >= 2 && strings.ContainsRune("-*+", rune(t[0])) && t[1] == ' ' {
|
||||
return &mdMarker{depth: indent / 2, width: indent + 2}
|
||||
}
|
||||
d := 0
|
||||
for d < len(t) && t[d] >= '0' && t[d] <= '9' {
|
||||
d++
|
||||
}
|
||||
if d == 0 || d > 9 || d+1 >= len(t) || !strings.ContainsRune(".)", rune(t[d])) || t[d+1] != ' ' {
|
||||
return nil
|
||||
}
|
||||
n, _ := strconv.Atoi(t[:d])
|
||||
return &mdMarker{number: max(n, 1), depth: indent / 2, width: indent + d + 2}
|
||||
}
|
||||
|
||||
func parseList(lines []mdLine, i int) (MdBlock, int) {
|
||||
b := MdBlock{Kind: BlockList}
|
||||
var texts []string
|
||||
for ; i < len(lines); i++ {
|
||||
l := lines[i].text
|
||||
if mk := mdListMarker(l); mk != nil {
|
||||
b.Items = append(b.Items, MdItem{Number: mk.number, Depth: mk.depth})
|
||||
texts = append(texts, strings.TrimSpace(l[mk.width:]))
|
||||
continue
|
||||
}
|
||||
if mdStarts(lines, i) {
|
||||
break
|
||||
}
|
||||
// A lazy line continues the item above.
|
||||
texts[len(texts)-1] += " " + strings.TrimSpace(l)
|
||||
}
|
||||
for j := range b.Items {
|
||||
b.Items[j].Spans = ParseInline(texts[j])
|
||||
}
|
||||
return b, i
|
||||
}
|
||||
|
||||
// mdCells splits a table row into trimmed cells.
|
||||
func mdCells(l string) []string {
|
||||
t := strings.TrimSpace(l)
|
||||
t = strings.TrimSuffix(strings.TrimPrefix(t, "|"), "|")
|
||||
cells := strings.Split(t, "|")
|
||||
for i := range cells {
|
||||
cells[i] = strings.TrimSpace(cells[i])
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
// mdDelimiter parses a table's delimiter row, or returns nil.
|
||||
func mdDelimiter(l string) []TableAlign {
|
||||
if !strings.Contains(l, "-") {
|
||||
return nil
|
||||
}
|
||||
var out []TableAlign
|
||||
for _, c := range mdCells(l) {
|
||||
left, right := strings.HasPrefix(c, ":"), strings.HasSuffix(c, ":")
|
||||
if strings.Trim(c, ":") == "" || strings.Trim(c, ":-") != "" {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case left && right:
|
||||
out = append(out, AlignCenter)
|
||||
case right:
|
||||
out = append(out, AlignRight)
|
||||
default:
|
||||
out = append(out, AlignLeft)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseTable(lines []mdLine, i int) (MdBlock, int) {
|
||||
b := MdBlock{Kind: BlockTable, Header: mdCells(lines[i].text), Align: mdDelimiter(lines[i+1].text)}
|
||||
n := len(b.Header)
|
||||
for len(b.Align) < n {
|
||||
b.Align = append(b.Align, AlignLeft)
|
||||
}
|
||||
b.Align = b.Align[:n]
|
||||
for i += 2; i < len(lines) && strings.Contains(lines[i].text, "|") && strings.TrimSpace(lines[i].text) != ""; i++ {
|
||||
row := mdCells(lines[i].text)
|
||||
for len(row) < n {
|
||||
row = append(row, "")
|
||||
}
|
||||
b.Rows = append(b.Rows, row[:n])
|
||||
}
|
||||
return b, i
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package internal
|
||||
|
||||
import "strings"
|
||||
|
||||
// mdStyle is the inline style in effect.
|
||||
type mdStyle struct{ strong, em, strike bool }
|
||||
|
||||
// ParseInline splits text into styled spans: emphasis, code, strikethrough and links.
|
||||
// An opener without its closer stays literal, as while a message streams in.
|
||||
func ParseInline(s string) []MdSpan {
|
||||
var out []MdSpan
|
||||
parseInline(s, mdStyle{}, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
func mdEmit(out *[]MdSpan, sp MdSpan) {
|
||||
if sp.Text == "" {
|
||||
return
|
||||
}
|
||||
if n := len(*out); n > 0 {
|
||||
last := &(*out)[n-1]
|
||||
if sp.URL == "" && last.URL == "" && last.Strong == sp.Strong && last.Em == sp.Em && last.Code == sp.Code && last.Strike == sp.Strike {
|
||||
last.Text += sp.Text
|
||||
return
|
||||
}
|
||||
}
|
||||
*out = append(*out, sp)
|
||||
}
|
||||
|
||||
func parseInline(s string, st mdStyle, out *[]MdSpan) {
|
||||
plain := func(t string) {
|
||||
mdEmit(out, MdSpan{Text: t, Strong: st.strong, Em: st.em, Strike: st.strike})
|
||||
}
|
||||
for i := 0; i < len(s); {
|
||||
switch c := s[i]; {
|
||||
case c == '\\' && i+1 < len(s) && strings.IndexByte("\\`*_~[]()#!>-|", s[i+1]) >= 0:
|
||||
plain(s[i+1 : i+2])
|
||||
i += 2
|
||||
continue
|
||||
case c == '`':
|
||||
if end := strings.IndexByte(s[i+1:], '`'); end >= 0 {
|
||||
mdEmit(out, MdSpan{Text: s[i+1 : i+1+end], Code: true, Strong: st.strong, Em: st.em, Strike: st.strike})
|
||||
i += end + 2
|
||||
continue
|
||||
}
|
||||
case c == '[':
|
||||
if text, url, n := mdLink(s[i:]); n > 0 {
|
||||
mdEmit(out, MdSpan{Text: text, URL: url, Strong: st.strong, Em: st.em, Strike: st.strike})
|
||||
i += n
|
||||
continue
|
||||
}
|
||||
case c == '*' || c == '_' || c == '~':
|
||||
if n := mdDelimited(s, i, st, out); n > 0 {
|
||||
i += n
|
||||
continue
|
||||
}
|
||||
}
|
||||
j := i + 1
|
||||
for j < len(s) && strings.IndexByte("\\`[*_~", s[j]) < 0 {
|
||||
j++
|
||||
}
|
||||
plain(s[i:j])
|
||||
i = j
|
||||
}
|
||||
}
|
||||
|
||||
// mdDelimited parses emphasis or strikethrough opening at i and returns its length, or 0.
|
||||
func mdDelimited(s string, i int, st mdStyle, out *[]MdSpan) int {
|
||||
c := s[i]
|
||||
n := 1
|
||||
if i+1 < len(s) && s[i+1] == c {
|
||||
n = 2
|
||||
}
|
||||
if c == '~' && n != 2 {
|
||||
return 0
|
||||
}
|
||||
// Intraword underscores stay literal, as in snake_case.
|
||||
if c == '_' && i > 0 && mdWord(s[i-1]) {
|
||||
return 0
|
||||
}
|
||||
marker := s[i : i+n]
|
||||
body := i + n
|
||||
if body >= len(s) || s[body] == ' ' {
|
||||
return 0
|
||||
}
|
||||
end := -1
|
||||
for j := body + 1; j+n <= len(s); j++ {
|
||||
if s[j:j+n] == marker && s[j-1] != ' ' && (n == 2 || j+1 >= len(s) || s[j+1] != c) {
|
||||
end = j
|
||||
break
|
||||
}
|
||||
}
|
||||
if end < 0 {
|
||||
return 0
|
||||
}
|
||||
inner := st
|
||||
switch {
|
||||
case c == '~':
|
||||
inner.strike = true
|
||||
case n == 2:
|
||||
inner.strong = true
|
||||
default:
|
||||
inner.em = true
|
||||
}
|
||||
parseInline(s[body:end], inner, out)
|
||||
return end + n - i
|
||||
}
|
||||
|
||||
func mdWord(b byte) bool {
|
||||
return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
|
||||
}
|
||||
|
||||
// mdLink parses [text](url) at the start of s and returns its length, or 0.
|
||||
func mdLink(s string) (text, url string, n int) {
|
||||
close := strings.Index(s, "](")
|
||||
if close < 1 || strings.IndexByte(s[1:close], '[') >= 0 {
|
||||
return "", "", 0
|
||||
}
|
||||
end := strings.IndexByte(s[close+2:], ')')
|
||||
if end < 0 {
|
||||
return "", "", 0
|
||||
}
|
||||
url = strings.TrimSpace(s[close+2 : close+2+end])
|
||||
if url == "" || strings.ContainsAny(url, " \n") {
|
||||
return "", "", 0
|
||||
}
|
||||
return s[1:close], url, close + 3 + end
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const mdSample = "# Title\n\nSome **bold** and *em* text with `code`\nand a [link](https://x.io).\n\n" +
|
||||
"- one\n- two\n - nested\n3. three\n\n> quoted *line*\n\n```go\nfunc f() {}\n```\n\n" +
|
||||
"| a | b |\n|:--|--:|\n| 1 | 2 |\n| 3 |\n\n---\n\nend ~~old~~ new"
|
||||
|
||||
func TestMarkdownBlocks(t *testing.T) {
|
||||
var m Markdown
|
||||
m.Update(mdSample)
|
||||
var kinds []BlockKind
|
||||
for _, b := range m.Blocks {
|
||||
kinds = append(kinds, b.Kind)
|
||||
}
|
||||
want := []BlockKind{BlockHeading, BlockParagraph, BlockList, BlockQuote, BlockCode, BlockTable, BlockRule, BlockParagraph}
|
||||
if !reflect.DeepEqual(kinds, want) {
|
||||
t.Fatalf("kinds %v, want %v", kinds, want)
|
||||
}
|
||||
b := m.Blocks
|
||||
if b[0].Level != 1 || b[0].Spans[0].Text != "Title" {
|
||||
t.Errorf("heading %+v", b[0])
|
||||
}
|
||||
p := b[1].Spans
|
||||
if p[1] != (MdSpan{Text: "bold", Strong: true}) || p[3] != (MdSpan{Text: "em", Em: true}) || p[5] != (MdSpan{Text: "code", Code: true}) {
|
||||
t.Errorf("paragraph spans %+v", p)
|
||||
}
|
||||
if p[7] != (MdSpan{Text: "link", URL: "https://x.io"}) {
|
||||
t.Errorf("link %+v", p[7])
|
||||
}
|
||||
it := b[2].Items
|
||||
if len(it) != 4 || it[2].Depth != 1 || it[2].Spans[0].Text != "nested" || it[3].Number != 3 {
|
||||
t.Errorf("list items %+v", it)
|
||||
}
|
||||
if b[3].Spans[1] != (MdSpan{Text: "line", Em: true}) {
|
||||
t.Errorf("quote %+v", b[3].Spans)
|
||||
}
|
||||
if b[4].Lang != "go" || b[4].Code != "func f() {}" || b[4].Open {
|
||||
t.Errorf("code %+v", b[4])
|
||||
}
|
||||
tb := b[5]
|
||||
if !reflect.DeepEqual(tb.Header, []string{"a", "b"}) || !reflect.DeepEqual(tb.Align, []TableAlign{AlignLeft, AlignRight}) ||
|
||||
!reflect.DeepEqual(tb.Rows, [][]string{{"1", "2"}, {"3", ""}}) {
|
||||
t.Errorf("table %+v", tb)
|
||||
}
|
||||
if b[7].Spans[1] != (MdSpan{Text: "old", Strike: true}) {
|
||||
t.Errorf("strike %+v", b[7].Spans)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownInlineLiterals(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"2 * 3 * 4": "2 * 3 * 4",
|
||||
"snake_case_name": "snake_case_name",
|
||||
"**unclosed": "**unclosed",
|
||||
`\*not em\*`: "*not em*",
|
||||
"[no link]": "[no link]",
|
||||
} {
|
||||
sp := ParseInline(in)
|
||||
if len(sp) != 1 || sp[0].Text != want || sp[0].Em || sp[0].Strong {
|
||||
t.Errorf("%q gave %+v", in, sp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkdownIncremental streams the sample byte by byte and checks finished blocks are kept.
|
||||
func TestMarkdownIncremental(t *testing.T) {
|
||||
var m Markdown
|
||||
for i := 1; i <= len(mdSample); i++ {
|
||||
before := append([]MdBlock(nil), m.Blocks...)
|
||||
from := m.Update(mdSample[:i])
|
||||
if from < len(before)-1 {
|
||||
t.Fatalf("at %d reparsed from %d of %d blocks", i, from, len(before))
|
||||
}
|
||||
for j := 0; j < from; j++ {
|
||||
if !reflect.DeepEqual(m.Blocks[j], before[j]) {
|
||||
t.Fatalf("at %d finished block %d changed", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
var whole Markdown
|
||||
whole.Update(mdSample)
|
||||
if !reflect.DeepEqual(m.Blocks, whole.Blocks) {
|
||||
t.Fatal("streamed parse differs from a whole parse")
|
||||
}
|
||||
if m.Update(mdSample) != len(m.Blocks) {
|
||||
t.Fatal("an unchanged source parsed again")
|
||||
}
|
||||
if m.Update("# Other") != 0 || len(m.Blocks) != 1 {
|
||||
t.Fatal("a replaced source kept old blocks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownOpenFence(t *testing.T) {
|
||||
var m Markdown
|
||||
m.Update("```py\nprint(1)\n")
|
||||
if len(m.Blocks) != 1 || !m.Blocks[0].Open || m.Blocks[0].Code != "print(1)" {
|
||||
t.Fatalf("open fence %+v", m.Blocks)
|
||||
}
|
||||
m.Update("```py\nprint(1)\n```\nafter")
|
||||
if len(m.Blocks) != 2 || m.Blocks[0].Open || !strings.Contains(m.Blocks[1].Spans[0].Text, "after") {
|
||||
t.Fatalf("closed fence %+v", m.Blocks)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user