From 46f7cffc7b4265a7fba3effd91db9246dcb15d4a Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 28 Dec 2023 22:35:39 +0800 Subject: [PATCH] feat: implement restore nodes --- plugin/gomark/ast/ast.go | 59 ++++------------ plugin/gomark/ast/block.go | 70 +++++++++++++++++++ plugin/gomark/ast/inline.go | 58 ++++++++++++++- plugin/gomark/parser/auto_link.go | 5 +- plugin/gomark/parser/auto_link_test.go | 6 +- plugin/gomark/parser/blockquote.go | 2 +- plugin/gomark/parser/blockquote_test.go | 3 +- plugin/gomark/parser/bold_italic_test.go | 3 +- plugin/gomark/parser/bold_test.go | 3 +- plugin/gomark/parser/code_block.go | 5 +- plugin/gomark/parser/code_block_test.go | 3 +- plugin/gomark/parser/code_test.go | 3 +- .../gomark/parser/escaping_character_test.go | 3 +- plugin/gomark/parser/heading.go | 5 +- plugin/gomark/parser/heading_test.go | 3 +- plugin/gomark/parser/horizontal_rule_test.go | 3 +- plugin/gomark/parser/image_test.go | 3 +- plugin/gomark/parser/italic_test.go | 3 +- plugin/gomark/parser/link_test.go | 3 +- plugin/gomark/parser/ordered_list.go | 5 +- plugin/gomark/parser/ordered_list_test.go | 3 +- plugin/gomark/parser/paragraph.go | 8 +-- plugin/gomark/parser/paragraph_test.go | 3 +- plugin/gomark/parser/parser.go | 10 +++ plugin/gomark/parser/parser_test.go | 60 +--------------- plugin/gomark/parser/strikethrough_test.go | 3 +- plugin/gomark/parser/tag_test.go | 3 +- plugin/gomark/parser/task_list.go | 5 +- plugin/gomark/parser/task_list_test.go | 3 +- plugin/gomark/parser/unordered_list.go | 5 +- plugin/gomark/parser/unordered_list_test.go | 3 +- plugin/gomark/renderer/html/html.go | 2 +- plugin/gomark/restore/restore.go | 14 ++++ plugin/gomark/restore/restore_test.go | 48 +++++++++++++ 34 files changed, 264 insertions(+), 154 deletions(-) create mode 100644 plugin/gomark/restore/restore.go create mode 100644 plugin/gomark/restore/restore_test.go diff --git a/plugin/gomark/ast/ast.go b/plugin/gomark/ast/ast.go index 8f136a3c..5c7d5f2c 100644 --- a/plugin/gomark/ast/ast.go +++ b/plugin/gomark/ast/ast.go @@ -28,57 +28,13 @@ const ( EscapingCharacterNode ) -func (t NodeType) String() string { - switch t { - case LineBreakNode: - return "LineBreakNode" - case ParagraphNode: - return "ParagraphNode" - case CodeBlockNode: - return "CodeBlockNode" - case HeadingNode: - return "HeadingNode" - case HorizontalRuleNode: - return "HorizontalRuleNode" - case BlockquoteNode: - return "BlockquoteNode" - case OrderedListNode: - return "OrderedListNode" - case UnorderedListNode: - return "UnorderedListNode" - case TaskListNode: - return "TaskListNode" - case TextNode: - return "TextNode" - case BoldNode: - return "BoldNode" - case ItalicNode: - return "ItalicNode" - case BoldItalicNode: - return "BoldItalicNode" - case CodeNode: - return "CodeNode" - case ImageNode: - return "ImageNode" - case LinkNode: - return "LinkNode" - case AutoLinkNode: - return "AutoLinkNode" - case TagNode: - return "TagNode" - case StrikethroughNode: - return "StrikethroughNode" - case EscapingCharacterNode: - return "EscapingCharacterNode" - default: - return "UnknownNode" - } -} - type Node interface { // Type returns a node type. Type() NodeType + // Restore returns a string representation of this node. + Restore() string + // PrevSibling returns a previous sibling node of this node. PrevSibling() Node @@ -113,3 +69,12 @@ func (n *BaseNode) SetPrevSibling(node Node) { func (n *BaseNode) SetNextSibling(node Node) { n.nextSibling = node } + +func IsBlockNode(node Node) bool { + switch node.Type() { + case ParagraphNode, CodeBlockNode, HeadingNode, HorizontalRuleNode, BlockquoteNode, OrderedListNode, UnorderedListNode, TaskListNode: + return true + default: + return false + } +} diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go index 2b3ec4c7..d292cc2d 100644 --- a/plugin/gomark/ast/block.go +++ b/plugin/gomark/ast/block.go @@ -1,5 +1,7 @@ package ast +import "fmt" + type BaseBlock struct { BaseNode } @@ -12,6 +14,10 @@ func (*LineBreak) Type() NodeType { return LineBreakNode } +func (*LineBreak) Restore() string { + return "\n" +} + type Paragraph struct { BaseBlock @@ -22,6 +28,14 @@ func (*Paragraph) Type() NodeType { return ParagraphNode } +func (n *Paragraph) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + return result +} + type CodeBlock struct { BaseBlock @@ -33,6 +47,10 @@ func (*CodeBlock) Type() NodeType { return CodeBlockNode } +func (n *CodeBlock) Restore() string { + return fmt.Sprintf("```%s\n%s\n```", n.Language, n.Content) +} + type Heading struct { BaseBlock @@ -44,6 +62,18 @@ func (*Heading) Type() NodeType { return HeadingNode } +func (n *Heading) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + symbol := "" + for i := 0; i < n.Level; i++ { + symbol += "#" + } + return fmt.Sprintf("%s %s", symbol, result) +} + type HorizontalRule struct { BaseBlock @@ -55,6 +85,10 @@ func (*HorizontalRule) Type() NodeType { return HorizontalRuleNode } +func (n *HorizontalRule) Restore() string { + return n.Symbol + n.Symbol + n.Symbol +} + type Blockquote struct { BaseBlock @@ -65,6 +99,14 @@ func (*Blockquote) Type() NodeType { return BlockquoteNode } +func (n *Blockquote) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + return fmt.Sprintf("> %s", result) +} + type OrderedList struct { BaseBlock @@ -76,6 +118,14 @@ func (*OrderedList) Type() NodeType { return OrderedListNode } +func (n *OrderedList) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + return fmt.Sprintf("%s. %s", n.Number, result) +} + type UnorderedList struct { BaseBlock @@ -88,6 +138,14 @@ func (*UnorderedList) Type() NodeType { return UnorderedListNode } +func (n *UnorderedList) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + return fmt.Sprintf("%s %s", n.Symbol, result) +} + type TaskList struct { BaseBlock @@ -100,3 +158,15 @@ type TaskList struct { func (*TaskList) Type() NodeType { return TaskListNode } + +func (n *TaskList) Restore() string { + var result string + for _, child := range n.Children { + result += child.Restore() + } + complete := " " + if n.Complete { + complete = "x" + } + return fmt.Sprintf("%s [%s] %s", n.Symbol, complete, result) +} diff --git a/plugin/gomark/ast/inline.go b/plugin/gomark/ast/inline.go index 4e8dfe62..af1d065b 100644 --- a/plugin/gomark/ast/inline.go +++ b/plugin/gomark/ast/inline.go @@ -1,5 +1,7 @@ package ast +import "fmt" + type BaseInline struct { BaseNode } @@ -14,6 +16,10 @@ func (*Text) Type() NodeType { return TextNode } +func (n *Text) Restore() string { + return n.Content +} + type Bold struct { BaseInline @@ -26,6 +32,15 @@ func (*Bold) Type() NodeType { return BoldNode } +func (n *Bold) Restore() string { + symbol := n.Symbol + n.Symbol + children := "" + for _, child := range n.Children { + children += child.Restore() + } + return fmt.Sprintf("%s%s%s", symbol, children, symbol) +} + type Italic struct { BaseInline @@ -38,6 +53,10 @@ func (*Italic) Type() NodeType { return ItalicNode } +func (n *Italic) Restore() string { + return fmt.Sprintf("%s%s%s", n.Symbol, n.Content, n.Symbol) +} + type BoldItalic struct { BaseInline @@ -50,6 +69,11 @@ func (*BoldItalic) Type() NodeType { return BoldItalicNode } +func (n *BoldItalic) Restore() string { + symbol := n.Symbol + n.Symbol + n.Symbol + return fmt.Sprintf("%s%s%s", symbol, n.Content, symbol) +} + type Code struct { BaseInline @@ -60,6 +84,10 @@ func (*Code) Type() NodeType { return CodeNode } +func (n *Code) Restore() string { + return fmt.Sprintf("`%s`", n.Content) +} + type Image struct { BaseInline @@ -71,6 +99,10 @@ func (*Image) Type() NodeType { return ImageNode } +func (n *Image) Restore() string { + return fmt.Sprintf("![%s](%s)", n.AltText, n.URL) +} + type Link struct { BaseInline @@ -82,16 +114,28 @@ func (*Link) Type() NodeType { return LinkNode } +func (n *Link) Restore() string { + return fmt.Sprintf("[%s](%s)", n.Text, n.URL) +} + type AutoLink struct { BaseInline - URL string + URL string + IsRawText bool } func (*AutoLink) Type() NodeType { return AutoLinkNode } +func (n *AutoLink) Restore() string { + if n.IsRawText { + return n.URL + } + return fmt.Sprintf("<%s>", n.URL) +} + type Tag struct { BaseInline @@ -102,6 +146,10 @@ func (*Tag) Type() NodeType { return TagNode } +func (n *Tag) Restore() string { + return fmt.Sprintf("<%s>", n.Content) +} + type Strikethrough struct { BaseInline @@ -112,6 +160,10 @@ func (*Strikethrough) Type() NodeType { return StrikethroughNode } +func (n *Strikethrough) Restore() string { + return fmt.Sprintf("~~%s~~", n.Content) +} + type EscapingCharacter struct { BaseInline @@ -121,3 +173,7 @@ type EscapingCharacter struct { func (*EscapingCharacter) Type() NodeType { return EscapingCharacterNode } + +func (n *EscapingCharacter) Restore() string { + return fmt.Sprintf("\\%s", n.Symbol) +} diff --git a/plugin/gomark/parser/auto_link.go b/plugin/gomark/parser/auto_link.go index 7acd7e20..48ba22ff 100644 --- a/plugin/gomark/parser/auto_link.go +++ b/plugin/gomark/parser/auto_link.go @@ -57,10 +57,13 @@ func (p *AutoLinkParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { } url := tokenizer.Stringify(tokens[:size]) + isRawText := true if tokens[0].Type == tokenizer.LessThan && tokens[len(tokens)-1].Type == tokenizer.GreaterThan { + isRawText = false url = tokenizer.Stringify(tokens[1 : len(tokens)-1]) } return &ast.AutoLink{ - URL: url, + URL: url, + IsRawText: isRawText, }, nil } diff --git a/plugin/gomark/parser/auto_link_test.go b/plugin/gomark/parser/auto_link_test.go index 582a26bd..1fd529c0 100644 --- a/plugin/gomark/parser/auto_link_test.go +++ b/plugin/gomark/parser/auto_link_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestAutoLinkParser(t *testing.T) { @@ -27,7 +28,8 @@ func TestAutoLinkParser(t *testing.T) { { text: "https://example.com", link: &ast.AutoLink{ - URL: "https://example.com", + URL: "https://example.com", + IsRawText: true, }, }, } @@ -35,6 +37,6 @@ func TestAutoLinkParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewAutoLinkParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.link}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.link}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/blockquote.go b/plugin/gomark/parser/blockquote.go index 479c7ace..f318bfc4 100644 --- a/plugin/gomark/parser/blockquote.go +++ b/plugin/gomark/parser/blockquote.go @@ -23,10 +23,10 @@ func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens[2:] { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { return 0, false diff --git a/plugin/gomark/parser/blockquote_test.go b/plugin/gomark/parser/blockquote_test.go index c7568074..d786f943 100644 --- a/plugin/gomark/parser/blockquote_test.go +++ b/plugin/gomark/parser/blockquote_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestBlockquoteParser(t *testing.T) { @@ -51,6 +52,6 @@ func TestBlockquoteParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewBlockquoteParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.blockquote}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.blockquote}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/bold_italic_test.go b/plugin/gomark/parser/bold_italic_test.go index b31790e5..83cf91b6 100644 --- a/plugin/gomark/parser/bold_italic_test.go +++ b/plugin/gomark/parser/bold_italic_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestBoldItalicParser(t *testing.T) { @@ -45,6 +46,6 @@ func TestBoldItalicParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewBoldItalicParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.boldItalic}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.boldItalic}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/bold_test.go b/plugin/gomark/parser/bold_test.go index 418ce719..02872049 100644 --- a/plugin/gomark/parser/bold_test.go +++ b/plugin/gomark/parser/bold_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestBoldParser(t *testing.T) { @@ -53,6 +54,6 @@ func TestBoldParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewBoldParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.bold}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.bold}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/code_block.go b/plugin/gomark/parser/code_block.go index 4eaf3a74..534b693f 100644 --- a/plugin/gomark/parser/code_block.go +++ b/plugin/gomark/parser/code_block.go @@ -40,7 +40,7 @@ func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (int, bool) { matched = true break } else if tokens[cursor+4].Type == tokenizer.Newline { - cursor += 5 + cursor += 4 matched = true break } @@ -65,9 +65,6 @@ func (p *CodeBlockParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { languageToken = nil contentStart = 4 } - if tokens[size-1].Type == tokenizer.Newline { - contentEnd = size - 5 - } codeBlock := &ast.CodeBlock{ Content: tokenizer.Stringify(tokens[contentStart:contentEnd]), diff --git a/plugin/gomark/parser/code_block_test.go b/plugin/gomark/parser/code_block_test.go index 4a46b9aa..bc5b13fd 100644 --- a/plugin/gomark/parser/code_block_test.go +++ b/plugin/gomark/parser/code_block_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestCodeBlockParser(t *testing.T) { @@ -59,6 +60,6 @@ func TestCodeBlockParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewCodeBlockParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.codeBlock}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.codeBlock}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/code_test.go b/plugin/gomark/parser/code_test.go index 8c4822d8..6dc28d43 100644 --- a/plugin/gomark/parser/code_test.go +++ b/plugin/gomark/parser/code_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestCodeParser(t *testing.T) { @@ -33,6 +34,6 @@ func TestCodeParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewCodeParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.code}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.code}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/escaping_character_test.go b/plugin/gomark/parser/escaping_character_test.go index da6b2d31..7c66db8b 100644 --- a/plugin/gomark/parser/escaping_character_test.go +++ b/plugin/gomark/parser/escaping_character_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestEscapingCharacterParser(t *testing.T) { @@ -25,6 +26,6 @@ func TestEscapingCharacterParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewEscapingCharacterParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.node}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.go index 9e6754a0..0c98afb9 100644 --- a/plugin/gomark/parser/heading.go +++ b/plugin/gomark/parser/heading.go @@ -34,10 +34,10 @@ func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens[level+1:] { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { return 0, false @@ -62,9 +62,6 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { } contentTokens := tokens[level+1 : size] - if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { - contentTokens = contentTokens[:len(contentTokens)-1] - } children, err := ParseInline(contentTokens) if err != nil { return nil, err diff --git a/plugin/gomark/parser/heading_test.go b/plugin/gomark/parser/heading_test.go index a68bb460..df3aaccc 100644 --- a/plugin/gomark/parser/heading_test.go +++ b/plugin/gomark/parser/heading_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestHeadingParser(t *testing.T) { @@ -80,6 +81,6 @@ Hello World`, for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewHeadingParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.heading}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.heading}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/horizontal_rule_test.go b/plugin/gomark/parser/horizontal_rule_test.go index d191ae8f..3822f80f 100644 --- a/plugin/gomark/parser/horizontal_rule_test.go +++ b/plugin/gomark/parser/horizontal_rule_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestHorizontalRuleParser(t *testing.T) { @@ -51,6 +52,6 @@ func TestHorizontalRuleParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewHorizontalRuleParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.horizontalRule}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.horizontalRule}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/image_test.go b/plugin/gomark/parser/image_test.go index f88e2a69..7bbf1fae 100644 --- a/plugin/gomark/parser/image_test.go +++ b/plugin/gomark/parser/image_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestImageParser(t *testing.T) { @@ -40,6 +41,6 @@ func TestImageParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewImageParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.image}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.image}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/italic_test.go b/plugin/gomark/parser/italic_test.go index f3627f71..5c60c442 100644 --- a/plugin/gomark/parser/italic_test.go +++ b/plugin/gomark/parser/italic_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestItalicParser(t *testing.T) { @@ -44,6 +45,6 @@ func TestItalicParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewItalicParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.italic}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.italic}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/link_test.go b/plugin/gomark/parser/link_test.go index 24287bdc..f7565094 100644 --- a/plugin/gomark/parser/link_test.go +++ b/plugin/gomark/parser/link_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestLinkParser(t *testing.T) { @@ -47,6 +48,6 @@ func TestLinkParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewLinkParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.link}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.link}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/ordered_list.go b/plugin/gomark/parser/ordered_list.go index d0d4343a..da3f7173 100644 --- a/plugin/gomark/parser/ordered_list.go +++ b/plugin/gomark/parser/ordered_list.go @@ -23,10 +23,10 @@ func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens[3:] { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { @@ -43,9 +43,6 @@ func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { } contentTokens := tokens[3:size] - if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { - contentTokens = contentTokens[:len(contentTokens)-1] - } children, err := ParseInline(contentTokens) if err != nil { return nil, err diff --git a/plugin/gomark/parser/ordered_list_test.go b/plugin/gomark/parser/ordered_list_test.go index 374c317a..4b1bcbcf 100644 --- a/plugin/gomark/parser/ordered_list_test.go +++ b/plugin/gomark/parser/ordered_list_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestOrderedListParser(t *testing.T) { @@ -53,6 +54,6 @@ func TestOrderedListParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewOrderedListParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.node}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/paragraph.go b/plugin/gomark/parser/paragraph.go index 8ac5a5a1..40a6fe42 100644 --- a/plugin/gomark/parser/paragraph.go +++ b/plugin/gomark/parser/paragraph.go @@ -18,10 +18,10 @@ func NewParagraphParser() *ParagraphParser { func (*ParagraphParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { return 0, false @@ -38,11 +38,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { return nil, errors.New("not matched") } - contentTokens := tokens[:size] - if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { - contentTokens = contentTokens[:len(contentTokens)-1] - } - children, err := ParseInline(contentTokens) + children, err := ParseInline(tokens[:size]) if err != nil { return nil, err } diff --git a/plugin/gomark/parser/paragraph_test.go b/plugin/gomark/parser/paragraph_test.go index cf154dd2..60a972f4 100644 --- a/plugin/gomark/parser/paragraph_test.go +++ b/plugin/gomark/parser/paragraph_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestParagraphParser(t *testing.T) { @@ -57,6 +58,6 @@ func TestParagraphParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewParagraphParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.paragraph}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.paragraph}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.go index c3e7633c..133bbad3 100644 --- a/plugin/gomark/parser/parser.go +++ b/plugin/gomark/parser/parser.go @@ -48,6 +48,7 @@ func ParseBlock(tokens []*tokenizer.Token) ([]ast.Node, error) { func ParseBlockWithParsers(tokens []*tokenizer.Token, blockParsers []BlockParser) ([]ast.Node, error) { nodes := []ast.Node{} var prevNode ast.Node + var skipNextLineBreakFlag bool for len(tokens) > 0 { for _, blockParser := range blockParsers { size, matched := blockParser.Match(tokens) @@ -57,12 +58,21 @@ func ParseBlockWithParsers(tokens []*tokenizer.Token, blockParsers []BlockParser return nil, errors.New("parse error") } + if node.Type() == ast.LineBreakNode && skipNextLineBreakFlag { + if prevNode != nil && ast.IsBlockNode(prevNode) { + tokens = tokens[size:] + skipNextLineBreakFlag = false + break + } + } + tokens = tokens[size:] if prevNode != nil { prevNode.SetNextSibling(node) node.SetPrevSibling(prevNode) } prevNode = node + skipNextLineBreakFlag = true nodes = append(nodes, node) break } diff --git a/plugin/gomark/parser/parser_test.go b/plugin/gomark/parser/parser_test.go index 827e6dad..2d5effab 100644 --- a/plugin/gomark/parser/parser_test.go +++ b/plugin/gomark/parser/parser_test.go @@ -1,13 +1,13 @@ package parser import ( - "strconv" "testing" "github.com/stretchr/testify/require" "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestParser(t *testing.T) { @@ -202,62 +202,6 @@ func TestParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) nodes, _ := Parse(tokens) - require.Equal(t, StringifyNodes(test.nodes), StringifyNodes(nodes)) + require.Equal(t, restore.Restore(test.nodes), restore.Restore(nodes)) } } - -func StringifyNodes(nodes []ast.Node) string { - var result string - for _, node := range nodes { - if node != nil { - result += StringifyNode(node) - } - } - return result -} - -func StringifyNode(node ast.Node) string { - switch n := node.(type) { - case *ast.LineBreak: - return "LineBreak()" - case *ast.CodeBlock: - return "CodeBlock(" + n.Language + ", " + n.Content + ")" - case *ast.Paragraph: - return "Paragraph(" + StringifyNodes(n.Children) + ")" - case *ast.Heading: - return "Heading(" + StringifyNodes(n.Children) + ")" - case *ast.HorizontalRule: - return "HorizontalRule(" + n.Symbol + ")" - case *ast.Blockquote: - return "Blockquote(" + StringifyNodes(n.Children) + ")" - case *ast.OrderedList: - return "OrderedList(" + n.Number + ", " + StringifyNodes(n.Children) + ")" - case *ast.UnorderedList: - return "UnorderedList(" + n.Symbol + ", " + StringifyNodes(n.Children) + ")" - case *ast.TaskList: - return "TaskList(" + n.Symbol + ", " + strconv.FormatBool(n.Complete) + ", " + StringifyNodes(n.Children) + ")" - case *ast.Text: - return "Text(" + n.Content + ")" - case *ast.Bold: - return "Bold(" + n.Symbol + StringifyNodes(n.Children) + n.Symbol + ")" - case *ast.Italic: - return "Italic(" + n.Symbol + n.Content + n.Symbol + ")" - case *ast.BoldItalic: - return "BoldItalic(" + n.Symbol + n.Content + n.Symbol + ")" - case *ast.Code: - return "Code(" + n.Content + ")" - case *ast.Image: - return "Image(" + n.URL + ", " + n.AltText + ")" - case *ast.Link: - return "Link(" + n.Text + ", " + n.URL + ")" - case *ast.AutoLink: - return "AutoLink(" + n.URL + ")" - case *ast.Tag: - return "Tag(" + n.Content + ")" - case *ast.Strikethrough: - return "Strikethrough(" + n.Content + ")" - case *ast.EscapingCharacter: - return "EscapingCharacter(" + n.Symbol + ")" - } - return "" -} diff --git a/plugin/gomark/parser/strikethrough_test.go b/plugin/gomark/parser/strikethrough_test.go index 7e9efe6f..9a67a9ac 100644 --- a/plugin/gomark/parser/strikethrough_test.go +++ b/plugin/gomark/parser/strikethrough_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestStrikethroughParser(t *testing.T) { @@ -41,6 +42,6 @@ func TestStrikethroughParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewStrikethroughParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.strikethrough}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.strikethrough}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/tag_test.go b/plugin/gomark/parser/tag_test.go index cc746742..7dbb1bc0 100644 --- a/plugin/gomark/parser/tag_test.go +++ b/plugin/gomark/parser/tag_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestTagParser(t *testing.T) { @@ -39,6 +40,6 @@ func TestTagParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewTagParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.tag}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.tag}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/task_list.go b/plugin/gomark/parser/task_list.go index abb62399..b46c1910 100644 --- a/plugin/gomark/parser/task_list.go +++ b/plugin/gomark/parser/task_list.go @@ -34,10 +34,10 @@ func (*TaskListParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens[6:] { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { return 0, false @@ -54,9 +54,6 @@ func (p *TaskListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { symbolToken := tokens[0] contentTokens := tokens[6:size] - if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { - contentTokens = contentTokens[:len(contentTokens)-1] - } children, err := ParseInline(contentTokens) if err != nil { return nil, err diff --git a/plugin/gomark/parser/task_list_test.go b/plugin/gomark/parser/task_list_test.go index 3676264b..d92775e4 100644 --- a/plugin/gomark/parser/task_list_test.go +++ b/plugin/gomark/parser/task_list_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestTaskListParser(t *testing.T) { @@ -52,6 +53,6 @@ func TestTaskListParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewTaskListParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.node}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/parser/unordered_list.go b/plugin/gomark/parser/unordered_list.go index 95e02cfd..c64daeb1 100644 --- a/plugin/gomark/parser/unordered_list.go +++ b/plugin/gomark/parser/unordered_list.go @@ -24,10 +24,10 @@ func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) { contentTokens := []*tokenizer.Token{} for _, token := range tokens[2:] { - contentTokens = append(contentTokens, token) if token.Type == tokenizer.Newline { break } + contentTokens = append(contentTokens, token) } if len(contentTokens) == 0 { return 0, false @@ -44,9 +44,6 @@ func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) symbolToken := tokens[0] contentTokens := tokens[2:size] - if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { - contentTokens = contentTokens[:len(contentTokens)-1] - } children, err := ParseInline(contentTokens) if err != nil { return nil, err diff --git a/plugin/gomark/parser/unordered_list_test.go b/plugin/gomark/parser/unordered_list_test.go index e3a7174c..0d4b1112 100644 --- a/plugin/gomark/parser/unordered_list_test.go +++ b/plugin/gomark/parser/unordered_list_test.go @@ -7,6 +7,7 @@ import ( "github.com/usememos/memos/plugin/gomark/ast" "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" ) func TestUnorderedListParser(t *testing.T) { @@ -50,6 +51,6 @@ func TestUnorderedListParser(t *testing.T) { for _, test := range tests { tokens := tokenizer.Tokenize(test.text) node, _ := NewUnorderedListParser().Parse(tokens) - require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + require.Equal(t, restore.Restore([]ast.Node{test.node}), restore.Restore([]ast.Node{node})) } } diff --git a/plugin/gomark/renderer/html/html.go b/plugin/gomark/renderer/html/html.go index 5422f286..4a4fd8be 100644 --- a/plugin/gomark/renderer/html/html.go +++ b/plugin/gomark/renderer/html/html.go @@ -83,7 +83,7 @@ func (r *HTMLRenderer) Render(astRoot []ast.Node) string { return r.output.String() } -func (r *HTMLRenderer) renderLineBreak(_ *ast.LineBreak) { +func (r *HTMLRenderer) renderLineBreak(*ast.LineBreak) { r.output.WriteString("
") } diff --git a/plugin/gomark/restore/restore.go b/plugin/gomark/restore/restore.go new file mode 100644 index 00000000..cdf91bdc --- /dev/null +++ b/plugin/gomark/restore/restore.go @@ -0,0 +1,14 @@ +package restore + +import "github.com/usememos/memos/plugin/gomark/ast" + +func Restore(nodes []ast.Node) string { + var result string + for _, node := range nodes { + if node == nil { + continue + } + result += node.Restore() + } + return result +} diff --git a/plugin/gomark/restore/restore_test.go b/plugin/gomark/restore/restore_test.go new file mode 100644 index 00000000..ad3dc0b8 --- /dev/null +++ b/plugin/gomark/restore/restore_test.go @@ -0,0 +1,48 @@ +package restore + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" +) + +func TestRestore(t *testing.T) { + tests := []struct { + nodes []ast.Node + rawText string + }{ + { + nodes: nil, + rawText: "", + }, + { + nodes: []ast.Node{ + &ast.Text{ + Content: "Hello world!", + }, + }, + rawText: "Hello world!", + }, + { + nodes: []ast.Node{ + &ast.Paragraph{ + Children: []ast.Node{ + &ast.Text{ + Content: "Here: ", + }, + &ast.Code{ + Content: "Hello world!", + }, + }, + }, + }, + rawText: "Here: `Hello world!`", + }, + } + + for _, test := range tests { + require.Equal(t, Restore(test.nodes), test.rawText) + } +}