diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go index b5390f3e..c69d6a6a 100644 --- a/plugin/gomark/ast/block.go +++ b/plugin/gomark/ast/block.go @@ -64,3 +64,26 @@ type Blockquote struct { func (*Blockquote) Type() NodeType { return BlockquoteNode } + +type OrderedList struct { + BaseBlock + + Number string + Children []Node +} + +func (*OrderedList) Type() NodeType { + return OrderedListNode +} + +type UnorderedList struct { + BaseBlock + + // Symbol is "*" or "-" or "+". + Symbol string + Children []Node +} + +func (*UnorderedList) Type() NodeType { + return UnorderedListNode +} diff --git a/plugin/gomark/ast/node.go b/plugin/gomark/ast/node.go index 6fbb23b5..9e4c5315 100644 --- a/plugin/gomark/ast/node.go +++ b/plugin/gomark/ast/node.go @@ -11,6 +11,8 @@ const ( HeadingNode HorizontalRuleNode BlockquoteNode + OrderedListNode + UnorderedListNode // Inline nodes. TextNode BoldNode diff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.go index 530b8a99..f1c1603f 100644 --- a/plugin/gomark/parser/bold.go +++ b/plugin/gomark/parser/bold.go @@ -23,7 +23,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) { return 0, false } prefixTokenType := prefixTokens[0].Type - if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline { + if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore { return 0, false } diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.go index 44f43a3e..5c43bab9 100644 --- a/plugin/gomark/parser/bold_italic.go +++ b/plugin/gomark/parser/bold_italic.go @@ -23,7 +23,7 @@ func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) { return 0, false } prefixTokenType := prefixTokens[0].Type - if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline { + if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore { return 0, false } diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.go index 284c8882..bd79f146 100644 --- a/plugin/gomark/parser/heading.go +++ b/plugin/gomark/parser/heading.go @@ -16,7 +16,7 @@ func NewHeadingParser() *HeadingParser { func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) { cursor := 0 for _, token := range tokens { - if token.Type == tokenizer.Hash { + if token.Type == tokenizer.PoundSign { cursor++ } else { break @@ -57,7 +57,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { level := 0 for _, token := range tokens { - if token.Type == tokenizer.Hash { + if token.Type == tokenizer.PoundSign { level++ } else { break diff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.go index a3db6d1d..75d6c50a 100644 --- a/plugin/gomark/parser/horizontal_rule.go +++ b/plugin/gomark/parser/horizontal_rule.go @@ -20,7 +20,7 @@ func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) { if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type { return 0, false } - if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk { + if tokens[0].Type != tokenizer.Hyphen && tokens[0].Type != tokenizer.Underscore && tokens[0].Type != tokenizer.Asterisk { return 0, false } if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline { diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.go index 1b29afeb..6a80496c 100644 --- a/plugin/gomark/parser/italic.go +++ b/plugin/gomark/parser/italic.go @@ -21,7 +21,7 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) { } prefixTokens := tokens[:1] - if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline { + if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore { return 0, false } prefixTokenType := prefixTokens[0].Type diff --git a/plugin/gomark/parser/ordered_list.go b/plugin/gomark/parser/ordered_list.go new file mode 100644 index 00000000..4e01b476 --- /dev/null +++ b/plugin/gomark/parser/ordered_list.go @@ -0,0 +1,54 @@ +package parser + +import ( + "errors" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type OrderedListParser struct{} + +func NewOrderedListParser() *OrderedListParser { + return &OrderedListParser{} +} + +func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 4 { + return 0, false + } + if tokens[0].Type != tokenizer.Number || tokens[1].Type != tokenizer.Dot || tokens[2].Type != tokenizer.Space { + return 0, false + } + + contentTokens := []*tokenizer.Token{} + for _, token := range tokens[3:] { + contentTokens = append(contentTokens, token) + if token.Type == tokenizer.Newline { + break + } + } + + if len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 3, true +} + +func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil, errors.New("not matched") + } + + contentTokens := tokens[3:size] + children, err := ParseInline(contentTokens) + if err != nil { + return nil, err + } + return &ast.OrderedList{ + Number: tokens[0].Value, + Children: children, + }, nil +} diff --git a/plugin/gomark/parser/ordered_list_test.go b/plugin/gomark/parser/ordered_list_test.go new file mode 100644 index 00000000..374c317a --- /dev/null +++ b/plugin/gomark/parser/ordered_list_test.go @@ -0,0 +1,58 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestOrderedListParser(t *testing.T) { + tests := []struct { + text string + node ast.Node + }{ + { + text: "1.asd", + node: nil, + }, + { + text: "1. Hello World", + node: &ast.OrderedList{ + Number: "1", + Children: []ast.Node{ + &ast.Text{ + Content: "Hello World", + }, + }, + }, + }, + { + text: "1aa. Hello World", + node: nil, + }, + { + text: "22. Hello *World*", + node: &ast.OrderedList{ + Number: "22", + Children: []ast.Node{ + &ast.Text{ + Content: "Hello ", + }, + &ast.Italic{ + Symbol: "*", + Content: "World", + }, + }, + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + node, _ := NewOrderedListParser().Parse(tokens) + require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + } +} diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.go index 3deccad3..9a7d872f 100644 --- a/plugin/gomark/parser/tag.go +++ b/plugin/gomark/parser/tag.go @@ -17,12 +17,12 @@ func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) { if len(tokens) < 2 { return 0, false } - if tokens[0].Type != tokenizer.Hash { + if tokens[0].Type != tokenizer.PoundSign { return 0, false } contentTokens := []*tokenizer.Token{} for _, token := range tokens[1:] { - if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.Hash { + if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign { break } contentTokens = append(contentTokens, token) diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.go index ce373001..e138ba82 100644 --- a/plugin/gomark/parser/tokenizer/tokenizer.go +++ b/plugin/gomark/parser/tokenizer/tokenizer.go @@ -3,9 +3,9 @@ package tokenizer type TokenType = string const ( - Underline TokenType = "_" + Underscore TokenType = "_" Asterisk TokenType = "*" - Hash TokenType = "#" + PoundSign TokenType = "#" Backtick TokenType = "`" LeftSquareBracket TokenType = "[" RightSquareBracket TokenType = "]" @@ -13,14 +13,17 @@ const ( RightParenthesis TokenType = ")" ExclamationMark TokenType = "!" Tilde TokenType = "~" - Dash TokenType = "-" + Hyphen TokenType = "-" + PlusSign TokenType = "+" + Dot TokenType = "." GreaterThan TokenType = ">" Newline TokenType = "\n" Space TokenType = " " ) const ( - Text TokenType = "" + Number TokenType = "number" + Text TokenType = "" ) type Token struct { @@ -40,11 +43,11 @@ func Tokenize(text string) []*Token { for _, c := range text { switch c { case '_': - tokens = append(tokens, NewToken(Underline, "_")) + tokens = append(tokens, NewToken(Underscore, "_")) case '*': tokens = append(tokens, NewToken(Asterisk, "*")) case '#': - tokens = append(tokens, NewToken(Hash, "#")) + tokens = append(tokens, NewToken(PoundSign, "#")) case '`': tokens = append(tokens, NewToken(Backtick, "`")) case '[': @@ -60,9 +63,13 @@ func Tokenize(text string) []*Token { case '~': tokens = append(tokens, NewToken(Tilde, "~")) case '-': - tokens = append(tokens, NewToken(Dash, "-")) + tokens = append(tokens, NewToken(Hyphen, "-")) case '>': tokens = append(tokens, NewToken(GreaterThan, ">")) + case '+': + tokens = append(tokens, NewToken(PlusSign, "+")) + case '.': + tokens = append(tokens, NewToken(Dot, ".")) case '\n': tokens = append(tokens, NewToken(Newline, "\n")) case ' ': @@ -72,10 +79,19 @@ func Tokenize(text string) []*Token { if len(tokens) > 0 { prevToken = tokens[len(tokens)-1] } - if prevToken == nil || prevToken.Type != Text { - tokens = append(tokens, NewToken(Text, string(c))) + + isNumber := c >= '0' && c <= '9' + if prevToken != nil { + if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) { + prevToken.Value += string(c) + continue + } + } + + if isNumber { + tokens = append(tokens, NewToken(Number, string(c))) } else { - prevToken.Value += string(c) + tokens = append(tokens, NewToken(Text, string(c))) } } } diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.go index a85651b3..284dba93 100644 --- a/plugin/gomark/parser/tokenizer/tokenizer_test.go +++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go @@ -41,7 +41,7 @@ func TestTokenize(t *testing.T) { world`, tokens: []*Token{ { - Type: Hash, + Type: PoundSign, Value: "#", }, { diff --git a/plugin/gomark/parser/unordered_list.go b/plugin/gomark/parser/unordered_list.go new file mode 100644 index 00000000..c64daeb1 --- /dev/null +++ b/plugin/gomark/parser/unordered_list.go @@ -0,0 +1,55 @@ +package parser + +import ( + "errors" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type UnorderedListParser struct{} + +func NewUnorderedListParser() *UnorderedListParser { + return &UnorderedListParser{} +} + +func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 3 { + return 0, false + } + symbolToken := tokens[0] + if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || tokens[1].Type != tokenizer.Space { + return 0, false + } + + contentTokens := []*tokenizer.Token{} + for _, token := range tokens[2:] { + if token.Type == tokenizer.Newline { + break + } + contentTokens = append(contentTokens, token) + } + if len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 2, true +} + +func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil, errors.New("not matched") + } + + symbolToken := tokens[0] + contentTokens := tokens[2:size] + children, err := ParseInline(contentTokens) + if err != nil { + return nil, err + } + return &ast.UnorderedList{ + Symbol: symbolToken.Type, + Children: children, + }, nil +} diff --git a/plugin/gomark/parser/unordered_list_test.go b/plugin/gomark/parser/unordered_list_test.go new file mode 100644 index 00000000..982d58be --- /dev/null +++ b/plugin/gomark/parser/unordered_list_test.go @@ -0,0 +1,51 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestUnorderedListParser(t *testing.T) { + tests := []struct { + text string + node ast.Node + }{ + { + text: "*asd", + node: nil, + }, + { + text: "+ Hello World", + node: &ast.UnorderedList{ + Symbol: tokenizer.PlusSign, + Children: []ast.Node{ + &ast.Text{ + Content: "Hello World", + }, + }, + }, + }, + { + text: "* **Hello**", + node: &ast.UnorderedList{ + Symbol: tokenizer.Asterisk, + Children: []ast.Node{ + &ast.Bold{ + Symbol: "*", + Content: "Hello", + }, + }, + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + node, _ := NewUnorderedListParser().Parse(tokens) + require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + } +} diff --git a/plugin/gomark/render/html/html.go b/plugin/gomark/render/html/html.go index aedbd222..46f90bc6 100644 --- a/plugin/gomark/render/html/html.go +++ b/plugin/gomark/render/html/html.go @@ -159,7 +159,7 @@ func (r *HTMLRender) renderLink(node *ast.Link) { func (r *HTMLRender) renderTag(node *ast.Tag) { r.output.WriteString(``) - r.output.WriteString(`# `) + r.output.WriteString(`#`) r.output.WriteString(node.Content) r.output.WriteString(``) } diff --git a/plugin/gomark/render/html/html_test.go b/plugin/gomark/render/html/html_test.go index 9c22097a..25164254 100644 --- a/plugin/gomark/render/html/html_test.go +++ b/plugin/gomark/render/html/html_test.go @@ -30,6 +30,10 @@ func TestHTMLRender(t *testing.T) { text: "**Hello** world!", expected: `

Hello world!

`, }, + { + text: "#article #memo", + expected: `

#article #memo

`, + }, } for _, test := range tests {