feat: implement gomark parsers

pull/2620/head
Steven 2 years ago
parent 2d9c5d16e1
commit 453707d18c

@ -1,6 +1,7 @@
package ast
type BaseBlock struct {
Node
}
type LineBreak struct {
@ -50,3 +51,28 @@ var NodeTypeHeading = NewNodeType("Heading")
func (*Heading) Type() NodeType {
return NodeTypeHeading
}
type HorizontalRule struct {
BaseBlock
// Symbol is "*" or "-" or "_".
Symbol string
}
var NodeTypeHorizontalRule = NewNodeType("HorizontalRule")
func (*HorizontalRule) Type() NodeType {
return NodeTypeHorizontalRule
}
type Blockquote struct {
BaseBlock
Children []Node
}
var NodeTypeBlockquote = NewNodeType("Blockquote")
func (*Blockquote) Type() NodeType {
return NodeTypeBlockquote
}

@ -1,6 +1,8 @@
package ast
type BaseInline struct{}
type BaseInline struct {
Node
}
type Text struct {
BaseInline
@ -28,6 +30,34 @@ func (*Bold) Type() NodeType {
return NodeTypeBold
}
type Italic struct {
BaseInline
// Symbol is "*" or "_"
Symbol string
Content string
}
var NodeTypeItalic = NewNodeType("Italic")
func (*Italic) Type() NodeType {
return NodeTypeItalic
}
type BoldItalic struct {
BaseInline
// Symbol is "*" or "_"
Symbol string
Content string
}
var NodeTypeBoldItalic = NewNodeType("BoldItalic")
func (*BoldItalic) Type() NodeType {
return NodeTypeBoldItalic
}
type Code struct {
BaseInline
@ -66,28 +96,26 @@ func (*Link) Type() NodeType {
return NodeTypeLink
}
type Italic struct {
type Tag struct {
BaseInline
// Symbol is "*" or "_"
Symbol string
Content string
}
var NodeTypeItalic = NewNodeType("Italic")
var NodeTypeTag = NewNodeType("Tag")
func (*Italic) Type() NodeType {
return NodeTypeItalic
func (*Tag) Type() NodeType {
return NodeTypeTag
}
type Tag struct {
type Strikethrough struct {
BaseInline
Content string
}
var NodeTypeTag = NewNodeType("Tag")
var NodeTypeStrikethrough = NewNodeType("Strikethrough")
func (*Tag) Type() NodeType {
return NodeTypeTag
func (*Strikethrough) Type() NodeType {
return NodeTypeStrikethrough
}

@ -0,0 +1,47 @@
package parser
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type BlockquoteParser struct{}
func NewBlockquoteParser() *BlockquoteParser {
return &BlockquoteParser{}
}
func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 4 {
return 0, false
}
if tokens[0].Type != tokenizer.GreaterThan || tokens[1].Type != tokenizer.Space {
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[2:] {
if token.Type == tokenizer.Newline {
break
}
contentTokens = append(contentTokens, token)
}
if len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 2, true
}
func (p *BlockquoteParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
contentTokens := tokens[2:size]
children := ParseInline(contentTokens)
return &ast.Blockquote{
Children: children,
}
}

@ -0,0 +1,47 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestBlockquoteParser(t *testing.T) {
tests := []struct {
text string
blockquote ast.Node
}{
{
text: "> Hello world",
blockquote: &ast.Blockquote{
Children: []ast.Node{
&ast.Text{
Content: "Hello world",
},
},
},
},
{
text: "> Hello\nworld",
blockquote: &ast.Blockquote{
Children: []ast.Node{
&ast.Text{
Content: "Hello",
},
},
},
},
{
text: ">Hello\nworld",
blockquote: nil,
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.blockquote, NewBlockquoteParser().Parse(tokens))
}
}

@ -7,10 +7,8 @@ import (
type BoldParser struct{}
var defaultBoldParser = &BoldParser{}
func NewBoldParser() InlineParser {
return defaultBoldParser
return &BoldParser{}
}
func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
@ -23,7 +21,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
if prefixTokenType != tokenizer.Star && prefixTokenType != tokenizer.Underline {
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
return 0, false
}

@ -0,0 +1,58 @@
package parser
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type BoldItalicParser struct{}
func NewBoldItalicParser() InlineParser {
return &BoldItalicParser{}
}
func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 7 {
return 0, false
}
prefixTokens := tokens[:3]
if prefixTokens[0].Type != prefixTokens[1].Type || prefixTokens[0].Type != prefixTokens[2].Type || prefixTokens[1].Type != prefixTokens[2].Type {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
return 0, false
}
cursor, matched := 3, false
for ; cursor < len(tokens)-2; cursor++ {
token, nextToken, endToken := tokens[cursor], tokens[cursor+1], tokens[cursor+2]
if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline || endToken.Type == tokenizer.Newline {
return 0, false
}
if token.Type == prefixTokenType && nextToken.Type == prefixTokenType && endToken.Type == prefixTokenType {
matched = true
break
}
}
if !matched {
return 0, false
}
return cursor + 3, true
}
func (p *BoldItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
prefixTokenType := tokens[0].Type
contentTokens := tokens[3 : size-3]
return &ast.BoldItalic{
Symbol: prefixTokenType,
Content: tokenizer.Stringify(contentTokens),
}
}

@ -0,0 +1,49 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestBoldItalicParser(t *testing.T) {
tests := []struct {
text string
boldItalic ast.Node
}{
{
text: "*Hello world!",
boldItalic: nil,
},
{
text: "***Hello***",
boldItalic: &ast.BoldItalic{
Symbol: "*",
Content: "Hello",
},
},
{
text: "*** Hello ***",
boldItalic: &ast.BoldItalic{
Symbol: "*",
Content: " Hello ",
},
},
{
text: "*** Hello * *",
boldItalic: nil,
},
{
text: "*** Hello **",
boldItalic: nil,
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.boldItalic, NewBoldItalicParser().Parse(tokens))
}
}

@ -7,10 +7,8 @@ import (
type CodeParser struct{}
var defaultCodeParser = &CodeParser{}
func NewCodeParser() *CodeParser {
return defaultCodeParser
return &CodeParser{}
}
func (*CodeParser) Match(tokens []*tokenizer.Token) (int, bool) {

@ -10,10 +10,8 @@ type CodeBlockParser struct {
Content string
}
var defaultCodeBlockParser = &CodeBlockParser{}
func NewCodeBlockParser() *CodeBlockParser {
return defaultCodeBlockParser
return &CodeBlockParser{}
}
func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (int, bool) {

@ -62,11 +62,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) ast.Node {
}
}
contentTokens := tokens[level+1 : size]
children := ParseInline(contentTokens, []InlineParser{
NewBoldParser(),
NewCodeParser(),
NewTextParser(),
})
children := ParseInline(contentTokens)
return &ast.Heading{
Level: level,
Children: children,

@ -0,0 +1,39 @@
package parser
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type HorizontalRuleParser struct{}
func NewHorizontalRuleParser() *HorizontalRuleParser {
return &HorizontalRuleParser{}
}
func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 3 {
return 0, false
}
if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {
return 0, false
}
if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {
return 0, false
}
if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {
return 0, false
}
return 3, true
}
func (p *HorizontalRuleParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &ast.HorizontalRule{
Symbol: tokens[0].Type,
}
}

@ -0,0 +1,49 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestHorizontalRuleParser(t *testing.T) {
tests := []struct {
text string
horizontalRule ast.Node
}{
{
text: "---",
horizontalRule: &ast.HorizontalRule{
Symbol: "-",
},
},
{
text: "****",
horizontalRule: nil,
},
{
text: "***",
horizontalRule: &ast.HorizontalRule{
Symbol: "*",
},
},
{
text: "-*-",
horizontalRule: nil,
},
{
text: "___",
horizontalRule: &ast.HorizontalRule{
Symbol: "_",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.horizontalRule, NewHorizontalRuleParser().Parse(tokens))
}
}

@ -7,10 +7,8 @@ import (
type ImageParser struct{}
var defaultImageParser = &ImageParser{}
func NewImageParser() *ImageParser {
return defaultImageParser
return &ImageParser{}
}
func (*ImageParser) Match(tokens []*tokenizer.Token) (int, bool) {

@ -1,6 +1,9 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type ItalicParser struct {
ContentTokens []*tokenizer.Token
@ -10,21 +13,21 @@ func NewItalicParser() *ItalicParser {
return &ItalicParser{}
}
func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {
func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 3 {
return nil
return 0, false
}
prefixTokens := tokens[:1]
if prefixTokens[0].Type != tokenizer.Star && prefixTokens[0].Type != tokenizer.Underline {
return nil
if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
contentTokens := []*tokenizer.Token{}
matched := false
for _, token := range tokens[1:] {
if token.Type == tokenizer.Newline {
return nil
return 0, false
}
if token.Type == prefixTokenType {
matched = true
@ -33,10 +36,22 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {
contentTokens = append(contentTokens, token)
}
if !matched || len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 2, true
}
func (p *ItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &ItalicParser{
ContentTokens: contentTokens,
prefixTokenType := tokens[0].Type
contentTokens := tokens[1 : size-1]
return &ast.Italic{
Symbol: prefixTokenType,
Content: tokenizer.Stringify(contentTokens),
}
}

@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestItalicParser(t *testing.T) {
tests := []struct {
text string
italic *ItalicParser
italic ast.Node
}{
{
text: "*Hello world!",
@ -19,76 +20,29 @@ func TestItalicParser(t *testing.T) {
},
{
text: "*Hello*",
italic: &ItalicParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "Hello",
},
},
italic: &ast.Italic{
Symbol: "*",
Content: "Hello",
},
},
{
text: "* Hello *",
italic: &ItalicParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "Hello",
},
{
Type: tokenizer.Space,
Value: " ",
},
},
italic: &ast.Italic{
Symbol: "*",
Content: " Hello ",
},
},
{
text: "** Hello * *",
italic: nil,
},
{
text: "*1* Hello * *",
italic: &ItalicParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "1",
},
},
italic: &ast.Italic{
Symbol: "*",
Content: "1",
},
},
{
text: `* \n * Hello * *`,
italic: &ItalicParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: `\n`,
},
{
Type: tokenizer.Space,
Value: " ",
},
},
},
},
{
text: "* \n * Hello * *",
italic: nil,
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.italic, NewItalicParser().Match(tokens))
require.Equal(t, test.italic, NewItalicParser().Parse(tokens))
}
}

@ -7,10 +7,8 @@ import (
type LineBreakParser struct{}
var defaultLineBreakParser = &LineBreakParser{}
func NewLineBreakParser() *LineBreakParser {
return defaultLineBreakParser
return &LineBreakParser{}
}
func (*LineBreakParser) Match(tokens []*tokenizer.Token) (int, bool) {

@ -1,58 +1,72 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type LinkParser struct {
ContentTokens []*tokenizer.Token
URL string
}
type LinkParser struct{}
func NewLinkParser() *LinkParser {
return &LinkParser{}
}
func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser {
if len(tokens) < 4 {
return nil
func (*LinkParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 5 {
return 0, false
}
if tokens[0].Type != tokenizer.LeftSquareBracket {
return nil
return 0, false
}
cursor, contentTokens := 1, []*tokenizer.Token{}
for ; cursor < len(tokens)-2; cursor++ {
if tokens[cursor].Type == tokenizer.Newline {
return nil
textTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
if token.Type == tokenizer.Newline {
return 0, false
}
if tokens[cursor].Type == tokenizer.RightSquareBracket {
if token.Type == tokenizer.RightSquareBracket {
break
}
contentTokens = append(contentTokens, tokens[cursor])
textTokens = append(textTokens, token)
}
if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
return nil
if len(textTokens)+4 >= len(tokens) {
return 0, false
}
if tokens[2+len(textTokens)].Type != tokenizer.LeftParenthesis {
return 0, false
}
matched, url := false, ""
for _, token := range tokens[cursor+2:] {
urlTokens := []*tokenizer.Token{}
for _, token := range tokens[3+len(textTokens):] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
return nil
return 0, false
}
if token.Type == tokenizer.RightParenthesis {
matched = true
break
}
url += token.Value
urlTokens = append(urlTokens, token)
}
if 4+len(urlTokens)+len(textTokens) > len(tokens) {
return 0, false
}
if !matched || url == "" {
return 4 + len(urlTokens) + len(textTokens), true
}
func (p *LinkParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
if len(contentTokens) == 0 {
contentTokens = append(contentTokens, &tokenizer.Token{
Type: tokenizer.Text,
Value: url,
})
textTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
if token.Type == tokenizer.RightSquareBracket {
break
}
textTokens = append(textTokens, token)
}
return &LinkParser{
ContentTokens: contentTokens,
URL: url,
urlTokens := tokens[2+len(textTokens)+1 : size-1]
return &ast.Link{
Text: tokenizer.Stringify(textTokens),
URL: tokenizer.Stringify(urlTokens),
}
}

@ -5,24 +5,20 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestLinkParser(t *testing.T) {
tests := []struct {
text string
link *LinkParser
link ast.Node
}{
{
text: "[](https://example.com)",
link: &LinkParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "https://example.com",
},
},
URL: "https://example.com",
link: &ast.Link{
Text: "",
URL: "https://example.com",
},
},
{
@ -35,27 +31,14 @@ func TestLinkParser(t *testing.T) {
},
{
text: "[hello world](https://example.com)",
link: &LinkParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "hello",
},
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "world",
},
},
URL: "https://example.com",
link: &ast.Link{
Text: "hello world",
URL: "https://example.com",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.link, NewLinkParser().Match(tokens))
require.Equal(t, test.link, NewLinkParser().Parse(tokens))
}
}

@ -9,10 +9,8 @@ type ParagraphParser struct {
ContentTokens []*tokenizer.Token
}
var defaultParagraphParser = &ParagraphParser{}
func NewParagraphParser() *ParagraphParser {
return defaultParagraphParser
return &ParagraphParser{}
}
func (*ParagraphParser) Match(tokens []*tokenizer.Token) (int, bool) {
@ -38,10 +36,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) ast.Node {
}
contentTokens := tokens[:size]
children := ParseInline(contentTokens, []InlineParser{
NewBoldParser(),
NewTextParser(),
})
children := ParseInline(contentTokens)
return &ast.Paragraph{
Children: children,
}

@ -1,6 +1,8 @@
package parser
import (
"errors"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
@ -23,32 +25,51 @@ type BlockParser interface {
BaseParser
}
func Parse(tokens []*tokenizer.Token) []ast.Node {
var defaultBlockParsers = []BlockParser{
NewCodeBlockParser(),
NewHorizontalRuleParser(),
NewHeadingParser(),
NewBlockquoteParser(),
NewParagraphParser(),
NewLineBreakParser(),
}
func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) {
nodes := []ast.Node{}
blockParsers := []BlockParser{
NewCodeBlockParser(),
NewParagraphParser(),
NewLineBreakParser(),
}
for len(tokens) > 0 {
for _, blockParser := range blockParsers {
for _, blockParser := range defaultBlockParsers {
cursor, matched := blockParser.Match(tokens)
if matched {
node := blockParser.Parse(tokens)
if node == nil {
return nil, errors.New("parse error")
}
nodes = append(nodes, node)
tokens = tokens[cursor:]
break
}
}
}
return nodes
return nodes, nil
}
var defaultInlineParsers = []InlineParser{
NewBoldItalicParser(),
NewImageParser(),
NewLinkParser(),
NewBoldParser(),
NewItalicParser(),
NewCodeParser(),
NewTagParser(),
NewStrikethroughParser(),
NewTextParser(),
}
func ParseInline(tokens []*tokenizer.Token, inlineParsers []InlineParser) []ast.Node {
func ParseInline(tokens []*tokenizer.Token) []ast.Node {
nodes := []ast.Node{}
var lastNode ast.Node
for len(tokens) > 0 {
for _, inlineParser := range inlineParsers {
for _, inlineParser := range defaultInlineParsers {
cursor, matched := inlineParser.Match(tokens)
if matched {
node := inlineParser.Parse(tokens)

@ -89,6 +89,8 @@ func TestParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.nodes, Parse(tokens))
nodes, err := Parse(tokens)
require.NoError(t, err)
require.Equal(t, test.nodes, nodes)
}
}

@ -0,0 +1,49 @@
package parser
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type StrikethroughParser struct{}
func NewStrikethroughParser() *StrikethroughParser {
return &StrikethroughParser{}
}
func (*StrikethroughParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 5 {
return 0, false
}
if tokens[0].Type != tokenizer.Tilde || tokens[1].Type != tokenizer.Tilde {
return 0, false
}
cursor, matched := 2, false
for ; cursor < len(tokens)-1; cursor++ {
token, nextToken := tokens[cursor], tokens[cursor+1]
if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline {
return 0, false
}
if token.Type == tokenizer.Tilde && nextToken.Type == tokenizer.Tilde {
matched = true
break
}
}
if !matched {
return 0, false
}
return cursor + 2, true
}
func (p *StrikethroughParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
contentTokens := tokens[2 : size-2]
return &ast.Strikethrough{
Content: tokenizer.Stringify(contentTokens),
}
}

@ -0,0 +1,45 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestStrikethroughParser(t *testing.T) {
tests := []struct {
text string
strikethrough ast.Node
}{
{
text: "~~Hello world",
strikethrough: nil,
},
{
text: "~~Hello~~",
strikethrough: &ast.Strikethrough{
Content: "Hello",
},
},
{
text: "~~ Hello ~~",
strikethrough: &ast.Strikethrough{
Content: " Hello ",
},
},
{
text: "~~1~~ Hello ~~~",
strikethrough: &ast.Strikethrough{
Content: "1",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.strikethrough, NewStrikethroughParser().Parse(tokens))
}
}

@ -1,21 +1,22 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type TagParser struct {
ContentTokens []*tokenizer.Token
}
type TagParser struct{}
func NewTagParser() *TagParser {
return &TagParser{}
}
func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {
func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 2 {
return nil
return 0, false
}
if tokens[0].Type != tokenizer.Hash {
return nil
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
@ -25,10 +26,20 @@ func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {
contentTokens = append(contentTokens, token)
}
if len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 1, true
}
func (p *TagParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &TagParser{
ContentTokens: contentTokens,
contentTokens := tokens[1:size]
return &ast.Tag{
Content: tokenizer.Stringify(contentTokens),
}
}

@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestTagParser(t *testing.T) {
tests := []struct {
text string
tag *TagParser
tag ast.Node
}{
{
text: "*Hello world",
@ -23,30 +24,20 @@ func TestTagParser(t *testing.T) {
},
{
text: "#tag",
tag: &TagParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "tag",
},
},
tag: &ast.Tag{
Content: "tag",
},
},
{
text: "#tag/subtag",
tag: &TagParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "tag/subtag",
},
},
text: "#tag/subtag 123",
tag: &ast.Tag{
Content: "tag/subtag",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.tag, NewTagParser().Match(tokens))
require.Equal(t, test.tag, NewTagParser().Parse(tokens))
}
}

@ -9,10 +9,8 @@ type TextParser struct {
Content string
}
var defaultTextParser = &TextParser{}
func NewTextParser() *TextParser {
return defaultTextParser
return &TextParser{}
}
func (*TextParser) Match(tokens []*tokenizer.Token) (int, bool) {

@ -4,7 +4,7 @@ type TokenType = string
const (
Underline TokenType = "_"
Star TokenType = "*"
Asterisk TokenType = "*"
Hash TokenType = "#"
Backtick TokenType = "`"
LeftSquareBracket TokenType = "["
@ -12,6 +12,9 @@ const (
LeftParenthesis TokenType = "("
RightParenthesis TokenType = ")"
ExclamationMark TokenType = "!"
Tilde TokenType = "~"
Dash TokenType = "-"
GreaterThan TokenType = ">"
Newline TokenType = "\n"
Space TokenType = " "
)
@ -39,7 +42,7 @@ func Tokenize(text string) []*Token {
case '_':
tokens = append(tokens, NewToken(Underline, "_"))
case '*':
tokens = append(tokens, NewToken(Star, "*"))
tokens = append(tokens, NewToken(Asterisk, "*"))
case '#':
tokens = append(tokens, NewToken(Hash, "#"))
case '`':
@ -54,6 +57,12 @@ func Tokenize(text string) []*Token {
tokens = append(tokens, NewToken(RightParenthesis, ")"))
case '!':
tokens = append(tokens, NewToken(ExclamationMark, "!"))
case '~':
tokens = append(tokens, NewToken(Tilde, "~"))
case '-':
tokens = append(tokens, NewToken(Dash, "-"))
case '>':
tokens = append(tokens, NewToken(GreaterThan, ">"))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case ' ':

@ -15,7 +15,7 @@ func TestTokenize(t *testing.T) {
text: "*Hello world!",
tokens: []*Token{
{
Type: Star,
Type: Asterisk,
Value: "*",
},
{

Loading…
Cancel
Save