aiden
/
memos
mirror of https://github.com/usememos/memos
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: implement part of nodes

Browse Source
pull/2620/head
Steven 1 year ago
parent dd83782522
commit b20e0097cf
16 changed files with 227 additions and 142 deletions
Show Stats Download Patch File Download Diff File

30
plugin/gomark/ast/block.go
Unescape Escape View File

@ -9,10 +9,6 @@ type LineBreak struct {
var NodeTypeLineBreak = NewNodeType("LineBreak")
func NewLineBreak() *LineBreak {
return &LineBreak{}
}
func (*LineBreak) Type() NodeType {
return NodeTypeLineBreak
}
@ -25,12 +21,6 @@ type Paragraph struct {
var NodeTypeParagraph = NewNodeType("Paragraph")
func NewParagraph(children []Node) *Paragraph {
return &Paragraph{
Children: children,
}
}
func (*Paragraph) Type() NodeType {
return NodeTypeParagraph
}
@ -44,13 +34,19 @@ type CodeBlock struct {
var NodeTypeCodeBlock = NewNodeType("CodeBlock")
func NewCodeBlock(language, content string) *CodeBlock {
return &CodeBlock{
Language: language,
Content: content,
}
}
func (*CodeBlock) Type() NodeType {
return NodeTypeCodeBlock
}
type Heading struct {
BaseBlock
Level int
Children []Node
}
var NodeTypeHeading = NewNodeType("Heading")
func (*Heading) Type() NodeType {
return NodeTypeHeading
}

77
plugin/gomark/ast/inline.go
Unescape Escape View File

@ -10,12 +10,6 @@ type Text struct {
var NodeTypeText = NewNodeType("Text")
func NewText(content string) *Text {
return &Text{
Content: content,
}
}
func (*Text) Type() NodeType {
return NodeTypeText
}
@ -30,13 +24,70 @@ type Bold struct {
var NodeTypeBold = NewNodeType("Bold")
func NewBold(symbol, content string) *Bold {
return &Bold{
Symbol: symbol,
Content: content,
}
}
func (*Bold) Type() NodeType {
return NodeTypeBold
}
type Code struct {
BaseInline
Content string
}
var NodeTypeCode = NewNodeType("Code")
func (*Code) Type() NodeType {
return NodeTypeCode
}
type Image struct {
BaseInline
AltText string
URL string
}
var NodeTypeImage = NewNodeType("Image")
func (*Image) Type() NodeType {
return NodeTypeImage
}
type Link struct {
BaseInline
Text string
URL string
}
var NodeTypeLink = NewNodeType("Link")
func (*Link) Type() NodeType {
return NodeTypeLink
}
type Italic struct {
BaseInline
// Symbol is "*" or "_"
Symbol string
Content string
}
var NodeTypeItalic = NewNodeType("Italic")
func (*Italic) Type() NodeType {
return NodeTypeItalic
}
type Tag struct {
BaseInline
Content string
}
var NodeTypeTag = NewNodeType("Tag")
func (*Tag) Type() NodeType {
return NodeTypeTag
}

3
plugin/gomark/parser/bold_test.go
Unescape Escape View File

@ -44,7 +44,6 @@ func TestBoldParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
parser := NewBoldParser()
require.Equal(t, test.bold, parser.Parse(tokens))
require.Equal(t, test.bold, NewBoldParser().Parse(tokens))
}
}

41
plugin/gomark/parser/code.go
Unescape Escape View File

@ -1,38 +1,51 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type CodeParser struct {
Content string
}
type CodeParser struct{}
var defaultCodeParser = &CodeParser{}
func NewCodeParser() *CodeParser {
return &CodeParser{}
return defaultCodeParser
}
func (*CodeParser) Match(tokens []*tokenizer.Token) *CodeParser {
func (*CodeParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 3 {
return nil
return 0, false
}
if tokens[0].Type != tokenizer.Backtick {
return nil
return 0, false
}
content, matched := "", false
contentTokens, matched := []*tokenizer.Token{}, false
for _, token := range tokens[1:] {
if token.Type == tokenizer.Newline {
return nil
return 0, false
}
if token.Type == tokenizer.Backtick {
matched = true
break
}
content += token.Value
contentTokens = append(contentTokens, token)
}
if !matched || len(content) == 0 {
if !matched || len(contentTokens) == 0 {
return 0, false
}
return len(contentTokens) + 2, true
}
func (p *CodeParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &CodeParser{
Content: content,
contentTokens := tokens[1 : size-1]
return &ast.Code{
Content: tokenizer.Stringify(contentTokens),
}
}

3
plugin/gomark/parser/code_block_test.go
Unescape Escape View File

@ -58,7 +58,6 @@ func TestCodeBlockParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
parser := NewCodeBlockParser()
require.Equal(t, test.codeBlock, parser.Parse(tokens))
require.Equal(t, test.codeBlock, NewCodeBlockParser().Parse(tokens))
}
}

8
plugin/gomark/parser/code_test.go
Unescape Escape View File

@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestCodeParser(t *testing.T) {
tests := []struct {
text string
code *CodeParser
code ast.Node
}{
{
text: "`Hello world!",
@ -19,7 +20,7 @@ func TestCodeParser(t *testing.T) {
},
{
text: "`Hello world!`",
code: &CodeParser{
code: &ast.Code{
Content: "Hello world!",
},
},
@ -31,7 +32,6 @@ func TestCodeParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
code := NewCodeParser()
require.Equal(t, test.code, code.Match(tokens))
require.Equal(t, test.code, NewCodeParser().Parse(tokens))
}
}

43
plugin/gomark/parser/heading.go
Unescape Escape View File

@ -1,19 +1,17 @@
package parser
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type HeadingParser struct {
Level int
ContentTokens []*tokenizer.Token
}
type HeadingParser struct{}
func NewHeadingParser() *HeadingParser {
return &HeadingParser{}
}
func (*HeadingParser) Match(tokens []*tokenizer.Token) *HeadingParser {
func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) {
cursor := 0
for _, token := range tokens {
if token.Type == tokenizer.Hash {
@ -23,14 +21,14 @@ func (*HeadingParser) Match(tokens []*tokenizer.Token) *HeadingParser {
}
}
if len(tokens) <= cursor+1 {
return nil
return 0, false
}
if tokens[cursor].Type != tokenizer.Space {
return nil
return 0, false
}
level := cursor
if level == 0 || level > 6 {
return nil
return 0, false
}
cursor++
@ -43,11 +41,34 @@ func (*HeadingParser) Match(tokens []*tokenizer.Token) *HeadingParser {
cursor++
}
if len(contentTokens) == 0 {
return 0, false
}
return cursor, true
}
func (p *HeadingParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &HeadingParser{
Level: level,
ContentTokens: contentTokens,
level := 0
for _, token := range tokens {
if token.Type == tokenizer.Hash {
level++
} else {
break
}
}
contentTokens := tokens[level+1 : size]
children := ParseInline(contentTokens, []InlineParser{
NewBoldParser(),
NewCodeParser(),
NewTextParser(),
})
return &ast.Heading{
Level: level,
Children: children,
}
}

74
plugin/gomark/parser/heading_test.go
Unescape Escape View File

@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestHeadingParser(t *testing.T) {
tests := []struct {
text string
heading *HeadingParser
heading ast.Node
}{
{
text: "*Hello world",
@ -19,48 +20,22 @@ func TestHeadingParser(t *testing.T) {
},
{
text: "## Hello World",
heading: &HeadingParser{
heading: &ast.Heading{
Level: 2,
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "Hello",
},
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "World",
Children: []ast.Node{
&ast.Text{
Content: "Hello World",
},
},
},
},
{
text: "# # Hello World",
heading: &HeadingParser{
heading: &ast.Heading{
Level: 1,
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Hash,
Value: "#",
},
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "Hello",
},
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "World",
Children: []ast.Node{
&ast.Text{
Content: "# Hello World",
},
},
},
@ -72,16 +47,26 @@ func TestHeadingParser(t *testing.T) {
{
text: `# 123
Hello World`,
heading: &HeadingParser{
heading: &ast.Heading{
Level: 1,
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "123",
Children: []ast.Node{
&ast.Text{
Content: "123 ",
},
},
},
},
{
text: "### **Hello** World",
heading: &ast.Heading{
Level: 3,
Children: []ast.Node{
&ast.Bold{
Symbol: "*",
Content: "Hello",
},
{
Type: tokenizer.Space,
Value: " ",
&ast.Text{
Content: " World",
},
},
},
@ -90,7 +75,6 @@ Hello World`,
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
heading := NewHeadingParser()
require.Equal(t, test.heading, heading.Match(tokens))
require.Equal(t, test.heading, NewHeadingParser().Parse(tokens))
}
}

60
plugin/gomark/parser/image.go
Unescape Escape View File

@ -1,30 +1,32 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
import (
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
type ImageParser struct {
AltText string
URL string
}
type ImageParser struct{}
var defaultImageParser = &ImageParser{}
func NewImageParser() *ImageParser {
return &ImageParser{}
return defaultImageParser
}
func (*ImageParser) Match(tokens []*tokenizer.Token) *ImageParser {
func (*ImageParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 5 {
return nil
return 0, false
}
if tokens[0].Type != tokenizer.ExclamationMark {
return nil
return 0, false
}
if tokens[1].Type != tokenizer.LeftSquareBracket {
return nil
return 0, false
}
cursor, altText := 2, ""
for ; cursor < len(tokens)-2; cursor++ {
if tokens[cursor].Type == tokenizer.Newline {
return nil
return 0, false
}
if tokens[cursor].Type == tokenizer.RightSquareBracket {
break
@ -32,24 +34,42 @@ func (*ImageParser) Match(tokens []*tokenizer.Token) *ImageParser {
altText += tokens[cursor].Value
}
if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
return nil
return 0, false
}
matched, url := false, ""
for _, token := range tokens[cursor+2:] {
cursor += 2
contentTokens, matched := []*tokenizer.Token{}, false
for _, token := range tokens[cursor:] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
return nil
return 0, false
}
if token.Type == tokenizer.RightParenthesis {
matched = true
break
}
url += token.Value
contentTokens = append(contentTokens, token)
}
if !matched || len(contentTokens) == 0 {
return 0, false
}
if !matched || url == "" {
return cursor + len(contentTokens) + 1, true
}
func (p *ImageParser) Parse(tokens []*tokenizer.Token) ast.Node {
size, ok := p.Match(tokens)
if size == 0 || !ok {
return nil
}
return &ImageParser{
AltText: altText,
URL: url,
altTextTokens := []*tokenizer.Token{}
for _, token := range tokens[2:] {
if token.Type == tokenizer.RightSquareBracket {
break
}
altTextTokens = append(altTextTokens, token)
}
contentTokens := tokens[2+len(altTextTokens)+2 : size-1]
return &ast.Image{
AltText: tokenizer.Stringify(altTextTokens),
URL: tokenizer.Stringify(contentTokens),
}
}

9
plugin/gomark/parser/image_test.go
Unescape Escape View File

@ -5,17 +5,18 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestImageParser(t *testing.T) {
tests := []struct {
text string
image *ImageParser
image ast.Node
}{
{
text: "![](https://example.com)",
image: &ImageParser{
image: &ast.Image{
AltText: "",
URL: "https://example.com",
},
@ -30,7 +31,7 @@ func TestImageParser(t *testing.T) {
},
{
text: "![al te](https://example.com)",
image: &ImageParser{
image: &ast.Image{
AltText: "al te",
URL: "https://example.com",
},
@ -38,6 +39,6 @@ func TestImageParser(t *testing.T) {
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.image, NewImageParser().Match(tokens))
require.Equal(t, test.image, NewImageParser().Parse(tokens))
}
}

3
plugin/gomark/parser/italic_test.go
Unescape Escape View File

@ -89,7 +89,6 @@ func TestItalicParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
italic := NewItalicParser()
require.Equal(t, test.italic, italic.Match(tokens))
require.Equal(t, test.italic, NewItalicParser().Match(tokens))
}
}

2
plugin/gomark/parser/line_break.go
Unescape Escape View File

@ -29,5 +29,5 @@ func (p *LineBreakParser) Parse(tokens []*tokenizer.Token) ast.Node {
return nil
}
return ast.NewLineBreak()
return &ast.LineBreak{}
}

4
plugin/gomark/parser/paragraph.go
Unescape Escape View File

@ -42,5 +42,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) ast.Node {
NewBoldParser(),
NewTextParser(),
})
return ast.NewParagraph(children)
return &ast.Paragraph{
Children: children,
}
}

3
plugin/gomark/parser/paragraph_test.go
Unescape Escape View File

@ -32,7 +32,6 @@ func TestParagraphParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
parser := NewParagraphParser()
require.Equal(t, test.paragraph, parser.Parse(tokens))
require.Equal(t, test.paragraph, NewParagraphParser().Parse(tokens))
}
}

3
plugin/gomark/parser/parser_test.go
Unescape Escape View File

@ -89,7 +89,6 @@ func TestParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
nodes := Parse(tokens)
require.Equal(t, test.nodes, nodes)
require.Equal(t, test.nodes, Parse(tokens))
}
}

6
plugin/gomark/parser/text.go
Unescape Escape View File

@ -24,7 +24,9 @@ func (*TextParser) Match(tokens []*tokenizer.Token) (int, bool) {
func (*TextParser) Parse(tokens []*tokenizer.Token) ast.Node {
if len(tokens) == 0 {
return ast.NewText("")
return &ast.Text{}
}
return &ast.Text{
Content: tokens[0].String(),
}
return ast.NewText(tokens[0].String())
}

Write Preview
Loading…
Cancel
Save