mirror of https://github.com/usememos/memos
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 lines
651 B
Go
29 lines
651 B
Go
package tokenizer
|
|
|
|
func tokenize(text string) []*Token {
|
|
tokens := []*Token{}
|
|
for _, c := range text {
|
|
switch c {
|
|
case '_':
|
|
tokens = append(tokens, NewToken(Underline, "_"))
|
|
case '*':
|
|
tokens = append(tokens, NewToken(Star, "*"))
|
|
case '\n':
|
|
tokens = append(tokens, NewToken(Newline, "\n"))
|
|
case ' ':
|
|
tokens = append(tokens, NewToken(Space, " "))
|
|
default:
|
|
var lastToken *Token
|
|
if len(tokens) > 0 {
|
|
lastToken = tokens[len(tokens)-1]
|
|
}
|
|
if lastToken == nil || lastToken.Type != Text {
|
|
tokens = append(tokens, NewToken(Text, string(c)))
|
|
} else {
|
|
lastToken.Value += string(c)
|
|
}
|
|
}
|
|
}
|
|
return tokens
|
|
}
|