mirror of https://github.com/usememos/memos
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
28 lines
366 B
Go
28 lines
366 B
Go
package tokenizer
|
|
|
|
type TokenType = string
|
|
|
|
const (
|
|
Underline TokenType = "_"
|
|
Star TokenType = "*"
|
|
Newline TokenType = "\n"
|
|
Hash TokenType = "#"
|
|
Space TokenType = " "
|
|
)
|
|
|
|
const (
|
|
Text TokenType = ""
|
|
)
|
|
|
|
type Token struct {
|
|
Type TokenType
|
|
Value string
|
|
}
|
|
|
|
func NewToken(tp, text string) *Token {
|
|
return &Token{
|
|
Type: tp,
|
|
Value: text,
|
|
}
|
|
}
|