tokenizer

package

v0.0.0-...-a164c29 Latest Latest Go to latest Published: Feb 25, 2022 License: MIT Imports: 2 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/Nv7-Github/gold

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
type Op
- func (o Op) String() string
type Paren
type Pos
- func NewPos(filename string) *Pos
type Stream
- func NewStream(filename string, code string) *Stream
type Token
type TokenType
- func (t TokenType) String() string
type Tokenizer
- func NewTokenizer(stream *Stream) *Tokenizer

Constants ¶

View Source

const Assign = "=>"

View Source

const BlockEnd = "end"

View Source

const BlockStart = "do"

View Source

const Else = "else"

View Source

const False = "false"

View Source

const True = "true"

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type Op ¶

type Op rune

const (
	Add Op = '+'
	Sub Op = '-'
	Mul Op = '*'
	Div Op = '/'
	Mod Op = '%'

	Eq Op = '='
	Ne Op = '!'
	Lt Op = '<'
	Gt Op = '>'

	And Op = '&'
	Or  Op = '|'
)

func (Op) String ¶

func (o Op) String() string

type Paren ¶

type Paren rune

const (
	LParen Paren = '('
	RParen Paren = ')'
	LBrack Paren = '['
	RBrack Paren = ']'
)

type Pos ¶

type Pos struct {
	Line     int
	Col      int
	Filename string
}

func NewPos ¶

func NewPos(filename string) *Pos

func (*Pos) Dup ¶

func (p *Pos) Dup() *Pos

func (*Pos) Error ¶

func (p *Pos) Error(msg string, args ...interface{}) error

func (*Pos) NextCol ¶

func (p *Pos) NextCol()

func (*Pos) NextLine ¶

func (p *Pos) NextLine()

func (*Pos) String ¶

func (p *Pos) String() string

type Stream ¶

type Stream struct {
	// contains filtered or unexported fields
}

func NewStream ¶

func NewStream(filename string, code string) *Stream

func (*Stream) CodePos ¶

func (s *Stream) CodePos() *Pos

func (*Stream) Eat ¶

func (s *Stream) Eat(amount int)

func (*Stream) HasNext ¶

func (s *Stream) HasNext() bool

func (*Stream) Peek ¶

func (s *Stream) Peek(off int) rune

type Token ¶

type Token struct {
	Type  TokenType
	Value string
	Pos   *Pos
}

type TokenType ¶

type TokenType int

const (
	StringLiteral TokenType = iota
	NumberLiteral
	BoolLiteral
	Identifier
	Operator
	Operation
	Parenthesis
	Not
	End
)

func (TokenType) String ¶

func (t TokenType) String() string

type Tokenizer ¶

type Tokenizer struct {
	Tokens []Token
	// contains filtered or unexported fields
}

func NewTokenizer ¶

func NewTokenizer(stream *Stream) *Tokenizer

func (*Tokenizer) CurrPos ¶

func (t *Tokenizer) CurrPos() *Pos

func (*Tokenizer) CurrTok ¶

func (t *Tokenizer) CurrTok() Token

func (*Tokenizer) Eat ¶

func (t *Tokenizer) Eat() bool

func (*Tokenizer) Filename ¶

func (t *Tokenizer) Filename() string

func (*Tokenizer) IsEnd ¶

func (t *Tokenizer) IsEnd() bool

func (*Tokenizer) Tokenize ¶

func (t *Tokenizer) Tokenize()

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL