tokens.go

Documentation: github.com/noirbizarre/gonja/tokens

     1  package tokens
     2  
     3  import "fmt"
     4  
     5  // TokenType identifies the type of a token
     6  type Type int
     7  
     8  // Known tokens
     9  const (
    10  	Error Type = iota
    11  	Add
    12  	Assign
    13  	Colon
    14  	Comma
    15  	Div
    16  	Dot
    17  	Eq
    18  	// EqEq
    19  	Floordiv
    20  	Gt
    21  	Gteq
    22  	Lbrace
    23  	Lbracket
    24  	Lparen
    25  	Lt
    26  	Lteq
    27  	// Not
    28  	// And
    29  	// Or
    30  	// Neq
    31  	Mod
    32  	Mul
    33  	Ne
    34  	Pipe
    35  	Pow
    36  	Rbrace
    37  	Rbracket
    38  	Rparen
    39  	Semicolon
    40  	Sub
    41  	Tilde
    42  	Whitespace
    43  	Float
    44  	Integer
    45  	Name
    46  	String
    47  	Operator
    48  	BlockBegin
    49  	BlockEnd
    50  	VariableBegin
    51  	VariableEnd
    52  	RawBegin
    53  	RawEnd
    54  	CommentBegin
    55  	CommentEnd
    56  	Comment
    57  	LinestatementBegin
    58  	LinestatementEnd
    59  	LinecommentBegin
    60  	LinecommentEnd
    61  	Linecomment
    62  	Data
    63  	Initial
    64  	EOF
    65  )
    66  
    67  // TokenNames maps token types to their human readable name
    68  var Names = map[Type]string{
    69  	Error:  "Error",
    70  	Add:    "Add",
    71  	Assign: "Assign",
    72  	Colon:  "Colon",
    73  	Comma:  "Comma",
    74  	Div:    "Div",
    75  	Dot:    "Dot",
    76  	Eq:     "Eq",
    77  	// EqEq:     "EqEq",
    78  	Floordiv: "Floordiv",
    79  	Gt:       "Gt",
    80  	Gteq:     "Gteq",
    81  	Lbrace:   "Lbrace",
    82  	Lbracket: "Lbracket",
    83  	Lparen:   "Lparen",
    84  	Lt:       "Lt",
    85  	Lteq:     "Lteq",
    86  	// Not:                "Not",
    87  	// And:                "And",
    88  	// Or:                 "Or",
    89  	// Neq:                "Neq",
    90  	Mod:                "Mod",
    91  	Mul:                "Mul",
    92  	Ne:                 "Ne",
    93  	Pipe:               "Pipe",
    94  	Pow:                "Pow",
    95  	Rbrace:             "Rbrace",
    96  	Rbracket:           "Rbracket",
    97  	Rparen:             "Rparen",
    98  	Semicolon:          "Semicolon",
    99  	Sub:                "Sub",
   100  	Tilde:              "Tilde",
   101  	Whitespace:         "Whitespace",
   102  	Float:              "Float",
   103  	Integer:            "Integer",
   104  	Name:               "Name",
   105  	String:             "String",
   106  	Operator:           "Operator",
   107  	BlockBegin:         "BlockBegin",
   108  	BlockEnd:           "BlockEnd",
   109  	VariableBegin:      "VariableBegin",
   110  	VariableEnd:        "VariableEnd",
   111  	RawBegin:           "RawBegin",
   112  	RawEnd:             "RawEnd",
   113  	CommentBegin:       "CommentBegin",
   114  	CommentEnd:         "CommentEnd",
   115  	Comment:            "Comment",
   116  	LinestatementBegin: "LinestatementBegin",
   117  	LinestatementEnd:   "LinestatementEnd",
   118  	LinecommentBegin:   "LinecommentBegin",
   119  	LinecommentEnd:     "LinecommentEnd",
   120  	Linecomment:        "Linecomment",
   121  	Data:               "Data",
   122  	Initial:            "Initial",
   123  	EOF:                "EOF",
   124  }
   125  
   126  // var SymbolTokens = map[TokenType]bool {
   127  
   128  // }
   129  
   130  // var KeywordTokens = map[TokenType]bool {
   131  
   132  // }
   133  
   134  // var NumberTokens = map[TokenType]bool {
   135  
   136  // }
   137  
   138  // Token represents a unit of lexing
   139  type Token struct {
   140  	Type Type
   141  	Val  string
   142  	Pos  int
   143  	Line int
   144  	Col  int
   145  }
   146  
   147  func (t Token) String() string {
   148  	val := t.Val
   149  	if len(val) > 1000 {
   150  		val = fmt.Sprintf("%s...%s", val[:10], val[len(val)-5:len(val)])
   151  	}
   152  
   153  	return fmt.Sprintf("<Token[%s] Val='%s' Pos=%d Line=%d Col=%d>",
   154  		Names[t.Type], val, t.Pos, t.Line, t.Col)
   155  }
   156
View as plain text