1 package tokens
2
3 import "fmt"
4
5 type Stream struct {
6 it TokenIterator
7 previous *Token
8 current *Token
9 next *Token
10 backup *Token
11 buffer []*Token
12 tokens []*Token
13 }
14
15 type TokenIterator interface {
16 Next() *Token
17 }
18
19 type chanIterator struct {
20 input chan *Token
21 }
22
23 func ChanIterator(input chan *Token) TokenIterator {
24 return &chanIterator{input}
25 }
26
27 func (ci *chanIterator) Next() *Token {
28 return <-ci.input
29 }
30
31 type sliceIterator struct {
32 input []*Token
33 idx int
34 }
35
36 func SliceIterator(input []*Token) TokenIterator {
37 length := len(input)
38 var last *Token
39 if length > 0 {
40 last = input[length-1]
41 }
42 if last == nil || last.Type != EOF {
43 input = append(input, &Token{Type: EOF})
44 }
45 return &sliceIterator{input, 0}
46 }
47
48 func (si *sliceIterator) Next() *Token {
49 if si.idx < len(si.input) {
50 tok := si.input[si.idx]
51 si.idx++
52 return tok
53 } else {
54 return nil
55 }
56 }
57
58 func NewStream(input interface{}) *Stream {
59 var it TokenIterator
60
61 switch t := input.(type) {
62 case chan *Token:
63 it = ChanIterator(t)
64 case []*Token:
65 it = SliceIterator(t)
66 default:
67 panic(fmt.Sprintf(`Unsupported stream input type "%T"`, t))
68 }
69
70 s := &Stream{
71 it: it,
72 buffer: []*Token{},
73 tokens: []*Token{},
74 }
75 s.init()
76 return s
77 }
78
79 func (s *Stream) init() {
80 s.current = s.nonIgnored()
81 if !s.End() {
82 s.next = s.nonIgnored()
83 }
84 }
85
86 func (s *Stream) nonIgnored() *Token {
87 var tok *Token
88 for tok = s.it.Next(); tok.Type == Whitespace; tok = s.it.Next() {
89 }
90 return tok
91 }
92
93 func (s *Stream) consume() *Token {
94 s.previous = s.current
95 s.current = s.next
96 if s.backup != nil {
97 s.next = s.backup
98 s.backup = nil
99 } else if s.End() {
100 s.next = nil
101 } else {
102 s.next = s.nonIgnored()
103 }
104 return s.previous
105 }
106
107 func (s *Stream) Current() *Token {
108 return s.current
109 }
110
111 func (s *Stream) Next() *Token {
112 return s.consume()
113 }
114
115 func (s *Stream) EOF() bool {
116 return s.current.Type == EOF
117 }
118
119 func (s *Stream) IsError() bool {
120 return s.current.Type == Error
121 }
122
123 func (s *Stream) End() bool {
124 return s.EOF() || s.IsError()
125 }
126
127 func (s *Stream) Peek() *Token {
128 return s.next
129 }
130
131 func (s *Stream) Backup() {
132 if s.previous == nil {
133 panic("Can't backup")
134 }
135 s.backup = s.next
136 s.next = s.current
137 s.current = s.previous
138 s.previous = nil
139 }
140
View as plain text