1
2
3
4
5 package parse
6
7 import (
8 "fmt"
9 "testing"
10 )
11
12
13 var itemName = map[itemType]string{
14 itemError: "error",
15 itemBool: "bool",
16 itemChar: "char",
17 itemCharConstant: "charconst",
18 itemComplex: "complex",
19 itemDeclare: ":=",
20 itemEOF: "EOF",
21 itemField: "field",
22 itemIdentifier: "identifier",
23 itemLeftDelim: "left delim",
24 itemLeftParen: "(",
25 itemNumber: "number",
26 itemPipe: "pipe",
27 itemRawString: "raw string",
28 itemRightDelim: "right delim",
29 itemRightParen: ")",
30 itemSpace: "space",
31 itemString: "string",
32 itemVariable: "variable",
33
34
35 itemDot: ".",
36 itemBlock: "block",
37 itemDefine: "define",
38 itemElse: "else",
39 itemIf: "if",
40 itemEnd: "end",
41 itemNil: "nil",
42 itemRange: "range",
43 itemTemplate: "template",
44 itemWith: "with",
45 }
46
47 func (i itemType) String() string {
48 s := itemName[i]
49 if s == "" {
50 return fmt.Sprintf("item%d", int(i))
51 }
52 return s
53 }
54
55 type lexTest struct {
56 name string
57 input string
58 items []item
59 }
60
61 func mkItem(typ itemType, text string) item {
62 return item{
63 typ: typ,
64 val: text,
65 }
66 }
67
68 var (
69 tDot = mkItem(itemDot, ".")
70 tBlock = mkItem(itemBlock, "block")
71 tEOF = mkItem(itemEOF, "")
72 tFor = mkItem(itemIdentifier, "for")
73 tLeft = mkItem(itemLeftDelim, "{{")
74 tLpar = mkItem(itemLeftParen, "(")
75 tPipe = mkItem(itemPipe, "|")
76 tQuote = mkItem(itemString, `"abc \n\t\" "`)
77 tRange = mkItem(itemRange, "range")
78 tRight = mkItem(itemRightDelim, "}}")
79 tRpar = mkItem(itemRightParen, ")")
80 tSpace = mkItem(itemSpace, " ")
81 raw = "`" + `abc\n\t\" ` + "`"
82 rawNL = "`now is{{\n}}the time`"
83 tRawQuote = mkItem(itemRawString, raw)
84 tRawQuoteNL = mkItem(itemRawString, rawNL)
85 )
86
87 var lexTests = []lexTest{
88 {"empty", "", []item{tEOF}},
89 {"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
90 {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
91 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{
92 mkItem(itemText, "hello-"),
93 mkItem(itemText, "-world"),
94 tEOF,
95 }},
96 {"punctuation", "{{,@% }}", []item{
97 tLeft,
98 mkItem(itemChar, ","),
99 mkItem(itemChar, "@"),
100 mkItem(itemChar, "%"),
101 tSpace,
102 tRight,
103 tEOF,
104 }},
105 {"parens", "{{((3))}}", []item{
106 tLeft,
107 tLpar,
108 tLpar,
109 mkItem(itemNumber, "3"),
110 tRpar,
111 tRpar,
112 tRight,
113 tEOF,
114 }},
115 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
116 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
117 {"block", `{{block "foo" .}}`, []item{
118 tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
119 }},
120 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
121 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
122 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
123 {"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
124 tLeft,
125 mkItem(itemNumber, "1"),
126 tSpace,
127 mkItem(itemNumber, "02"),
128 tSpace,
129 mkItem(itemNumber, "0x14"),
130 tSpace,
131 mkItem(itemNumber, "0X14"),
132 tSpace,
133 mkItem(itemNumber, "-7.2i"),
134 tSpace,
135 mkItem(itemNumber, "1e3"),
136 tSpace,
137 mkItem(itemNumber, "1E3"),
138 tSpace,
139 mkItem(itemNumber, "+1.2e-4"),
140 tSpace,
141 mkItem(itemNumber, "4.2i"),
142 tSpace,
143 mkItem(itemComplex, "1+2i"),
144 tSpace,
145 mkItem(itemNumber, "1_2"),
146 tSpace,
147 mkItem(itemNumber, "0x1.e_fp4"),
148 tSpace,
149 mkItem(itemNumber, "0X1.E_FP4"),
150 tRight,
151 tEOF,
152 }},
153 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
154 tLeft,
155 mkItem(itemCharConstant, `'a'`),
156 tSpace,
157 mkItem(itemCharConstant, `'\n'`),
158 tSpace,
159 mkItem(itemCharConstant, `'\''`),
160 tSpace,
161 mkItem(itemCharConstant, `'\\'`),
162 tSpace,
163 mkItem(itemCharConstant, `'\u00FF'`),
164 tSpace,
165 mkItem(itemCharConstant, `'\xFF'`),
166 tSpace,
167 mkItem(itemCharConstant, `'本'`),
168 tRight,
169 tEOF,
170 }},
171 {"bools", "{{true false}}", []item{
172 tLeft,
173 mkItem(itemBool, "true"),
174 tSpace,
175 mkItem(itemBool, "false"),
176 tRight,
177 tEOF,
178 }},
179 {"dot", "{{.}}", []item{
180 tLeft,
181 tDot,
182 tRight,
183 tEOF,
184 }},
185 {"nil", "{{nil}}", []item{
186 tLeft,
187 mkItem(itemNil, "nil"),
188 tRight,
189 tEOF,
190 }},
191 {"dots", "{{.x . .2 .x.y.z}}", []item{
192 tLeft,
193 mkItem(itemField, ".x"),
194 tSpace,
195 tDot,
196 tSpace,
197 mkItem(itemNumber, ".2"),
198 tSpace,
199 mkItem(itemField, ".x"),
200 mkItem(itemField, ".y"),
201 mkItem(itemField, ".z"),
202 tRight,
203 tEOF,
204 }},
205 {"keywords", "{{range if else end with}}", []item{
206 tLeft,
207 mkItem(itemRange, "range"),
208 tSpace,
209 mkItem(itemIf, "if"),
210 tSpace,
211 mkItem(itemElse, "else"),
212 tSpace,
213 mkItem(itemEnd, "end"),
214 tSpace,
215 mkItem(itemWith, "with"),
216 tRight,
217 tEOF,
218 }},
219 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
220 tLeft,
221 mkItem(itemVariable, "$c"),
222 tSpace,
223 mkItem(itemDeclare, ":="),
224 tSpace,
225 mkItem(itemIdentifier, "printf"),
226 tSpace,
227 mkItem(itemVariable, "$"),
228 tSpace,
229 mkItem(itemVariable, "$hello"),
230 tSpace,
231 mkItem(itemVariable, "$23"),
232 tSpace,
233 mkItem(itemVariable, "$"),
234 tSpace,
235 mkItem(itemVariable, "$var"),
236 mkItem(itemField, ".Field"),
237 tSpace,
238 mkItem(itemField, ".Method"),
239 tRight,
240 tEOF,
241 }},
242 {"variable invocation", "{{$x 23}}", []item{
243 tLeft,
244 mkItem(itemVariable, "$x"),
245 tSpace,
246 mkItem(itemNumber, "23"),
247 tRight,
248 tEOF,
249 }},
250 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
251 mkItem(itemText, "intro "),
252 tLeft,
253 mkItem(itemIdentifier, "echo"),
254 tSpace,
255 mkItem(itemIdentifier, "hi"),
256 tSpace,
257 mkItem(itemNumber, "1.2"),
258 tSpace,
259 tPipe,
260 mkItem(itemIdentifier, "noargs"),
261 tPipe,
262 mkItem(itemIdentifier, "args"),
263 tSpace,
264 mkItem(itemNumber, "1"),
265 tSpace,
266 mkItem(itemString, `"hi"`),
267 tRight,
268 mkItem(itemText, " outro"),
269 tEOF,
270 }},
271 {"declaration", "{{$v := 3}}", []item{
272 tLeft,
273 mkItem(itemVariable, "$v"),
274 tSpace,
275 mkItem(itemDeclare, ":="),
276 tSpace,
277 mkItem(itemNumber, "3"),
278 tRight,
279 tEOF,
280 }},
281 {"2 declarations", "{{$v , $w := 3}}", []item{
282 tLeft,
283 mkItem(itemVariable, "$v"),
284 tSpace,
285 mkItem(itemChar, ","),
286 tSpace,
287 mkItem(itemVariable, "$w"),
288 tSpace,
289 mkItem(itemDeclare, ":="),
290 tSpace,
291 mkItem(itemNumber, "3"),
292 tRight,
293 tEOF,
294 }},
295 {"field of parenthesized expression", "{{(.X).Y}}", []item{
296 tLeft,
297 tLpar,
298 mkItem(itemField, ".X"),
299 tRpar,
300 mkItem(itemField, ".Y"),
301 tRight,
302 tEOF,
303 }},
304 {"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
305 mkItem(itemText, "hello-"),
306 tLeft,
307 mkItem(itemNumber, "3"),
308 tRight,
309 mkItem(itemText, "-world"),
310 tEOF,
311 }},
312 {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
313 mkItem(itemText, "hello-"),
314 mkItem(itemText, "-world"),
315 tEOF,
316 }},
317
318 {"badchar", "#{{\x01}}", []item{
319 mkItem(itemText, "#"),
320 tLeft,
321 mkItem(itemError, "unrecognized character in action: U+0001"),
322 }},
323 {"unclosed action", "{{\n}}", []item{
324 tLeft,
325 mkItem(itemError, "unclosed action"),
326 }},
327 {"EOF in action", "{{range", []item{
328 tLeft,
329 tRange,
330 mkItem(itemError, "unclosed action"),
331 }},
332 {"unclosed quote", "{{\"\n\"}}", []item{
333 tLeft,
334 mkItem(itemError, "unterminated quoted string"),
335 }},
336 {"unclosed raw quote", "{{`xx}}", []item{
337 tLeft,
338 mkItem(itemError, "unterminated raw quoted string"),
339 }},
340 {"unclosed char constant", "{{'\n}}", []item{
341 tLeft,
342 mkItem(itemError, "unterminated character constant"),
343 }},
344 {"bad number", "{{3k}}", []item{
345 tLeft,
346 mkItem(itemError, `bad number syntax: "3k"`),
347 }},
348 {"unclosed paren", "{{(3}}", []item{
349 tLeft,
350 tLpar,
351 mkItem(itemNumber, "3"),
352 mkItem(itemError, `unclosed left paren`),
353 }},
354 {"extra right paren", "{{3)}}", []item{
355 tLeft,
356 mkItem(itemNumber, "3"),
357 tRpar,
358 mkItem(itemError, `unexpected right paren U+0029 ')'`),
359 }},
360
361
362
363
364 {"long pipeline deadlock", "{{|||||}}", []item{
365 tLeft,
366 tPipe,
367 tPipe,
368 tPipe,
369 tPipe,
370 tPipe,
371 tRight,
372 tEOF,
373 }},
374 {"text with bad comment", "hello-{{/*/}}-world", []item{
375 mkItem(itemText, "hello-"),
376 mkItem(itemError, `unclosed comment`),
377 }},
378 {"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
379 mkItem(itemText, "hello-"),
380 mkItem(itemError, `comment ends before closing delimiter`),
381 }},
382
383
384 {"unmatched right delimiter", "hello-{.}}-world", []item{
385 mkItem(itemText, "hello-{.}}-world"),
386 tEOF,
387 }},
388 }
389
390
391 func collect(t *lexTest, left, right string) (items []item) {
392 l := lex(t.name, t.input, left, right)
393 for {
394 item := l.nextItem()
395 items = append(items, item)
396 if item.typ == itemEOF || item.typ == itemError {
397 break
398 }
399 }
400 return
401 }
402
403 func equal(i1, i2 []item, checkPos bool) bool {
404 if len(i1) != len(i2) {
405 return false
406 }
407 for k := range i1 {
408 if i1[k].typ != i2[k].typ {
409 return false
410 }
411 if i1[k].val != i2[k].val {
412 return false
413 }
414 if checkPos && i1[k].pos != i2[k].pos {
415 return false
416 }
417 if checkPos && i1[k].line != i2[k].line {
418 return false
419 }
420 }
421 return true
422 }
423
424 func TestLex(t *testing.T) {
425 for _, test := range lexTests {
426 items := collect(&test, "", "")
427 if !equal(items, test.items, false) {
428 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
429 }
430 }
431 }
432
433
434 var lexDelimTests = []lexTest{
435 {"punctuation", "$$,@%{{}}@@", []item{
436 tLeftDelim,
437 mkItem(itemChar, ","),
438 mkItem(itemChar, "@"),
439 mkItem(itemChar, "%"),
440 mkItem(itemChar, "{"),
441 mkItem(itemChar, "{"),
442 mkItem(itemChar, "}"),
443 mkItem(itemChar, "}"),
444 tRightDelim,
445 tEOF,
446 }},
447 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
448 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
449 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
450 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
451 }
452
453 var (
454 tLeftDelim = mkItem(itemLeftDelim, "$$")
455 tRightDelim = mkItem(itemRightDelim, "@@")
456 )
457
458 func TestDelims(t *testing.T) {
459 for _, test := range lexDelimTests {
460 items := collect(&test, "$$", "@@")
461 if !equal(items, test.items, false) {
462 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
463 }
464 }
465 }
466
467 var lexPosTests = []lexTest{
468 {"empty", "", []item{{itemEOF, 0, "", 1}}},
469 {"punctuation", "{{,@%#}}", []item{
470 {itemLeftDelim, 0, "{{", 1},
471 {itemChar, 2, ",", 1},
472 {itemChar, 3, "@", 1},
473 {itemChar, 4, "%", 1},
474 {itemChar, 5, "#", 1},
475 {itemRightDelim, 6, "}}", 1},
476 {itemEOF, 8, "", 1},
477 }},
478 {"sample", "0123{{hello}}xyz", []item{
479 {itemText, 0, "0123", 1},
480 {itemLeftDelim, 4, "{{", 1},
481 {itemIdentifier, 6, "hello", 1},
482 {itemRightDelim, 11, "}}", 1},
483 {itemText, 13, "xyz", 1},
484 {itemEOF, 16, "", 1},
485 }},
486 {"trimafter", "{{x -}}\n{{y}}", []item{
487 {itemLeftDelim, 0, "{{", 1},
488 {itemIdentifier, 2, "x", 1},
489 {itemRightDelim, 5, "}}", 1},
490 {itemLeftDelim, 8, "{{", 2},
491 {itemIdentifier, 10, "y", 2},
492 {itemRightDelim, 11, "}}", 2},
493 {itemEOF, 13, "", 2},
494 }},
495 {"trimbefore", "{{x}}\n{{- y}}", []item{
496 {itemLeftDelim, 0, "{{", 1},
497 {itemIdentifier, 2, "x", 1},
498 {itemRightDelim, 3, "}}", 1},
499 {itemLeftDelim, 6, "{{", 2},
500 {itemIdentifier, 10, "y", 2},
501 {itemRightDelim, 11, "}}", 2},
502 {itemEOF, 13, "", 2},
503 }},
504 }
505
506
507
508 func TestPos(t *testing.T) {
509 for _, test := range lexPosTests {
510 items := collect(&test, "", "")
511 if !equal(items, test.items, true) {
512 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
513 if len(items) == len(test.items) {
514
515 for i := range items {
516 if !equal(items[i:i+1], test.items[i:i+1], true) {
517 i1 := items[i]
518 i2 := test.items[i]
519 t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
520 i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
521 }
522 }
523 }
524 }
525 }
526 }
527
528
529 func TestShutdown(t *testing.T) {
530
531 const text = "erroneous{{define}}{{else}}1234"
532 lexer := lex("foo", text, "{{", "}}")
533 _, err := New("root").parseLexer(lexer)
534 if err == nil {
535 t.Fatalf("expected error")
536 }
537
538 token, ok := <-lexer.items
539 if ok {
540 t.Fatalf("input was not drained; got %v", token)
541 }
542 }
543
544
545
546 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
547 defer t.recover(&err)
548 t.ParseName = t.Name
549 t.startParse(nil, lex, map[string]*Tree{})
550 t.parse()
551 t.add()
552 t.stopParse()
553 return t, nil
554 }
555
View as plain text