Source file src/github.com/pelletier/go-toml/v2/unstable/parser_test.go

Documentation: github.com/pelletier/go-toml/v2/unstable

     1  package unstable
     2  
     3  import (
     4  	"fmt"
     5  	"strconv"
     6  	"strings"
     7  	"testing"
     8  
     9  	"github.com/stretchr/testify/require"
    10  )
    11  
    12  func TestParser_AST_Numbers(t *testing.T) {
    13  	examples := []struct {
    14  		desc  string
    15  		input string
    16  		kind  Kind
    17  		err   bool
    18  	}{
    19  		{
    20  			desc:  "integer just digits",
    21  			input: `1234`,
    22  			kind:  Integer,
    23  		},
    24  		{
    25  			desc:  "integer zero",
    26  			input: `0`,
    27  			kind:  Integer,
    28  		},
    29  		{
    30  			desc:  "integer sign",
    31  			input: `+99`,
    32  			kind:  Integer,
    33  		},
    34  		{
    35  			desc:  "integer hex uppercase",
    36  			input: `0xDEADBEEF`,
    37  			kind:  Integer,
    38  		},
    39  		{
    40  			desc:  "integer hex lowercase",
    41  			input: `0xdead_beef`,
    42  			kind:  Integer,
    43  		},
    44  		{
    45  			desc:  "integer octal",
    46  			input: `0o01234567`,
    47  			kind:  Integer,
    48  		},
    49  		{
    50  			desc:  "integer binary",
    51  			input: `0b11010110`,
    52  			kind:  Integer,
    53  		},
    54  		{
    55  			desc:  "float zero",
    56  			input: `0.0`,
    57  			kind:  Float,
    58  		},
    59  		{
    60  			desc:  "float positive zero",
    61  			input: `+0.0`,
    62  			kind:  Float,
    63  		},
    64  		{
    65  			desc:  "float negative zero",
    66  			input: `-0.0`,
    67  			kind:  Float,
    68  		},
    69  		{
    70  			desc:  "float pi",
    71  			input: `3.1415`,
    72  			kind:  Float,
    73  		},
    74  		{
    75  			desc:  "float negative",
    76  			input: `-0.01`,
    77  			kind:  Float,
    78  		},
    79  		{
    80  			desc:  "float signed exponent",
    81  			input: `5e+22`,
    82  			kind:  Float,
    83  		},
    84  		{
    85  			desc:  "float exponent lowercase",
    86  			input: `1e06`,
    87  			kind:  Float,
    88  		},
    89  		{
    90  			desc:  "float exponent uppercase",
    91  			input: `-2E-2`,
    92  			kind:  Float,
    93  		},
    94  		{
    95  			desc:  "float fractional with exponent",
    96  			input: `6.626e-34`,
    97  			kind:  Float,
    98  		},
    99  		{
   100  			desc:  "float underscores",
   101  			input: `224_617.445_991_228`,
   102  			kind:  Float,
   103  		},
   104  		{
   105  			desc:  "inf",
   106  			input: `inf`,
   107  			kind:  Float,
   108  		},
   109  		{
   110  			desc:  "inf negative",
   111  			input: `-inf`,
   112  			kind:  Float,
   113  		},
   114  		{
   115  			desc:  "inf positive",
   116  			input: `+inf`,
   117  			kind:  Float,
   118  		},
   119  		{
   120  			desc:  "nan",
   121  			input: `nan`,
   122  			kind:  Float,
   123  		},
   124  		{
   125  			desc:  "nan negative",
   126  			input: `-nan`,
   127  			kind:  Float,
   128  		},
   129  		{
   130  			desc:  "nan positive",
   131  			input: `+nan`,
   132  			kind:  Float,
   133  		},
   134  	}
   135  
   136  	for _, e := range examples {
   137  		e := e
   138  		t.Run(e.desc, func(t *testing.T) {
   139  			p := Parser{}
   140  			p.Reset([]byte(`A = ` + e.input))
   141  			p.NextExpression()
   142  			err := p.Error()
   143  			if e.err {
   144  				require.Error(t, err)
   145  			} else {
   146  				require.NoError(t, err)
   147  
   148  				expected := astNode{
   149  					Kind: KeyValue,
   150  					Children: []astNode{
   151  						{Kind: e.kind, Data: []byte(e.input)},
   152  						{Kind: Key, Data: []byte(`A`)},
   153  					},
   154  				}
   155  				compareNode(t, expected, p.Expression())
   156  			}
   157  		})
   158  	}
   159  }
   160  
   161  type (
   162  	astNode struct {
   163  		Kind     Kind
   164  		Data     []byte
   165  		Children []astNode
   166  	}
   167  )
   168  
   169  func compareNode(t *testing.T, e astNode, n *Node) {
   170  	t.Helper()
   171  	require.Equal(t, e.Kind, n.Kind)
   172  	require.Equal(t, e.Data, n.Data)
   173  
   174  	compareIterator(t, e.Children, n.Children())
   175  }
   176  
   177  func compareIterator(t *testing.T, expected []astNode, actual Iterator) {
   178  	t.Helper()
   179  	idx := 0
   180  
   181  	for actual.Next() {
   182  		n := actual.Node()
   183  
   184  		if idx >= len(expected) {
   185  			t.Fatal("extra child in actual tree")
   186  		}
   187  		e := expected[idx]
   188  
   189  		compareNode(t, e, n)
   190  
   191  		idx++
   192  	}
   193  
   194  	if idx < len(expected) {
   195  		t.Fatal("missing children in actual", "idx =", idx, "expected =", len(expected))
   196  	}
   197  }
   198  
   199  //nolint:funlen
   200  func TestParser_AST(t *testing.T) {
   201  	examples := []struct {
   202  		desc  string
   203  		input string
   204  		ast   astNode
   205  		err   bool
   206  	}{
   207  		{
   208  			desc:  "simple string assignment",
   209  			input: `A = "hello"`,
   210  			ast: astNode{
   211  				Kind: KeyValue,
   212  				Children: []astNode{
   213  					{
   214  						Kind: String,
   215  						Data: []byte(`hello`),
   216  					},
   217  					{
   218  						Kind: Key,
   219  						Data: []byte(`A`),
   220  					},
   221  				},
   222  			},
   223  		},
   224  		{
   225  			desc:  "simple bool assignment",
   226  			input: `A = true`,
   227  			ast: astNode{
   228  				Kind: KeyValue,
   229  				Children: []astNode{
   230  					{
   231  						Kind: Bool,
   232  						Data: []byte(`true`),
   233  					},
   234  					{
   235  						Kind: Key,
   236  						Data: []byte(`A`),
   237  					},
   238  				},
   239  			},
   240  		},
   241  		{
   242  			desc:  "array of strings",
   243  			input: `A = ["hello", ["world", "again"]]`,
   244  			ast: astNode{
   245  				Kind: KeyValue,
   246  				Children: []astNode{
   247  					{
   248  						Kind: Array,
   249  						Children: []astNode{
   250  							{
   251  								Kind: String,
   252  								Data: []byte(`hello`),
   253  							},
   254  							{
   255  								Kind: Array,
   256  								Children: []astNode{
   257  									{
   258  										Kind: String,
   259  										Data: []byte(`world`),
   260  									},
   261  									{
   262  										Kind: String,
   263  										Data: []byte(`again`),
   264  									},
   265  								},
   266  							},
   267  						},
   268  					},
   269  					{
   270  						Kind: Key,
   271  						Data: []byte(`A`),
   272  					},
   273  				},
   274  			},
   275  		},
   276  		{
   277  			desc:  "array of arrays of strings",
   278  			input: `A = ["hello", "world"]`,
   279  			ast: astNode{
   280  				Kind: KeyValue,
   281  				Children: []astNode{
   282  					{
   283  						Kind: Array,
   284  						Children: []astNode{
   285  							{
   286  								Kind: String,
   287  								Data: []byte(`hello`),
   288  							},
   289  							{
   290  								Kind: String,
   291  								Data: []byte(`world`),
   292  							},
   293  						},
   294  					},
   295  					{
   296  						Kind: Key,
   297  						Data: []byte(`A`),
   298  					},
   299  				},
   300  			},
   301  		},
   302  		{
   303  			desc:  "inline table",
   304  			input: `name = { first = "Tom", last = "Preston-Werner" }`,
   305  			ast: astNode{
   306  				Kind: KeyValue,
   307  				Children: []astNode{
   308  					{
   309  						Kind: InlineTable,
   310  						Children: []astNode{
   311  							{
   312  								Kind: KeyValue,
   313  								Children: []astNode{
   314  									{Kind: String, Data: []byte(`Tom`)},
   315  									{Kind: Key, Data: []byte(`first`)},
   316  								},
   317  							},
   318  							{
   319  								Kind: KeyValue,
   320  								Children: []astNode{
   321  									{Kind: String, Data: []byte(`Preston-Werner`)},
   322  									{Kind: Key, Data: []byte(`last`)},
   323  								},
   324  							},
   325  						},
   326  					},
   327  					{
   328  						Kind: Key,
   329  						Data: []byte(`name`),
   330  					},
   331  				},
   332  			},
   333  		},
   334  	}
   335  
   336  	for _, e := range examples {
   337  		e := e
   338  		t.Run(e.desc, func(t *testing.T) {
   339  			p := Parser{}
   340  			p.Reset([]byte(e.input))
   341  			p.NextExpression()
   342  			err := p.Error()
   343  			if e.err {
   344  				require.Error(t, err)
   345  			} else {
   346  				require.NoError(t, err)
   347  				compareNode(t, e.ast, p.Expression())
   348  			}
   349  		})
   350  	}
   351  }
   352  
   353  func BenchmarkParseBasicStringWithUnicode(b *testing.B) {
   354  	p := &Parser{}
   355  	b.Run("4", func(b *testing.B) {
   356  		input := []byte(`"\u1234\u5678\u9ABC\u1234\u5678\u9ABC"`)
   357  		b.ReportAllocs()
   358  		b.SetBytes(int64(len(input)))
   359  
   360  		for i := 0; i < b.N; i++ {
   361  			p.parseBasicString(input)
   362  		}
   363  	})
   364  	b.Run("8", func(b *testing.B) {
   365  		input := []byte(`"\u12345678\u9ABCDEF0\u12345678\u9ABCDEF0"`)
   366  		b.ReportAllocs()
   367  		b.SetBytes(int64(len(input)))
   368  
   369  		for i := 0; i < b.N; i++ {
   370  			p.parseBasicString(input)
   371  		}
   372  	})
   373  }
   374  
   375  func BenchmarkParseBasicStringsEasy(b *testing.B) {
   376  	p := &Parser{}
   377  
   378  	for _, size := range []int{1, 4, 8, 16, 21} {
   379  		b.Run(strconv.Itoa(size), func(b *testing.B) {
   380  			input := []byte(`"` + strings.Repeat("A", size) + `"`)
   381  
   382  			b.ReportAllocs()
   383  			b.SetBytes(int64(len(input)))
   384  
   385  			for i := 0; i < b.N; i++ {
   386  				p.parseBasicString(input)
   387  			}
   388  		})
   389  	}
   390  }
   391  
   392  func TestParser_AST_DateTimes(t *testing.T) {
   393  	examples := []struct {
   394  		desc  string
   395  		input string
   396  		kind  Kind
   397  		err   bool
   398  	}{
   399  		{
   400  			desc:  "offset-date-time with delim 'T' and UTC offset",
   401  			input: `2021-07-21T12:08:05Z`,
   402  			kind:  DateTime,
   403  		},
   404  		{
   405  			desc:  "offset-date-time with space delim and +8hours offset",
   406  			input: `2021-07-21 12:08:05+08:00`,
   407  			kind:  DateTime,
   408  		},
   409  		{
   410  			desc:  "local-date-time with nano second",
   411  			input: `2021-07-21T12:08:05.666666666`,
   412  			kind:  LocalDateTime,
   413  		},
   414  		{
   415  			desc:  "local-date-time",
   416  			input: `2021-07-21T12:08:05`,
   417  			kind:  LocalDateTime,
   418  		},
   419  		{
   420  			desc:  "local-date",
   421  			input: `2021-07-21`,
   422  			kind:  LocalDate,
   423  		},
   424  	}
   425  
   426  	for _, e := range examples {
   427  		e := e
   428  		t.Run(e.desc, func(t *testing.T) {
   429  			p := Parser{}
   430  			p.Reset([]byte(`A = ` + e.input))
   431  			p.NextExpression()
   432  			err := p.Error()
   433  			if e.err {
   434  				require.Error(t, err)
   435  			} else {
   436  				require.NoError(t, err)
   437  
   438  				expected := astNode{
   439  					Kind: KeyValue,
   440  					Children: []astNode{
   441  						{Kind: e.kind, Data: []byte(e.input)},
   442  						{Kind: Key, Data: []byte(`A`)},
   443  					},
   444  				}
   445  				compareNode(t, expected, p.Expression())
   446  			}
   447  		})
   448  	}
   449  }
   450  
   451  // This example demonstrates how to parse a TOML document and preserving
   452  // comments.  Comments are stored in the AST as Comment nodes. This example
   453  // displays the structure of the full AST generated by the parser using the
   454  // following structure:
   455  //
   456  //  1. Each root-level expression is separated by three dashes.
   457  //  2. Bytes associated to a node are displayed in square brackets.
   458  //  3. Siblings have the same indentation.
   459  //  4. Children of a node are indented one level.
   460  func ExampleParser_comments() {
   461  	doc := `# Top of the document comment.
   462  # Optional, any amount of lines.
   463  
   464  # Above table.
   465  [table] # Next to table.
   466  # Above simple value.
   467  key = "value" # Next to simple value.
   468  # Below simple value.
   469  
   470  # Some comment alone.
   471  
   472  # Multiple comments, on multiple lines.
   473  
   474  # Above inline table.
   475  name = { first = "Tom", last = "Preston-Werner" } # Next to inline table.
   476  # Below inline table.
   477  
   478  # Above array.
   479  array = [ 1, 2, 3 ] # Next to one-line array.
   480  # Below array.
   481  
   482  # Above multi-line array.
   483  key5 = [ # Next to start of inline array.
   484    # Second line before array content.
   485    1, # Next to first element.
   486    # After first element.
   487    # Before second element.
   488    2,
   489    3, # Next to last element
   490    # After last element.
   491  ] # Next to end of array.
   492  # Below multi-line array.
   493  
   494  # Before array table.
   495  [[products]] # Next to array table.
   496  # After array table.
   497  `
   498  
   499  	var printGeneric func(*Parser, int, *Node)
   500  	printGeneric = func(p *Parser, indent int, e *Node) {
   501  		if e == nil {
   502  			return
   503  		}
   504  		s := p.Shape(e.Raw)
   505  		x := fmt.Sprintf("%d:%d->%d:%d (%d->%d)", s.Start.Line, s.Start.Column, s.End.Line, s.End.Column, s.Start.Offset, s.End.Offset)
   506  		fmt.Printf("%-25s | %s%s [%s]\n", x, strings.Repeat("  ", indent), e.Kind, e.Data)
   507  		printGeneric(p, indent+1, e.Child())
   508  		printGeneric(p, indent, e.Next())
   509  	}
   510  
   511  	printTree := func(p *Parser) {
   512  		for p.NextExpression() {
   513  			e := p.Expression()
   514  			fmt.Println("---")
   515  			printGeneric(p, 0, e)
   516  		}
   517  		if err := p.Error(); err != nil {
   518  			panic(err)
   519  		}
   520  	}
   521  
   522  	p := &Parser{
   523  		KeepComments: true,
   524  	}
   525  	p.Reset([]byte(doc))
   526  	printTree(p)
   527  
   528  	// Output:
   529  	// ---
   530  	// 1:1->1:31 (0->30)         | Comment [# Top of the document comment.]
   531  	// ---
   532  	// 2:1->2:33 (31->63)        | Comment [# Optional, any amount of lines.]
   533  	// ---
   534  	// 4:1->4:15 (65->79)        | Comment [# Above table.]
   535  	// ---
   536  	// 1:1->1:1 (0->0)           | Table []
   537  	// 5:2->5:7 (81->86)         |   Key [table]
   538  	// 5:9->5:25 (88->104)       | Comment [# Next to table.]
   539  	// ---
   540  	// 6:1->6:22 (105->126)      | Comment [# Above simple value.]
   541  	// ---
   542  	// 1:1->1:1 (0->0)           | KeyValue []
   543  	// 7:7->7:14 (133->140)      |   String [value]
   544  	// 7:1->7:4 (127->130)       |   Key [key]
   545  	// 7:15->7:38 (141->164)     | Comment [# Next to simple value.]
   546  	// ---
   547  	// 8:1->8:22 (165->186)      | Comment [# Below simple value.]
   548  	// ---
   549  	// 10:1->10:22 (188->209)    | Comment [# Some comment alone.]
   550  	// ---
   551  	// 12:1->12:40 (211->250)    | Comment [# Multiple comments, on multiple lines.]
   552  	// ---
   553  	// 14:1->14:22 (252->273)    | Comment [# Above inline table.]
   554  	// ---
   555  	// 1:1->1:1 (0->0)           | KeyValue []
   556  	// 15:8->15:9 (281->282)     |   InlineTable []
   557  	// 1:1->1:1 (0->0)           |     KeyValue []
   558  	// 15:18->15:23 (291->296)   |       String [Tom]
   559  	// 15:10->15:15 (283->288)   |       Key [first]
   560  	// 1:1->1:1 (0->0)           |     KeyValue []
   561  	// 15:32->15:48 (305->321)   |       String [Preston-Werner]
   562  	// 15:25->15:29 (298->302)   |       Key [last]
   563  	// 15:1->15:5 (274->278)     |   Key [name]
   564  	// 15:51->15:74 (324->347)   | Comment [# Next to inline table.]
   565  	// ---
   566  	// 16:1->16:22 (348->369)    | Comment [# Below inline table.]
   567  	// ---
   568  	// 18:1->18:15 (371->385)    | Comment [# Above array.]
   569  	// ---
   570  	// 1:1->1:1 (0->0)           | KeyValue []
   571  	// 1:1->1:1 (0->0)           |   Array []
   572  	// 19:11->19:12 (396->397)   |     Integer [1]
   573  	// 19:14->19:15 (399->400)   |     Integer [2]
   574  	// 19:17->19:18 (402->403)   |     Integer [3]
   575  	// 19:1->19:6 (386->391)     |   Key [array]
   576  	// 19:21->19:46 (406->431)   | Comment [# Next to one-line array.]
   577  	// ---
   578  	// 20:1->20:15 (432->446)    | Comment [# Below array.]
   579  	// ---
   580  	// 22:1->22:26 (448->473)    | Comment [# Above multi-line array.]
   581  	// ---
   582  	// 1:1->1:1 (0->0)           | KeyValue []
   583  	// 1:1->1:1 (0->0)           |   Array []
   584  	// 23:10->23:42 (483->515)   |     Comment [# Next to start of inline array.]
   585  	// 24:3->24:38 (518->553)    |       Comment [# Second line before array content.]
   586  	// 25:3->25:4 (556->557)     |     Integer [1]
   587  	// 25:6->25:30 (559->583)    |     Comment [# Next to first element.]
   588  	// 26:3->26:25 (586->608)    |       Comment [# After first element.]
   589  	// 27:3->27:27 (611->635)    |       Comment [# Before second element.]
   590  	// 28:3->28:4 (638->639)     |     Integer [2]
   591  	// 29:3->29:4 (643->644)     |     Integer [3]
   592  	// 29:6->29:28 (646->668)    |     Comment [# Next to last element]
   593  	// 30:3->30:24 (671->692)    |       Comment [# After last element.]
   594  	// 23:1->23:5 (474->478)     |   Key [key5]
   595  	// 31:3->31:26 (695->718)    | Comment [# Next to end of array.]
   596  	// ---
   597  	// 32:1->32:26 (719->744)    | Comment [# Below multi-line array.]
   598  	// ---
   599  	// 34:1->34:22 (746->767)    | Comment [# Before array table.]
   600  	// ---
   601  	// 1:1->1:1 (0->0)           | ArrayTable []
   602  	// 35:3->35:11 (770->778)    |   Key [products]
   603  	// 35:14->35:36 (781->803)   | Comment [# Next to array table.]
   604  	// ---
   605  	// 36:1->36:21 (804->824)    | Comment [# After array table.]
   606  }
   607  
   608  func ExampleParser() {
   609  	doc := `
   610  	hello = "world"
   611  	value = 42
   612  	`
   613  	p := Parser{}
   614  	p.Reset([]byte(doc))
   615  	for p.NextExpression() {
   616  		e := p.Expression()
   617  		fmt.Printf("Expression: %s\n", e.Kind)
   618  		value := e.Value()
   619  		it := e.Key()
   620  		k := it.Node() // shortcut: we know there is no dotted key in the example
   621  		fmt.Printf("%s -> (%s) %s\n", k.Data, value.Kind, value.Data)
   622  	}
   623  
   624  	// Output:
   625  	// Expression: KeyValue
   626  	// hello -> (String) world
   627  	// Expression: KeyValue
   628  	// value -> (Integer) 42
   629  }
   630  

View as plain text