Source file src/google.golang.org/protobuf/internal/encoding/text/decode_test.go

Documentation: google.golang.org/protobuf/internal/encoding/text

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package text_test
     6  
     7  import (
     8  	"fmt"
     9  	"math"
    10  	"strings"
    11  	"testing"
    12  	"unicode/utf8"
    13  
    14  	"github.com/google/go-cmp/cmp"
    15  
    16  	"google.golang.org/protobuf/internal/encoding/text"
    17  	"google.golang.org/protobuf/internal/flags"
    18  )
    19  
    20  var eofErr = text.ErrUnexpectedEOF.Error()
    21  
    22  type R struct {
    23  	// K is expected Kind of the returned Token object from calling Decoder.Read.
    24  	K text.Kind
    25  	// E is expected error substring from calling Decoder.Read if set.
    26  	E string
    27  	// T contains NT (if K is Name) or ST (if K is Scalar) or nil (others)
    28  	T interface{}
    29  	// P is expected Token.Pos if set > 0.
    30  	P int
    31  	// RS is expected result from Token.RawString() if not empty.
    32  	RS string
    33  }
    34  
    35  // NT contains data for checking against a name token.
    36  type NT struct {
    37  	K text.NameKind
    38  	// Sep is true if name token should have separator character, else false.
    39  	Sep bool
    40  	// If K is IdentName or TypeName, invoke corresponding getter and compare against this field.
    41  	S string
    42  	// If K is FieldNumber, invoke getter and compare against this field.
    43  	N int32
    44  }
    45  
    46  // ST contains data for checking against a scalar token.
    47  type ST struct {
    48  	// checker that is expected to return OK.
    49  	ok checker
    50  	// checker that is expected to return not OK.
    51  	nok checker
    52  }
    53  
    54  // checker provides API for the token wrapper API call types Str, Enum, Bool,
    55  // Uint64, Uint32, Int64, Int32, Float64, Float32.
    56  type checker interface {
    57  	// checkOk checks and expects for token API call to return ok and compare
    58  	// against implementation-stored value. Returns empty string if success,
    59  	// else returns error message describing the error.
    60  	checkOk(text.Token) string
    61  	// checkNok checks and expects for token API call to return not ok. Returns
    62  	// empty string if success, else returns error message describing the error.
    63  	checkNok(text.Token) string
    64  }
    65  
    66  type Str struct {
    67  	val string
    68  }
    69  
    70  func (s Str) checkOk(tok text.Token) string {
    71  	got, ok := tok.String()
    72  	if !ok {
    73  		return fmt.Sprintf("Token.String() returned not OK for token: %v", tok.RawString())
    74  	}
    75  	if got != s.val {
    76  		return fmt.Sprintf("Token.String() got %q want %q for token: %v", got, s.val, tok.RawString())
    77  	}
    78  	return ""
    79  }
    80  
    81  func (s Str) checkNok(tok text.Token) string {
    82  	if _, ok := tok.String(); ok {
    83  		return fmt.Sprintf("Token.String() returned OK for token: %v", tok.RawString())
    84  	}
    85  	return ""
    86  }
    87  
    88  type Enum struct {
    89  	val string
    90  }
    91  
    92  func (e Enum) checkOk(tok text.Token) string {
    93  	got, ok := tok.Enum()
    94  	if !ok {
    95  		return fmt.Sprintf("Token.Enum() returned not OK for token: %v", tok.RawString())
    96  	}
    97  	if got != e.val {
    98  		return fmt.Sprintf("Token.Enum() got %q want %q for token: %v", got, e.val, tok.RawString())
    99  	}
   100  	return ""
   101  }
   102  
   103  func (e Enum) checkNok(tok text.Token) string {
   104  	if _, ok := tok.Enum(); ok {
   105  		return fmt.Sprintf("Token.Enum() returned OK for token: %v", tok.RawString())
   106  	}
   107  	return ""
   108  }
   109  
   110  type Bool struct {
   111  	val bool
   112  }
   113  
   114  func (b Bool) checkOk(tok text.Token) string {
   115  	got, ok := tok.Bool()
   116  	if !ok {
   117  		return fmt.Sprintf("Token.Bool() returned not OK for token: %v", tok.RawString())
   118  	}
   119  	if got != b.val {
   120  		return fmt.Sprintf("Token.Bool() got %v want %v for token: %v", got, b.val, tok.RawString())
   121  	}
   122  	return ""
   123  }
   124  
   125  func (b Bool) checkNok(tok text.Token) string {
   126  	if _, ok := tok.Bool(); ok {
   127  		return fmt.Sprintf("Token.Bool() returned OK for token: %v", tok.RawString())
   128  	}
   129  	return ""
   130  }
   131  
   132  type Uint64 struct {
   133  	val uint64
   134  }
   135  
   136  func (n Uint64) checkOk(tok text.Token) string {
   137  	got, ok := tok.Uint64()
   138  	if !ok {
   139  		return fmt.Sprintf("Token.Uint64() returned not OK for token: %v", tok.RawString())
   140  	}
   141  	if got != n.val {
   142  		return fmt.Sprintf("Token.Uint64() got %v want %v for token: %v", got, n.val, tok.RawString())
   143  	}
   144  	return ""
   145  }
   146  
   147  func (n Uint64) checkNok(tok text.Token) string {
   148  	if _, ok := tok.Uint64(); ok {
   149  		return fmt.Sprintf("Token.Uint64() returned OK for token: %v", tok.RawString())
   150  	}
   151  	return ""
   152  }
   153  
   154  type Uint32 struct {
   155  	val uint32
   156  }
   157  
   158  func (n Uint32) checkOk(tok text.Token) string {
   159  	got, ok := tok.Uint32()
   160  	if !ok {
   161  		return fmt.Sprintf("Token.Uint32() returned not OK for token: %v", tok.RawString())
   162  	}
   163  	if got != n.val {
   164  		return fmt.Sprintf("Token.Uint32() got %v want %v for token: %v", got, n.val, tok.RawString())
   165  	}
   166  	return ""
   167  }
   168  
   169  func (n Uint32) checkNok(tok text.Token) string {
   170  	if _, ok := tok.Uint32(); ok {
   171  		return fmt.Sprintf("Token.Uint32() returned OK for token: %v", tok.RawString())
   172  	}
   173  	return ""
   174  }
   175  
   176  type Int64 struct {
   177  	val int64
   178  }
   179  
   180  func (n Int64) checkOk(tok text.Token) string {
   181  	got, ok := tok.Int64()
   182  	if !ok {
   183  		return fmt.Sprintf("Token.Int64() returned not OK for token: %v", tok.RawString())
   184  	}
   185  	if got != n.val {
   186  		return fmt.Sprintf("Token.Int64() got %v want %v for token: %v", got, n.val, tok.RawString())
   187  	}
   188  	return ""
   189  }
   190  
   191  func (n Int64) checkNok(tok text.Token) string {
   192  	if _, ok := tok.Int64(); ok {
   193  		return fmt.Sprintf("Token.Int64() returned OK for token: %v", tok.RawString())
   194  	}
   195  	return ""
   196  }
   197  
   198  type Int32 struct {
   199  	val int32
   200  }
   201  
   202  func (n Int32) checkOk(tok text.Token) string {
   203  	got, ok := tok.Int32()
   204  	if !ok {
   205  		return fmt.Sprintf("Token.Int32() returned not OK for token: %v", tok.RawString())
   206  	}
   207  	if got != n.val {
   208  		return fmt.Sprintf("Token.Int32() got %v want %v for token: %v", got, n.val, tok.RawString())
   209  	}
   210  	return ""
   211  }
   212  
   213  func (n Int32) checkNok(tok text.Token) string {
   214  	if _, ok := tok.Int32(); ok {
   215  		return fmt.Sprintf("Token.Int32() returned OK for token: %v", tok.RawString())
   216  	}
   217  	return ""
   218  }
   219  
   220  type Float64 struct {
   221  	val float64
   222  }
   223  
   224  func (n Float64) checkOk(tok text.Token) string {
   225  	got, ok := tok.Float64()
   226  	if !ok {
   227  		return fmt.Sprintf("Token.Float64() returned not OK for token: %v", tok.RawString())
   228  	}
   229  	if math.Float64bits(got) != math.Float64bits(n.val) {
   230  		return fmt.Sprintf("Token.Float64() got %v want %v for token: %v", got, n.val, tok.RawString())
   231  	}
   232  	return ""
   233  }
   234  
   235  func (n Float64) checkNok(tok text.Token) string {
   236  	if _, ok := tok.Float64(); ok {
   237  		return fmt.Sprintf("Token.Float64() returned OK for token: %v", tok.RawString())
   238  	}
   239  	return ""
   240  }
   241  
   242  type Float32 struct {
   243  	val float32
   244  }
   245  
   246  func (n Float32) checkOk(tok text.Token) string {
   247  	got, ok := tok.Float32()
   248  	if !ok {
   249  		return fmt.Sprintf("Token.Float32() returned not OK for token: %v", tok.RawString())
   250  	}
   251  	if math.Float32bits(got) != math.Float32bits(n.val) {
   252  		return fmt.Sprintf("Token.Float32() got %v want %v for token: %v", got, n.val, tok.RawString())
   253  	}
   254  	return ""
   255  }
   256  
   257  func (n Float32) checkNok(tok text.Token) string {
   258  	if _, ok := tok.Float32(); ok {
   259  		return fmt.Sprintf("Token.Float32() returned OK for token: %v", tok.RawString())
   260  	}
   261  	return ""
   262  }
   263  
   264  func TestDecoder(t *testing.T) {
   265  	const space = " \n\r\t"
   266  	tests := []struct {
   267  		in string
   268  		// want is a list of expected Tokens returned from calling Decoder.Read.
   269  		// An item makes the test code invoke Decoder.Read and compare against
   270  		// R.K and R.E. If R.K is Name, it compares
   271  		want []R
   272  	}{
   273  		{
   274  			in:   "",
   275  			want: []R{{K: text.EOF}},
   276  		},
   277  		{
   278  			in:   "# comment",
   279  			want: []R{{K: text.EOF}},
   280  		},
   281  		{
   282  			in:   space + "# comment" + space,
   283  			want: []R{{K: text.EOF}},
   284  		},
   285  		{
   286  			in:   space,
   287  			want: []R{{K: text.EOF, P: len(space)}},
   288  		},
   289  		{
   290  			// Calling Read after EOF will keep returning EOF for
   291  			// succeeding Read calls.
   292  			in: space,
   293  			want: []R{
   294  				{K: text.EOF},
   295  				{K: text.EOF},
   296  				{K: text.EOF},
   297  			},
   298  		},
   299  		{
   300  			// NUL is an invalid whitespace since C++ uses C-strings.
   301  			in:   "\x00",
   302  			want: []R{{E: "invalid field name: \x00"}},
   303  		},
   304  
   305  		// Field names.
   306  		{
   307  			in: "name",
   308  			want: []R{
   309  				{K: text.Name, T: NT{K: text.IdentName, S: "name"}, RS: "name"},
   310  				{E: eofErr},
   311  			},
   312  		},
   313  		{
   314  			in: space + "name:" + space,
   315  			want: []R{
   316  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   317  				{E: eofErr},
   318  			},
   319  		},
   320  		{
   321  			in: space + "name" + space + ":" + space,
   322  			want: []R{
   323  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   324  				{E: eofErr},
   325  			},
   326  		},
   327  		{
   328  			in: "name # comment",
   329  			want: []R{
   330  				{K: text.Name, T: NT{K: text.IdentName, S: "name"}},
   331  				{E: eofErr},
   332  			},
   333  		},
   334  		{
   335  			// Comments only extend until the newline.
   336  			in: "# comment \nname",
   337  			want: []R{
   338  				{K: text.Name, T: NT{K: text.IdentName, S: "name"}, P: 11},
   339  			},
   340  		},
   341  		{
   342  			in: "name # comment \n:",
   343  			want: []R{
   344  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   345  			},
   346  		},
   347  		{
   348  			in: "name123",
   349  			want: []R{
   350  				{K: text.Name, T: NT{K: text.IdentName, S: "name123"}},
   351  			},
   352  		},
   353  		{
   354  			in: "name_123",
   355  			want: []R{
   356  				{K: text.Name, T: NT{K: text.IdentName, S: "name_123"}},
   357  			},
   358  		},
   359  		{
   360  			in: "_123",
   361  			want: []R{
   362  				{K: text.Name, T: NT{K: text.IdentName, S: "_123"}},
   363  			},
   364  		},
   365  		{
   366  			in:   ":",
   367  			want: []R{{E: "syntax error (line 1:1): invalid field name: :"}},
   368  		},
   369  		{
   370  			in:   "\n\n\n {",
   371  			want: []R{{E: "syntax error (line 4:2): invalid field name: {"}},
   372  		},
   373  		{
   374  			in:   "123name",
   375  			want: []R{{E: "invalid field name: 123name"}},
   376  		},
   377  		{
   378  			in:   `/`,
   379  			want: []R{{E: `invalid field name: /`}},
   380  		},
   381  		{
   382  			in:   `世界`,
   383  			want: []R{{E: `invalid field name: 世`}},
   384  		},
   385  		{
   386  			in:   `1a/b`,
   387  			want: []R{{E: `invalid field name: 1a`}},
   388  		},
   389  		{
   390  			in:   `1c\d`,
   391  			want: []R{{E: `invalid field name: 1c`}},
   392  		},
   393  		{
   394  			in:   "\x84f",
   395  			want: []R{{E: "invalid field name: \x84"}},
   396  		},
   397  		{
   398  			in:   "\uFFFDxxx",
   399  			want: []R{{E: "invalid field name: \uFFFD"}},
   400  		},
   401  		{
   402  			in:   "-a234567890123456789012345678901234567890abc",
   403  			want: []R{{E: "invalid field name: -a2345678901234567890123456789012…"}},
   404  		},
   405  		{
   406  			in: "[type]",
   407  			want: []R{
   408  				{K: text.Name, T: NT{K: text.TypeName, S: "type"}, RS: "[type]"},
   409  			},
   410  		},
   411  		{
   412  			// V1 allows this syntax. C++ does not, however, C++ also fails if
   413  			// field is Any and does not contain '/'.
   414  			in: "[/type]",
   415  			want: []R{
   416  				{K: text.Name, T: NT{K: text.TypeName, S: "/type"}},
   417  			},
   418  		},
   419  		{
   420  			in:   "[.type]",
   421  			want: []R{{E: "invalid type URL/extension field name: [.type]"}},
   422  		},
   423  		{
   424  			in: "[pkg.Foo.extension_field]",
   425  			want: []R{
   426  				{K: text.Name, T: NT{K: text.TypeName, S: "pkg.Foo.extension_field"}},
   427  			},
   428  		},
   429  		{
   430  			in: "[domain.com/type]",
   431  			want: []R{
   432  				{K: text.Name, T: NT{K: text.TypeName, S: "domain.com/type"}},
   433  			},
   434  		},
   435  		{
   436  			in: "[domain.com/pkg.type]",
   437  			want: []R{
   438  				{K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}},
   439  			},
   440  		},
   441  		{
   442  			in: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]",
   443  			want: []R{
   444  				{
   445  					K: text.Name,
   446  					T: NT{
   447  						K: text.TypeName,
   448  						S: "sub.domain.com/path/to/proto.package.name",
   449  					},
   450  					RS: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]",
   451  				},
   452  			},
   453  		},
   454  		{
   455  			// V2 no longer allows a quoted string for the Any type URL.
   456  			in:   `["domain.com/pkg.type"]`,
   457  			want: []R{{E: `invalid type URL/extension field name: ["`}},
   458  		},
   459  		{
   460  			// V2 no longer allows a quoted string for the Any type URL.
   461  			in:   `['domain.com/pkg.type']`,
   462  			want: []R{{E: `invalid type URL/extension field name: ['`}},
   463  		},
   464  		{
   465  			in:   "[pkg.Foo.extension_field:",
   466  			want: []R{{E: "invalid type URL/extension field name: [pkg.Foo.extension_field:"}},
   467  		},
   468  		{
   469  			// V2 no longer allows whitespace within identifier "word".
   470  			in:   "[proto.packa ge.field]",
   471  			want: []R{{E: "invalid type URL/extension field name: [proto.packa g"}},
   472  		},
   473  		{
   474  			// V2 no longer allows comments within identifier "word".
   475  			in:   "[proto.packa # comment\n ge.field]",
   476  			want: []R{{E: "invalid type URL/extension field name: [proto.packa # comment\n g"}},
   477  		},
   478  		{
   479  			in:   "[proto.package.]",
   480  			want: []R{{E: "invalid type URL/extension field name: [proto.package."}},
   481  		},
   482  		{
   483  			in:   "[proto.package/]",
   484  			want: []R{{E: "invalid type URL/extension field name: [proto.package/"}},
   485  		},
   486  		{
   487  			in: `message_field{[bad@]`,
   488  			want: []R{
   489  				{K: text.Name},
   490  				{K: text.MessageOpen},
   491  				{E: `invalid type URL/extension field name: [bad@`},
   492  			},
   493  		},
   494  		{
   495  			in: `message_field{[invalid//type]`,
   496  			want: []R{
   497  				{K: text.Name},
   498  				{K: text.MessageOpen},
   499  				{E: `invalid type URL/extension field name: [invalid//`},
   500  			},
   501  		},
   502  		{
   503  			in: `message_field{[proto.package.]`,
   504  			want: []R{
   505  				{K: text.Name},
   506  				{K: text.MessageOpen},
   507  				{E: `invalid type URL/extension field name: [proto.package.`},
   508  			},
   509  		},
   510  		{
   511  			in:   "[proto.package",
   512  			want: []R{{E: eofErr}},
   513  		},
   514  		{
   515  			in: "[" + space + "type" + space + "]" + space + ":",
   516  			want: []R{
   517  				{
   518  					K: text.Name,
   519  					T: NT{
   520  						K:   text.TypeName,
   521  						Sep: true,
   522  						S:   "type",
   523  					},
   524  					RS: "[" + space + "type" + space + "]",
   525  				},
   526  			},
   527  		},
   528  		{
   529  			// Whitespaces/comments are only allowed betweeb
   530  			in: "[" + space + "domain" + space + "." + space + "com # comment\n" +
   531  				"/" + "pkg" + space + "." + space + "type" + space + "]",
   532  			want: []R{
   533  				{K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}},
   534  			},
   535  		},
   536  		{
   537  			in: "42",
   538  			want: []R{
   539  				{K: text.Name, T: NT{K: text.FieldNumber, N: 42}},
   540  			},
   541  		},
   542  		{
   543  			in:   "0x42:",
   544  			want: []R{{E: "invalid field number: 0x42"}},
   545  		},
   546  		{
   547  			in:   "042:",
   548  			want: []R{{E: "invalid field number: 042"}},
   549  		},
   550  		{
   551  			in:   "123.456:",
   552  			want: []R{{E: "invalid field number: 123.456"}},
   553  		},
   554  		{
   555  			in:   "-123",
   556  			want: []R{{E: "invalid field number: -123"}},
   557  		},
   558  		{
   559  			in:   "- \t 123.321e6",
   560  			want: []R{{E: "invalid field number: -123.321e6"}},
   561  		},
   562  		{
   563  			in:   "-",
   564  			want: []R{{E: "invalid field name: -"}},
   565  		},
   566  		{
   567  			in:   "- ",
   568  			want: []R{{E: "invalid field name: -"}},
   569  		},
   570  		{
   571  			in:   "- # negative\n 123",
   572  			want: []R{{E: "invalid field number: -123"}},
   573  		},
   574  		{
   575  			// Field number > math.MaxInt32.
   576  			in:   "2147483648:",
   577  			want: []R{{E: "invalid field number: 2147483648"}},
   578  		},
   579  
   580  		// String field value. More string parsing specific testing in
   581  		// TestUnmarshalString.
   582  		{
   583  			in: `name: "hello world"`,
   584  			want: []R{
   585  				{K: text.Name},
   586  				{
   587  					K:  text.Scalar,
   588  					T:  ST{ok: Str{"hello world"}, nok: Enum{}},
   589  					RS: `"hello world"`,
   590  				},
   591  				{K: text.EOF},
   592  			},
   593  		},
   594  		{
   595  			in: `name: 'hello'`,
   596  			want: []R{
   597  				{K: text.Name},
   598  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   599  			},
   600  		},
   601  		{
   602  			in: `name: "hello'`,
   603  			want: []R{
   604  				{K: text.Name},
   605  				{E: eofErr},
   606  			},
   607  		},
   608  		{
   609  			in: `name: 'hello`,
   610  			want: []R{
   611  				{K: text.Name},
   612  				{E: eofErr},
   613  			},
   614  		},
   615  		{
   616  			// Field name without separator is ok. prototext package will need
   617  			// to determine that this is not valid for scalar values.
   618  			in: space + `name` + space + `"hello"` + space,
   619  			want: []R{
   620  				{K: text.Name},
   621  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   622  			},
   623  		},
   624  		{
   625  			in: `name'hello'`,
   626  			want: []R{
   627  				{K: text.Name},
   628  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   629  			},
   630  		},
   631  		{
   632  			in: `name: ` + space + `"hello"` + space + `,`,
   633  			want: []R{
   634  				{K: text.Name},
   635  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   636  				{K: text.EOF},
   637  			},
   638  		},
   639  		{
   640  			in: `name` + space + `:` + `"hello"` + space + `;` + space,
   641  			want: []R{
   642  				{K: text.Name},
   643  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   644  				{K: text.EOF},
   645  			},
   646  		},
   647  		{
   648  			in: `name:"hello" , ,`,
   649  			want: []R{
   650  				{K: text.Name},
   651  				{K: text.Scalar},
   652  				{E: "(line 1:16): invalid field name: ,"},
   653  			},
   654  		},
   655  		{
   656  			in: `name:"hello" , ;`,
   657  			want: []R{
   658  				{K: text.Name},
   659  				{K: text.Scalar},
   660  				{E: "(line 1:16): invalid field name: ;"},
   661  			},
   662  		},
   663  		{
   664  			in: `name:"hello" name:'world'`,
   665  			want: []R{
   666  				{K: text.Name},
   667  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   668  				{K: text.Name},
   669  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
   670  				{K: text.EOF},
   671  			},
   672  		},
   673  		{
   674  			in: `name:"hello", name:"world"`,
   675  			want: []R{
   676  				{K: text.Name},
   677  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   678  				{K: text.Name},
   679  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
   680  				{K: text.EOF},
   681  			},
   682  		},
   683  		{
   684  			in: `name:"hello"; name:"world",`,
   685  			want: []R{
   686  				{K: text.Name},
   687  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   688  				{K: text.Name},
   689  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
   690  				{K: text.EOF},
   691  			},
   692  		},
   693  		{
   694  			in: `foo:"hello"bar:"world"`,
   695  			want: []R{
   696  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}},
   697  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   698  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "bar"}},
   699  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
   700  				{K: text.EOF},
   701  			},
   702  		},
   703  		{
   704  			in: `foo:"hello"[bar]:"world"`,
   705  			want: []R{
   706  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}},
   707  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   708  				{K: text.Name, T: NT{K: text.TypeName, Sep: true, S: "bar"}},
   709  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
   710  				{K: text.EOF},
   711  			},
   712  		},
   713  		{
   714  			in: `name:"foo"` + space + `"bar"` + space + `'qux'`,
   715  			want: []R{
   716  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   717  				{K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
   718  				{K: text.EOF},
   719  			},
   720  		},
   721  		{
   722  			in: `name:"foo"'bar'"qux"`,
   723  			want: []R{
   724  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   725  				{K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
   726  				{K: text.EOF},
   727  			},
   728  		},
   729  		{
   730  			in: `name:"foo"` + space + `"bar" # comment` + "\n'qux' # comment",
   731  			want: []R{
   732  				{K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}},
   733  				{K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
   734  				{K: text.EOF},
   735  			},
   736  		},
   737  
   738  		// Lists.
   739  		{
   740  			in: `name: [`,
   741  			want: []R{
   742  				{K: text.Name},
   743  				{K: text.ListOpen},
   744  				{E: eofErr},
   745  			},
   746  		},
   747  		{
   748  			in: `name: []`,
   749  			want: []R{
   750  				{K: text.Name},
   751  				{K: text.ListOpen},
   752  				{K: text.ListClose},
   753  				{K: text.EOF},
   754  			},
   755  		},
   756  		{
   757  			in: `name []`,
   758  			want: []R{
   759  				{K: text.Name},
   760  				{K: text.ListOpen},
   761  				{K: text.ListClose},
   762  				{K: text.EOF},
   763  			},
   764  		},
   765  		{
   766  			in: `name: [,`,
   767  			want: []R{
   768  				{K: text.Name},
   769  				{K: text.ListOpen},
   770  				{E: `(line 1:8): invalid scalar value: ,`},
   771  			},
   772  		},
   773  		{
   774  			in: `name: [0`,
   775  			want: []R{
   776  				{K: text.Name},
   777  				{K: text.ListOpen},
   778  				{K: text.Scalar},
   779  				{E: eofErr},
   780  			},
   781  		},
   782  		{
   783  			in: `name: [` + space + `"hello"` + space + `]` + space,
   784  			want: []R{
   785  				{K: text.Name},
   786  				{K: text.ListOpen},
   787  				{K: text.Scalar, T: ST{ok: Str{"hello"}}, P: len(space) + 7},
   788  				{K: text.ListClose},
   789  				{K: text.EOF},
   790  			},
   791  		},
   792  		{
   793  			in: `name: ["hello",]`,
   794  			want: []R{
   795  				{K: text.Name},
   796  				{K: text.ListOpen},
   797  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
   798  				{E: `invalid scalar value: ]`},
   799  			},
   800  		},
   801  		{
   802  			in: `name: ["foo"` + space + `'bar' "qux"]`,
   803  			want: []R{
   804  				{K: text.Name},
   805  				{K: text.ListOpen},
   806  				{K: text.Scalar, T: ST{ok: Str{"foobarqux"}}},
   807  				{K: text.ListClose},
   808  				{K: text.EOF},
   809  			},
   810  		},
   811  		{
   812  			in: `name:` + space + `["foo",` + space + "'bar', # comment\n\n" + `"qux"]`,
   813  			want: []R{
   814  				{K: text.Name},
   815  				{K: text.ListOpen},
   816  				{K: text.Scalar, T: ST{ok: Str{"foo"}}},
   817  				{K: text.Scalar, T: ST{ok: Str{"bar"}}},
   818  				{K: text.Scalar, T: ST{ok: Str{"qux"}}},
   819  				{K: text.ListClose},
   820  				{K: text.EOF},
   821  			},
   822  		},
   823  
   824  		{
   825  			// List within list is not allowed.
   826  			in: `name: [[]]`,
   827  			want: []R{
   828  				{K: text.Name},
   829  				{K: text.ListOpen},
   830  				{E: `syntax error (line 1:8): invalid scalar value: [`},
   831  			},
   832  		},
   833  		{
   834  			// List items need to be separated by ,.
   835  			in: `name: ["foo" true]`,
   836  			want: []R{
   837  				{K: text.Name},
   838  				{K: text.ListOpen},
   839  				{K: text.Scalar, T: ST{ok: Str{"foo"}}},
   840  				{E: `syntax error (line 1:14): unexpected character 't'`},
   841  			},
   842  		},
   843  		{
   844  			in: `name: ["foo"; "bar"]`,
   845  			want: []R{
   846  				{K: text.Name},
   847  				{K: text.ListOpen},
   848  				{K: text.Scalar, T: ST{ok: Str{"foo"}}},
   849  				{E: `syntax error (line 1:13): unexpected character ';'`},
   850  			},
   851  		},
   852  		{
   853  			in: `name: ["foo", true, ENUM, 1.0]`,
   854  			want: []R{
   855  				{K: text.Name},
   856  				{K: text.ListOpen},
   857  				{K: text.Scalar, T: ST{ok: Str{"foo"}}},
   858  				{K: text.Scalar, T: ST{ok: Enum{"true"}}},
   859  				{K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
   860  				{K: text.Scalar, T: ST{ok: Float32{1.0}}},
   861  				{K: text.ListClose},
   862  			},
   863  		},
   864  
   865  		// Boolean literal values.
   866  		{
   867  			in: `name: True`,
   868  			want: []R{
   869  				{K: text.Name},
   870  				{
   871  					K: text.Scalar,
   872  					T: ST{ok: Bool{true}},
   873  				},
   874  				{K: text.EOF},
   875  			},
   876  		},
   877  		{
   878  			in: `name false`,
   879  			want: []R{
   880  				{K: text.Name},
   881  				{
   882  					K: text.Scalar,
   883  					T: ST{ok: Bool{false}},
   884  				},
   885  				{K: text.EOF},
   886  			},
   887  		},
   888  		{
   889  			in: `name: [t, f, True, False, true, false, 1, 0, 0x01, 0x00, 01, 00]`,
   890  			want: []R{
   891  				{K: text.Name},
   892  				{K: text.ListOpen},
   893  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   894  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   895  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   896  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   897  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   898  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   899  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   900  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   901  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   902  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   903  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   904  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   905  				{K: text.ListClose},
   906  			},
   907  		},
   908  		{
   909  			// Looks like boolean but not.
   910  			in: `name: [tRUe, falSE, -1, -0, -0x01, -0x00, -01, -00, 0.0]`,
   911  			want: []R{
   912  				{K: text.Name},
   913  				{K: text.ListOpen},
   914  				{K: text.Scalar, T: ST{nok: Bool{}}},
   915  				{K: text.Scalar, T: ST{nok: Bool{}}},
   916  				{K: text.Scalar, T: ST{nok: Bool{}}},
   917  				{K: text.Scalar, T: ST{nok: Bool{}}},
   918  				{K: text.Scalar, T: ST{nok: Bool{}}},
   919  				{K: text.Scalar, T: ST{nok: Bool{}}},
   920  				{K: text.Scalar, T: ST{nok: Bool{}}},
   921  				{K: text.Scalar, T: ST{nok: Bool{}}},
   922  				{K: text.Scalar, T: ST{nok: Bool{}}},
   923  				{K: text.ListClose},
   924  			},
   925  		},
   926  		{
   927  			in: `foo: true[bar] false`,
   928  			want: []R{
   929  				{K: text.Name},
   930  				{K: text.Scalar, T: ST{ok: Bool{true}}},
   931  				{K: text.Name},
   932  				{K: text.Scalar, T: ST{ok: Bool{false}}},
   933  			},
   934  		},
   935  
   936  		// Enum field values.
   937  		{
   938  			in: space + `name: ENUM`,
   939  			want: []R{
   940  				{K: text.Name},
   941  				{K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
   942  			},
   943  		},
   944  		{
   945  			in: space + `name:[TRUE, FALSE, T, F, t, f]`,
   946  			want: []R{
   947  				{K: text.Name},
   948  				{K: text.ListOpen},
   949  				{K: text.Scalar, T: ST{ok: Enum{"TRUE"}}},
   950  				{K: text.Scalar, T: ST{ok: Enum{"FALSE"}}},
   951  				{K: text.Scalar, T: ST{ok: Enum{"T"}}},
   952  				{K: text.Scalar, T: ST{ok: Enum{"F"}}},
   953  				{K: text.Scalar, T: ST{ok: Enum{"t"}}},
   954  				{K: text.Scalar, T: ST{ok: Enum{"f"}}},
   955  				{K: text.ListClose},
   956  			},
   957  		},
   958  		{
   959  			in: `foo: Enum1[bar]:Enum2`,
   960  			want: []R{
   961  				{K: text.Name},
   962  				{K: text.Scalar, T: ST{ok: Enum{"Enum1"}}},
   963  				{K: text.Name},
   964  				{K: text.Scalar, T: ST{ok: Enum{"Enum2"}}},
   965  			},
   966  		},
   967  		{
   968  			// Invalid enum values.
   969  			in: `name: [-inf, -foo, "string", 42, 1.0, 0x47]`,
   970  			want: []R{
   971  				{K: text.Name},
   972  				{K: text.ListOpen},
   973  				{K: text.Scalar, T: ST{nok: Enum{}}},
   974  				{K: text.Scalar, T: ST{nok: Enum{}}},
   975  				{K: text.Scalar, T: ST{nok: Enum{}}},
   976  				{K: text.Scalar, T: ST{nok: Enum{}}},
   977  				{K: text.Scalar, T: ST{nok: Enum{}}},
   978  				{K: text.Scalar, T: ST{nok: Enum{}}},
   979  				{K: text.ListClose},
   980  			},
   981  		},
   982  		{
   983  			in: `name: true.`,
   984  			want: []R{
   985  				{K: text.Name},
   986  				{E: `invalid scalar value: true.`},
   987  			},
   988  		},
   989  
   990  		// Numeric values.
   991  		{
   992  			in: `nums:42 nums:0x2A nums:052`,
   993  			want: []R{
   994  				{K: text.Name},
   995  				{K: text.Scalar, T: ST{ok: Uint64{42}}},
   996  				{K: text.Name},
   997  				{K: text.Scalar, T: ST{ok: Uint64{42}}},
   998  				{K: text.Name},
   999  				{K: text.Scalar, T: ST{ok: Uint64{42}}},
  1000  			},
  1001  		},
  1002  		{
  1003  			in: `nums:[-42, -0x2a, -052]`,
  1004  			want: []R{
  1005  				{K: text.Name},
  1006  				{K: text.ListOpen},
  1007  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1008  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1009  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1010  				{K: text.ListClose},
  1011  			},
  1012  		},
  1013  		{
  1014  			in: `nums:[-42, -0x2a, -052]`,
  1015  			want: []R{
  1016  				{K: text.Name},
  1017  				{K: text.ListOpen},
  1018  				{K: text.Scalar, T: ST{ok: Int64{-42}}},
  1019  				{K: text.Scalar, T: ST{ok: Int64{-42}}},
  1020  				{K: text.Scalar, T: ST{ok: Int64{-42}}},
  1021  				{K: text.ListClose},
  1022  			},
  1023  		},
  1024  		{
  1025  			in: `nums: [0,0x0,00,-9876543210,9876543210,0x0123456789abcdef,-0x0123456789abcdef,01234567,-01234567]`,
  1026  			want: []R{
  1027  				{K: text.Name},
  1028  				{K: text.ListOpen},
  1029  				{K: text.Scalar, T: ST{ok: Uint64{0}}},
  1030  				{K: text.Scalar, T: ST{ok: Int64{0}}},
  1031  				{K: text.Scalar, T: ST{ok: Uint64{0}}},
  1032  				{K: text.Scalar, T: ST{ok: Int64{-9876543210}}},
  1033  				{K: text.Scalar, T: ST{ok: Uint64{9876543210}}},
  1034  				{K: text.Scalar, T: ST{ok: Uint64{0x0123456789abcdef}}},
  1035  				{K: text.Scalar, T: ST{ok: Int64{-0x0123456789abcdef}}},
  1036  				{K: text.Scalar, T: ST{ok: Uint64{01234567}}},
  1037  				{K: text.Scalar, T: ST{ok: Int64{-01234567}}},
  1038  				{K: text.ListClose},
  1039  			},
  1040  		},
  1041  		{
  1042  			in: `nums: [0,0x0,00,-876543210,876543210,0x01234,-0x01234,01234567,-01234567]`,
  1043  			want: []R{
  1044  				{K: text.Name},
  1045  				{K: text.ListOpen},
  1046  				{K: text.Scalar, T: ST{ok: Uint32{0}}},
  1047  				{K: text.Scalar, T: ST{ok: Int32{0}}},
  1048  				{K: text.Scalar, T: ST{ok: Uint32{0}}},
  1049  				{K: text.Scalar, T: ST{ok: Int32{-876543210}}},
  1050  				{K: text.Scalar, T: ST{ok: Uint32{876543210}}},
  1051  				{K: text.Scalar, T: ST{ok: Uint32{0x01234}}},
  1052  				{K: text.Scalar, T: ST{ok: Int32{-0x01234}}},
  1053  				{K: text.Scalar, T: ST{ok: Uint32{01234567}}},
  1054  				{K: text.Scalar, T: ST{ok: Int32{-01234567}}},
  1055  				{K: text.ListClose},
  1056  			},
  1057  		},
  1058  		{
  1059  			in: `nums: [` +
  1060  				fmt.Sprintf("%d", uint64(math.MaxUint64)) + `,` +
  1061  				fmt.Sprintf("%d", uint32(math.MaxUint32)) + `,` +
  1062  				fmt.Sprintf("%d", int64(math.MaxInt64)) + `,` +
  1063  				fmt.Sprintf("%d", int64(math.MinInt64)) + `,` +
  1064  				fmt.Sprintf("%d", int32(math.MaxInt32)) + `,` +
  1065  				fmt.Sprintf("%d", int32(math.MinInt32)) +
  1066  				`]`,
  1067  			want: []R{
  1068  				{K: text.Name},
  1069  				{K: text.ListOpen},
  1070  				{K: text.Scalar, T: ST{ok: Uint64{math.MaxUint64}}},
  1071  				{K: text.Scalar, T: ST{ok: Uint32{math.MaxUint32}}},
  1072  				{K: text.Scalar, T: ST{ok: Int64{math.MaxInt64}}},
  1073  				{K: text.Scalar, T: ST{ok: Int64{math.MinInt64}}},
  1074  				{K: text.Scalar, T: ST{ok: Int32{math.MaxInt32}}},
  1075  				{K: text.Scalar, T: ST{ok: Int32{math.MinInt32}}},
  1076  				{K: text.ListClose},
  1077  			},
  1078  		},
  1079  		{
  1080  			// Integer exceeds range.
  1081  			in: `nums: [` +
  1082  				`18446744073709551616,` + // max uint64 + 1
  1083  				fmt.Sprintf("%d", uint64(math.MaxUint32+1)) + `,` +
  1084  				fmt.Sprintf("%d", uint64(math.MaxInt64+1)) + `,` +
  1085  				`-9223372036854775809,` + // min int64 - 1
  1086  				fmt.Sprintf("%d", uint64(math.MaxInt32+1)) + `,` +
  1087  				fmt.Sprintf("%d", int64(math.MinInt32-1)) + `` +
  1088  				`]`,
  1089  			want: []R{
  1090  				{K: text.Name},
  1091  				{K: text.ListOpen},
  1092  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1093  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1094  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1095  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1096  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1097  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1098  				{K: text.ListClose},
  1099  			},
  1100  		},
  1101  		{
  1102  			in: `nums: [0xbeefbeef, 0xbeefbeefbeefbeef]`,
  1103  			want: []R{
  1104  				{K: text.Name},
  1105  				{K: text.ListOpen},
  1106  				{
  1107  					K: text.Scalar,
  1108  					T: func() ST {
  1109  						if flags.ProtoLegacy {
  1110  							return ST{ok: Int32{-1091584273}}
  1111  						}
  1112  						return ST{nok: Int32{}}
  1113  					}(),
  1114  				},
  1115  				{
  1116  					K: text.Scalar,
  1117  					T: func() ST {
  1118  						if flags.ProtoLegacy {
  1119  							return ST{ok: Int64{-4688318750159552785}}
  1120  						}
  1121  						return ST{nok: Int64{}}
  1122  					}(),
  1123  				},
  1124  				{K: text.ListClose},
  1125  			},
  1126  		},
  1127  		{
  1128  			in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`,
  1129  			want: []R{
  1130  				{K: text.Name},
  1131  				{K: text.ListOpen},
  1132  				{K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1133  				{K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1134  				{K: text.Scalar, T: ST{ok: Float64{1.0}}},
  1135  				{K: text.Scalar, T: ST{ok: Float64{10.0}}},
  1136  				{K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1137  				{K: text.Scalar, T: ST{ok: Float64{-1.0}}},
  1138  				{K: text.Scalar, T: ST{ok: Float64{-10.0}}},
  1139  				{K: text.Scalar, T: ST{ok: Float64{1.0}}},
  1140  				{K: text.Scalar, T: ST{ok: Float64{0.1e-3}}},
  1141  				{K: text.Scalar, T: ST{ok: Float64{1.5e+5}}},
  1142  				{K: text.Scalar, T: ST{ok: Float64{1.0e+10}}},
  1143  				{K: text.Scalar, T: ST{ok: Float64{0.0}}},
  1144  				{K: text.ListClose},
  1145  			},
  1146  		},
  1147  		{
  1148  			in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`,
  1149  			want: []R{
  1150  				{K: text.Name},
  1151  				{K: text.ListOpen},
  1152  				{K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1153  				{K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1154  				{K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1155  				{K: text.Scalar, T: ST{ok: Float32{10.0}}},
  1156  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1157  				{K: text.Scalar, T: ST{ok: Float32{-1.0}}},
  1158  				{K: text.Scalar, T: ST{ok: Float32{-10.0}}},
  1159  				{K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1160  				{K: text.Scalar, T: ST{ok: Float32{0.1e-3}}},
  1161  				{K: text.Scalar, T: ST{ok: Float32{1.5e+5}}},
  1162  				{K: text.Scalar, T: ST{ok: Float32{1.0e+10}}},
  1163  				{K: text.Scalar, T: ST{ok: Float32{0.0}}},
  1164  				{K: text.ListClose},
  1165  			},
  1166  		},
  1167  		{
  1168  			in: `nums: [0.,1f,10F,1e1,1.10]`,
  1169  			want: []R{
  1170  				{K: text.Name},
  1171  				{K: text.ListOpen},
  1172  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1173  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1174  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1175  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1176  				{K: text.Scalar, T: ST{nok: Int64{}}},
  1177  				{K: text.ListClose},
  1178  			},
  1179  		},
  1180  		{
  1181  			in: `nums: [0.,1f,10F,1e1,1.10]`,
  1182  			want: []R{
  1183  				{K: text.Name},
  1184  				{K: text.ListOpen},
  1185  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1186  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1187  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1188  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1189  				{K: text.Scalar, T: ST{nok: Int32{}}},
  1190  				{K: text.ListClose},
  1191  			},
  1192  		},
  1193  		{
  1194  			in: `nums: [0.,1f,10F,1e1,1.10]`,
  1195  			want: []R{
  1196  				{K: text.Name},
  1197  				{K: text.ListOpen},
  1198  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1199  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1200  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1201  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1202  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1203  				{K: text.ListClose},
  1204  			},
  1205  		},
  1206  		{
  1207  			in: `nums: [0.,1f,10F,1e1,1.10]`,
  1208  			want: []R{
  1209  				{K: text.Name},
  1210  				{K: text.ListOpen},
  1211  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1212  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1213  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1214  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1215  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1216  				{K: text.ListClose},
  1217  			},
  1218  		},
  1219  		{
  1220  			in: `nums: [` +
  1221  				fmt.Sprintf("%g", math.MaxFloat32) + `,` +
  1222  				fmt.Sprintf("%g", -math.MaxFloat32) + `,` +
  1223  				fmt.Sprintf("%g", math.MaxFloat32*2) + `,` +
  1224  				fmt.Sprintf("%g", -math.MaxFloat32*2) + `,` +
  1225  				`3.59539e+308,` + // math.MaxFloat64 * 2
  1226  				`-3.59539e+308,` + // -math.MaxFloat64 * 2
  1227  				fmt.Sprintf("%d000", uint64(math.MaxUint64)) +
  1228  				`]`,
  1229  			want: []R{
  1230  				{K: text.Name},
  1231  				{K: text.ListOpen},
  1232  				{K: text.Scalar, T: ST{ok: Float32{float32(math.MaxFloat32)}}},
  1233  				{K: text.Scalar, T: ST{ok: Float32{float32(-math.MaxFloat32)}}},
  1234  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1235  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1236  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1237  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1238  				{K: text.Scalar, T: ST{ok: Float32{float32(math.MaxUint64) * 1000}}},
  1239  				{K: text.ListClose},
  1240  			},
  1241  		},
  1242  		{
  1243  			in: `nums: [` +
  1244  				fmt.Sprintf("%g", math.MaxFloat64) + `,` +
  1245  				fmt.Sprintf("%g", -math.MaxFloat64) + `,` +
  1246  				`3.59539e+308,` + // math.MaxFloat64 * 2
  1247  				`-3.59539e+308,` + // -math.MaxFloat64 * 2
  1248  				fmt.Sprintf("%d000", uint64(math.MaxUint64)) +
  1249  				`]`,
  1250  			want: []R{
  1251  				{K: text.Name},
  1252  				{K: text.ListOpen},
  1253  				{K: text.Scalar, T: ST{ok: Float64{math.MaxFloat64}}},
  1254  				{K: text.Scalar, T: ST{ok: Float64{-math.MaxFloat64}}},
  1255  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1256  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1257  				{K: text.Scalar, T: ST{ok: Float64{float64(math.MaxUint64) * 1000}}},
  1258  				{K: text.ListClose},
  1259  			},
  1260  		},
  1261  		{
  1262  			// -0 is only valid for signed types. It is not valid for unsigned types.
  1263  			in: `num: [-0, -0]`,
  1264  			want: []R{
  1265  				{K: text.Name},
  1266  				{K: text.ListOpen},
  1267  				{K: text.Scalar, T: ST{nok: Uint32{}}},
  1268  				{K: text.Scalar, T: ST{nok: Uint64{}}},
  1269  				{K: text.ListClose},
  1270  			},
  1271  		},
  1272  		{
  1273  			// -0 is only valid for signed types. It is not valid for unsigned types.
  1274  			in: `num: [-0, -0]`,
  1275  			want: []R{
  1276  				{K: text.Name},
  1277  				{K: text.ListOpen},
  1278  				{K: text.Scalar, T: ST{ok: Int32{0}}},
  1279  				{K: text.Scalar, T: ST{ok: Int64{0}}},
  1280  				{K: text.ListClose},
  1281  			},
  1282  		},
  1283  		{
  1284  			// Negative zeros on float64 should preserve sign bit.
  1285  			in: `num: [-0, -.0]`,
  1286  			want: []R{
  1287  				{K: text.Name},
  1288  				{K: text.ListOpen},
  1289  				{K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1290  				{K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}},
  1291  				{K: text.ListClose},
  1292  			},
  1293  		},
  1294  		{
  1295  			// Negative zeros on float32 should preserve sign bit.
  1296  			in: `num: [-0, -.0]`,
  1297  			want: []R{
  1298  				{K: text.Name},
  1299  				{K: text.ListOpen},
  1300  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1301  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}},
  1302  				{K: text.ListClose},
  1303  			},
  1304  		},
  1305  		{
  1306  			in: `num: +0`,
  1307  			want: []R{
  1308  				{K: text.Name},
  1309  				{E: `invalid scalar value: +`},
  1310  			},
  1311  		},
  1312  		{
  1313  			in: `num: 01.1234`,
  1314  			want: []R{
  1315  				{K: text.Name},
  1316  				{E: `invalid scalar value: 01.1234`},
  1317  			},
  1318  		},
  1319  		{
  1320  			in: `num: 0x`,
  1321  			want: []R{
  1322  				{K: text.Name},
  1323  				{E: `invalid scalar value: 0x`},
  1324  			},
  1325  		},
  1326  		{
  1327  			in: `num: 0xX`,
  1328  			want: []R{
  1329  				{K: text.Name},
  1330  				{E: `invalid scalar value: 0xX`},
  1331  			},
  1332  		},
  1333  		{
  1334  			in: `num: 0800`,
  1335  			want: []R{
  1336  				{K: text.Name},
  1337  				{E: `invalid scalar value: 0800`},
  1338  			},
  1339  		},
  1340  		{
  1341  			in: `num: 1.`,
  1342  			want: []R{
  1343  				{K: text.Name},
  1344  				{K: text.Scalar, T: ST{ok: Float32{1.0}}},
  1345  			},
  1346  		},
  1347  		{
  1348  			in: `num: -.`,
  1349  			want: []R{
  1350  				{K: text.Name},
  1351  				{E: `invalid scalar value: -.`},
  1352  			},
  1353  		},
  1354  
  1355  		// Float special literal values, case-insensitive match.
  1356  		{
  1357  			in: `name:[nan, NaN, Nan, NAN]`,
  1358  			want: []R{
  1359  				{K: text.Name},
  1360  				{K: text.ListOpen},
  1361  				{K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1362  				{K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1363  				{K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1364  				{K: text.Scalar, T: ST{ok: Float64{math.NaN()}}},
  1365  				{K: text.ListClose},
  1366  			},
  1367  		},
  1368  		{
  1369  			in: `name:[inf, INF, infinity, Infinity, INFinity]`,
  1370  			want: []R{
  1371  				{K: text.Name},
  1372  				{K: text.ListOpen},
  1373  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1374  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1375  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1376  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1377  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}},
  1378  				{K: text.ListClose},
  1379  			},
  1380  		},
  1381  		{
  1382  			in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`,
  1383  			want: []R{
  1384  				{K: text.Name},
  1385  				{K: text.ListOpen},
  1386  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1387  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1388  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1389  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1390  				{K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}},
  1391  				{K: text.ListClose},
  1392  			},
  1393  		},
  1394  		{
  1395  			in: `name:[nan, NaN, Nan, NAN]`,
  1396  			want: []R{
  1397  				{K: text.Name},
  1398  				{K: text.ListOpen},
  1399  				{K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1400  				{K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1401  				{K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1402  				{K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}},
  1403  				{K: text.ListClose},
  1404  			},
  1405  		},
  1406  		{
  1407  			in: `name:[inf, INF, infinity, Infinity, INFinity]`,
  1408  			want: []R{
  1409  				{K: text.Name},
  1410  				{K: text.ListOpen},
  1411  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1412  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1413  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1414  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1415  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}},
  1416  				{K: text.ListClose},
  1417  			},
  1418  		},
  1419  		{
  1420  			in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`,
  1421  			want: []R{
  1422  				{K: text.Name},
  1423  				{K: text.ListOpen},
  1424  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1425  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1426  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1427  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1428  				{K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}},
  1429  				{K: text.ListClose},
  1430  			},
  1431  		},
  1432  		{
  1433  			// C++ permits this, but we currently reject this. It is easy to add
  1434  			// if needed.
  1435  			in: `name: -nan`,
  1436  			want: []R{
  1437  				{K: text.Name},
  1438  				{K: text.Scalar, T: ST{nok: Float64{}}},
  1439  			},
  1440  		},
  1441  		// Messages.
  1442  		{
  1443  			in: `m: {}`,
  1444  			want: []R{
  1445  				{K: text.Name},
  1446  				{K: text.MessageOpen},
  1447  				{K: text.MessageClose},
  1448  				{K: text.EOF},
  1449  			},
  1450  		},
  1451  		{
  1452  			in: `m: <>`,
  1453  			want: []R{
  1454  				{K: text.Name},
  1455  				{K: text.MessageOpen},
  1456  				{K: text.MessageClose},
  1457  				{K: text.EOF},
  1458  			},
  1459  		},
  1460  		{
  1461  			in: space + `m {` + space + "\n# comment\n" + `}` + space,
  1462  			want: []R{
  1463  				{K: text.Name},
  1464  				{K: text.MessageOpen},
  1465  				{K: text.MessageClose},
  1466  			},
  1467  		},
  1468  		{
  1469  			in: `m { foo: < bar: "hello" > }`,
  1470  			want: []R{
  1471  				{K: text.Name, RS: "m"},
  1472  				{K: text.MessageOpen},
  1473  
  1474  				{K: text.Name, RS: "foo"},
  1475  				{K: text.MessageOpen},
  1476  
  1477  				{K: text.Name, RS: "bar"},
  1478  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1479  
  1480  				{K: text.MessageClose},
  1481  
  1482  				{K: text.MessageClose},
  1483  			},
  1484  		},
  1485  		{
  1486  			in: `list [ <s:"hello">, {s:"world"} ]`,
  1487  			want: []R{
  1488  				{K: text.Name, RS: "list"},
  1489  				{K: text.ListOpen},
  1490  
  1491  				{K: text.MessageOpen},
  1492  				{K: text.Name, RS: "s"},
  1493  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1494  				{K: text.MessageClose},
  1495  
  1496  				{K: text.MessageOpen},
  1497  				{K: text.Name, RS: "s"},
  1498  				{K: text.Scalar, T: ST{ok: Str{"world"}}},
  1499  				{K: text.MessageClose},
  1500  
  1501  				{K: text.ListClose},
  1502  				{K: text.EOF},
  1503  			},
  1504  		},
  1505  		{
  1506  			in: `m: { >`,
  1507  			want: []R{
  1508  				{K: text.Name},
  1509  				{K: text.MessageOpen},
  1510  				{E: `mismatched close character '>'`},
  1511  			},
  1512  		},
  1513  		{
  1514  			in: `m: <s: "hello"}`,
  1515  			want: []R{
  1516  				{K: text.Name},
  1517  				{K: text.MessageOpen},
  1518  
  1519  				{K: text.Name},
  1520  				{K: text.Scalar, T: ST{ok: Str{"hello"}}},
  1521  
  1522  				{E: `mismatched close character '}'`},
  1523  			},
  1524  		},
  1525  		{
  1526  			in:   `{}`,
  1527  			want: []R{{E: `invalid field name: {`}},
  1528  		},
  1529  		{
  1530  			in: `
  1531  m: {
  1532    foo: true;
  1533    bar: {
  1534  	enum: ENUM
  1535  	list: [ < >, { } ] ;
  1536    }
  1537    [qux]: "end"
  1538  }
  1539  				`,
  1540  			want: []R{
  1541  				{K: text.Name},
  1542  				{K: text.MessageOpen},
  1543  
  1544  				{K: text.Name, RS: "foo"},
  1545  				{K: text.Scalar, T: ST{ok: Bool{true}}},
  1546  
  1547  				{K: text.Name, RS: "bar"},
  1548  				{K: text.MessageOpen},
  1549  
  1550  				{K: text.Name, RS: "enum"},
  1551  				{K: text.Scalar, T: ST{ok: Enum{"ENUM"}}},
  1552  
  1553  				{K: text.Name, RS: "list"},
  1554  				{K: text.ListOpen},
  1555  				{K: text.MessageOpen},
  1556  				{K: text.MessageClose},
  1557  				{K: text.MessageOpen},
  1558  				{K: text.MessageClose},
  1559  				{K: text.ListClose},
  1560  
  1561  				{K: text.MessageClose},
  1562  
  1563  				{K: text.Name, RS: "[qux]"},
  1564  				{K: text.Scalar, T: ST{ok: Str{"end"}}},
  1565  
  1566  				{K: text.MessageClose},
  1567  				{K: text.EOF},
  1568  			},
  1569  		},
  1570  
  1571  		// Other syntax errors.
  1572  		{
  1573  			in: "x: -",
  1574  			want: []R{
  1575  				{K: text.Name},
  1576  				{E: `syntax error (line 1:4): invalid scalar value: -`},
  1577  			},
  1578  		},
  1579  		{
  1580  			in: "x:[\"💩\"x",
  1581  			want: []R{
  1582  				{K: text.Name},
  1583  				{K: text.ListOpen},
  1584  				{K: text.Scalar, T: ST{ok: Str{"💩"}}, P: 3},
  1585  				{E: `syntax error (line 1:7)`},
  1586  			},
  1587  		},
  1588  		{
  1589  			in: "x:\n\n[\"🔥🔥🔥\"x",
  1590  			want: []R{
  1591  				{K: text.Name},
  1592  				{K: text.ListOpen},
  1593  				{K: text.Scalar, T: ST{ok: Str{"🔥🔥🔥"}}, P: 5},
  1594  				{E: `syntax error (line 3:7)`},
  1595  			},
  1596  		},
  1597  		{
  1598  			// multi-rune emojis; could be column:8
  1599  			in: "x:[\"👍🏻👍🏿\"x",
  1600  			want: []R{
  1601  				{K: text.Name},
  1602  				{K: text.ListOpen},
  1603  				{K: text.Scalar, T: ST{ok: Str{"👍🏻👍🏿"}}, P: 3},
  1604  				{E: `syntax error (line 1:10)`},
  1605  			},
  1606  		},
  1607  	}
  1608  
  1609  	for _, tc := range tests {
  1610  		t.Run("", func(t *testing.T) {
  1611  			tc := tc
  1612  			in := []byte(tc.in)
  1613  			dec := text.NewDecoder(in[:len(in):len(in)])
  1614  			for i, want := range tc.want {
  1615  				peekTok, peekErr := dec.Peek()
  1616  				tok, err := dec.Read()
  1617  				if err != nil {
  1618  					if want.E == "" {
  1619  						errorf(t, tc.in, "Read() got unexpected error: %v", err)
  1620  					} else if !strings.Contains(err.Error(), want.E) {
  1621  						errorf(t, tc.in, "Read() got %q, want %q", err, want.E)
  1622  					}
  1623  					return
  1624  				}
  1625  				if want.E != "" {
  1626  					errorf(t, tc.in, "Read() got nil error, want %q", want.E)
  1627  					return
  1628  				}
  1629  				gotK := tok.Kind()
  1630  				if gotK != want.K {
  1631  					errorf(t, tc.in, "Read() got %v, want %v", gotK, want.K)
  1632  					return
  1633  				}
  1634  				checkToken(t, tok, i, want, tc.in)
  1635  				if !cmp.Equal(tok, peekTok, cmp.Comparer(text.TokenEquals)) {
  1636  					errorf(t, tc.in, "Peek() %+v != Read() token %+v", peekTok, tok)
  1637  				}
  1638  				if err != peekErr {
  1639  					errorf(t, tc.in, "Peek() error %v != Read() error %v", err, peekErr)
  1640  				}
  1641  			}
  1642  		})
  1643  	}
  1644  }
  1645  
  1646  func checkToken(t *testing.T, tok text.Token, idx int, r R, in string) {
  1647  	// Validate Token.Pos() if R.P is set.
  1648  	if r.P > 0 {
  1649  		got := tok.Pos()
  1650  		if got != r.P {
  1651  			errorf(t, in, "want#%d: Token.Pos() got %v want %v", idx, got, r.P)
  1652  		}
  1653  	}
  1654  
  1655  	// Validate Token.RawString if R.RS is set.
  1656  	if len(r.RS) > 0 {
  1657  		got := tok.RawString()
  1658  		if got != r.RS {
  1659  			errorf(t, in, "want#%d: Token.RawString() got %v want %v", idx, got, r.P)
  1660  		}
  1661  	}
  1662  
  1663  	// Skip checking for Token details if r.T is not set.
  1664  	if r.T == nil {
  1665  		return
  1666  	}
  1667  
  1668  	switch tok.Kind() {
  1669  	case text.Name:
  1670  		want := r.T.(NT)
  1671  		kind := tok.NameKind()
  1672  		if kind != want.K {
  1673  			errorf(t, in, "want#%d: Token.NameKind() got %v want %v", idx, kind, want.K)
  1674  			return
  1675  		}
  1676  		switch kind {
  1677  		case text.IdentName:
  1678  			got := tok.IdentName()
  1679  			if got != want.S {
  1680  				errorf(t, in, "want#%d: Token.IdentName() got %v want %v", idx, got, want.S)
  1681  			}
  1682  		case text.TypeName:
  1683  			got := tok.TypeName()
  1684  			if got != want.S {
  1685  				errorf(t, in, "want#%d: Token.TypeName() got %v want %v", idx, got, want.S)
  1686  			}
  1687  		case text.FieldNumber:
  1688  			got := tok.FieldNumber()
  1689  			if got != want.N {
  1690  				errorf(t, in, "want#%d: Token.FieldNumber() got %v want %v", idx, got, want.N)
  1691  			}
  1692  		}
  1693  
  1694  	case text.Scalar:
  1695  		want := r.T.(ST)
  1696  		if ok := want.ok; ok != nil {
  1697  			if err := ok.checkOk(tok); err != "" {
  1698  				errorf(t, in, "want#%d: %s", idx, err)
  1699  			}
  1700  		}
  1701  		if nok := want.nok; nok != nil {
  1702  			if err := nok.checkNok(tok); err != "" {
  1703  				errorf(t, in, "want#%d: %s", idx, err)
  1704  			}
  1705  		}
  1706  	}
  1707  }
  1708  
  1709  func errorf(t *testing.T, in string, fmtStr string, args ...interface{}) {
  1710  	t.Helper()
  1711  	vargs := []interface{}{in}
  1712  	for _, arg := range args {
  1713  		vargs = append(vargs, arg)
  1714  	}
  1715  	t.Errorf("input:\n%s\n~end~\n"+fmtStr, vargs...)
  1716  }
  1717  
  1718  func TestUnmarshalString(t *testing.T) {
  1719  	tests := []struct {
  1720  		in string
  1721  		// want is expected string result.
  1722  		want string
  1723  		// err is expected error substring from calling DecodeString if set.
  1724  		err string
  1725  	}{
  1726  		{
  1727  			in: func() string {
  1728  				var b []byte
  1729  				for i := 0; i < utf8.RuneSelf; i++ {
  1730  					switch i {
  1731  					case 0, '\\', '\n', '\'': // these must be escaped, so ignore them
  1732  					default:
  1733  						b = append(b, byte(i))
  1734  					}
  1735  				}
  1736  				return "'" + string(b) + "'"
  1737  			}(),
  1738  			want: "\x01\x02\x03\x04\x05\x06\a\b\t\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f",
  1739  		},
  1740  		{
  1741  			in:  "'\xde\xad\xbe\xef'",
  1742  			err: `invalid UTF-8 detected`,
  1743  		},
  1744  		{
  1745  			// Valid UTF-8 wire encoding, but sub-optimal encoding.
  1746  			in:  "'\xc0\x80'",
  1747  			err: "invalid UTF-8 detected",
  1748  		},
  1749  		{
  1750  			// Valid UTF-8 wire encoding, but invalid rune (surrogate pair).
  1751  			in:  "'\xed\xa0\x80'",
  1752  			err: "invalid UTF-8 detected",
  1753  		},
  1754  		{
  1755  			// Valid UTF-8 wire encoding, but invalid rune (above max rune).
  1756  			in:  "'\xf7\xbf\xbf\xbf'",
  1757  			err: "invalid UTF-8 detected",
  1758  		},
  1759  		{
  1760  			// Valid UTF-8 wire encoding of the RuneError rune.
  1761  			in:   "'\xef\xbf\xbd'",
  1762  			want: string(utf8.RuneError),
  1763  		},
  1764  		{
  1765  			in:   "'hello\u1234world'",
  1766  			want: "hello\u1234world",
  1767  		},
  1768  		{
  1769  			in:   `'\"\'\\\?\a\b\n\r\t\v\f\1\12\123\xA\xaB\x12\uAb8f\U0010FFFF'`,
  1770  			want: "\"'\\?\a\b\n\r\t\v\f\x01\nS\n\xab\x12\uab8f\U0010ffff",
  1771  		},
  1772  		{
  1773  			in:  `str: '\8'`,
  1774  			err: `invalid escape code "\\8" in string`,
  1775  		},
  1776  		{
  1777  			in:   `'\1x'`,
  1778  			want: "\001x",
  1779  		},
  1780  		{
  1781  			in:   `'\12x'`,
  1782  			want: "\012x",
  1783  		},
  1784  		{
  1785  			in:   `'\123x'`,
  1786  			want: "\123x",
  1787  		},
  1788  		{
  1789  			in:   `'\1234x'`,
  1790  			want: "\1234x",
  1791  		},
  1792  		{
  1793  			in:   `'\1'`,
  1794  			want: "\001",
  1795  		},
  1796  		{
  1797  			in:   `'\12'`,
  1798  			want: "\012",
  1799  		},
  1800  		{
  1801  			in:   `'\123'`,
  1802  			want: "\123",
  1803  		},
  1804  		{
  1805  			in:   `'\1234'`,
  1806  			want: "\1234",
  1807  		},
  1808  		{
  1809  			in:   `'\377'`,
  1810  			want: "\377",
  1811  		},
  1812  		{
  1813  			// Overflow octal escape.
  1814  			in:  `'\400'`,
  1815  			err: `invalid octal escape code "\\400" in string`,
  1816  		},
  1817  		{
  1818  			in:   `'\xfx'`,
  1819  			want: "\x0fx",
  1820  		},
  1821  		{
  1822  			in:   `'\xffx'`,
  1823  			want: "\xffx",
  1824  		},
  1825  		{
  1826  			in:   `'\xfffx'`,
  1827  			want: "\xfffx",
  1828  		},
  1829  		{
  1830  			in:   `'\xf'`,
  1831  			want: "\x0f",
  1832  		},
  1833  		{
  1834  			in:   `'\xff'`,
  1835  			want: "\xff",
  1836  		},
  1837  		{
  1838  			in:   `'\xfff'`,
  1839  			want: "\xfff",
  1840  		},
  1841  		{
  1842  			in:  `'\xz'`,
  1843  			err: `invalid hex escape code "\\x" in string`,
  1844  		},
  1845  		{
  1846  			in:  `'\uPo'`,
  1847  			err: eofErr,
  1848  		},
  1849  		{
  1850  			in:  `'\uPoo'`,
  1851  			err: `invalid Unicode escape code "\\uPoo'" in string`,
  1852  		},
  1853  		{
  1854  			in:  `str: '\uPoop'`,
  1855  			err: `invalid Unicode escape code "\\uPoop" in string`,
  1856  		},
  1857  		{
  1858  			// Unmatched surrogate pair.
  1859  			in:  `str: '\uDEAD'`,
  1860  			err: `unexpected EOF`, // trying to reader other half
  1861  		},
  1862  		{
  1863  			// Surrogate pair with invalid other half.
  1864  			in:  `str: '\uDEAD\u0000'`,
  1865  			err: `invalid Unicode escape code "\\u0000" in string`,
  1866  		},
  1867  		{
  1868  			// Properly matched surrogate pair.
  1869  			in:   `'\uD800\uDEAD'`,
  1870  			want: "𐊭",
  1871  		},
  1872  		{
  1873  			// Overflow on Unicode rune.
  1874  			in:  `'\U00110000'`,
  1875  			err: `invalid Unicode escape code "\\U00110000" in string`,
  1876  		},
  1877  		{
  1878  			in:  `'\z'`,
  1879  			err: `invalid escape code "\\z" in string`,
  1880  		},
  1881  		{
  1882  			// Strings cannot have NUL literal since C-style strings forbid them.
  1883  			in:  "'\x00'",
  1884  			err: `invalid character '\x00' in string`,
  1885  		},
  1886  		{
  1887  			// Strings cannot have newline literal. The C++ permits them if an
  1888  			// option is specified to allow them. In Go, we always forbid them.
  1889  			in:  "'\n'",
  1890  			err: `invalid character '\n' in string`,
  1891  		},
  1892  	}
  1893  
  1894  	for _, tc := range tests {
  1895  		t.Run("", func(t *testing.T) {
  1896  			got, err := text.UnmarshalString(tc.in)
  1897  			if err != nil {
  1898  				if tc.err == "" {
  1899  					errorf(t, tc.in, "UnmarshalString() got unexpected error: %q", err)
  1900  				} else if !strings.Contains(err.Error(), tc.err) {
  1901  					errorf(t, tc.in, "UnmarshalString() error got %q, want %q", err, tc.err)
  1902  				}
  1903  				return
  1904  			}
  1905  			if tc.err != "" {
  1906  				errorf(t, tc.in, "UnmarshalString() got nil error, want %q", tc.err)
  1907  				return
  1908  			}
  1909  			if got != tc.want {
  1910  				errorf(t, tc.in, "UnmarshalString()\n[got]\n%s\n[want]\n%s", got, tc.want)
  1911  			}
  1912  		})
  1913  	}
  1914  }
  1915  
  1916  // Tests line and column number produced by Decoder.Position.
  1917  func TestPosition(t *testing.T) {
  1918  	dec := text.NewDecoder([]byte("0123456789\n12345\n789"))
  1919  
  1920  	tests := []struct {
  1921  		pos int
  1922  		row int
  1923  		col int
  1924  	}{
  1925  		{
  1926  			pos: 0,
  1927  			row: 1,
  1928  			col: 1,
  1929  		},
  1930  		{
  1931  			pos: 10,
  1932  			row: 1,
  1933  			col: 11,
  1934  		},
  1935  		{
  1936  			pos: 11,
  1937  			row: 2,
  1938  			col: 1,
  1939  		},
  1940  		{
  1941  			pos: 18,
  1942  			row: 3,
  1943  			col: 2,
  1944  		},
  1945  	}
  1946  
  1947  	for _, tc := range tests {
  1948  		t.Run("", func(t *testing.T) {
  1949  			row, col := dec.Position(tc.pos)
  1950  			if row != tc.row || col != tc.col {
  1951  				t.Errorf("Position(%d) got (%d,%d) want (%d,%d)", tc.pos, row, col, tc.row, tc.col)
  1952  			}
  1953  		})
  1954  	}
  1955  }
  1956  

View as plain text