decoder_test.go

Documentation: github.com/bytedance/sonic/decoder

     1  /*
     2   * Copyright 2021 ByteDance Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package decoder
    18  
    19  import (
    20      `encoding/json`
    21      `runtime`
    22      `runtime/debug`
    23      `strings`
    24      `sync`
    25      `testing`
    26      `time`
    27  
    28      `github.com/davecgh/go-spew/spew`
    29      `github.com/stretchr/testify/assert`
    30      `github.com/stretchr/testify/require`
    31  )
    32  
    33  func TestMain(m *testing.M) {
    34      go func ()  {
    35          if !debugAsyncGC {
    36              return
    37          }
    38          println("Begin GC looping...")
    39          for {
    40             runtime.GC()
    41             debug.FreeOSMemory() 
    42          }
    43          println("stop GC looping!")
    44      }()
    45      time.Sleep(time.Millisecond)
    46      m.Run()
    47  }
    48  
    49  func TestGC(t *testing.T) {
    50      if debugSyncGC {
    51          return 
    52      }
    53      var w interface{}
    54      out, err := decode(TwitterJson, &w, true)
    55      if err != nil {
    56          t.Fatal(err)
    57      }
    58      if out != len(TwitterJson) {
    59          t.Fatal(out)
    60      }
    61      wg := &sync.WaitGroup{}
    62      N := 10000
    63      for i:=0; i<N; i++ {
    64          wg.Add(1)
    65          go func (wg *sync.WaitGroup)  {
    66              defer wg.Done()
    67              var w interface{}
    68              out, err := decode(TwitterJson, &w, true)
    69              if err != nil {
    70                  t.Error(err)
    71                  return
    72              }
    73              if out != len(TwitterJson) {
    74                  t.Error(out)
    75                  return
    76              }
    77              runtime.GC()
    78          }(wg)
    79      }
    80      wg.Wait()
    81  }
    82  
    83  var _BindingValue TwitterStruct
    84  
    85  func init() {
    86      _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
    87  }
    88  
    89  
    90  func TestSkipMismatchTypeError(t *testing.T) {
    91      t.Run("struct", func(t *testing.T) {
    92          println("TestSkipError")
    93          type skiptype struct {
    94              A int `json:"a"`
    95              B string `json:"b"`
    96  
    97              Pass *int `json:"pass"`
    98  
    99              C struct{
   100  
   101                  Pass4 interface{} `json:"pass4"`
   102  
   103                  D struct{
   104                      E float32 `json:"e"`
   105                  } `json:"d"`
   106  
   107                  Pass2 int `json:"pass2"`
   108  
   109              } `json:"c"`
   110  
   111              E bool `json:"e"`
   112              F []int `json:"f"`
   113              G map[string]int `json:"g"`
   114              H bool `json:"h,string"`
   115  
   116              Pass3 int `json:"pass2"`
   117  
   118              I json.Number `json:"i"`
   119          }
   120          var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
   121          var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}`
   122          d := NewDecoder(data)
   123          err := d.Decode(obj)
   124          err2 := json.Unmarshal([]byte(data), obj2)
   125          println(err2.Error())
   126          assert.Equal(t, err2 == nil, err == nil)
   127          // assert.Equal(t, len(data), d.i)
   128          assert.Equal(t, obj2, obj)
   129          if err == nil {
   130              t.Fatal("invalid error")
   131          }
   132      })
   133      t.Run("short array", func(t *testing.T) {
   134          var obj, obj2 = &[]int{}, &[]int{}
   135          var data = `[""]`
   136          d := NewDecoder(data)
   137          err := d.Decode(obj)
   138          err2 := json.Unmarshal([]byte(data), obj2)
   139          // println(err2.Error())
   140          assert.Equal(t, err2 == nil, err == nil)
   141          // assert.Equal(t, len(data), d.i)
   142          assert.Equal(t, obj2, obj)
   143      })
   144  
   145      t.Run("int ", func(t *testing.T) {
   146          var obj int = 123
   147          var obj2 int = 123
   148          var data = `[""]`
   149          d := NewDecoder(data)
   150          err := d.Decode(&obj)
   151          err2 := json.Unmarshal([]byte(data), &obj2)
   152          println(err.Error(), obj, obj2)
   153          assert.Equal(t, err2 == nil, err == nil)
   154          // assert.Equal(t, len(data), d.i)
   155          assert.Equal(t, obj2, obj)
   156      })
   157  
   158      t.Run("array", func(t *testing.T) {
   159          var obj, obj2 = &[]int{}, &[]int{}
   160          var data = `["",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]`
   161          d := NewDecoder(data)
   162          err := d.Decode(obj)
   163          err2 := json.Unmarshal([]byte(data), obj2)
   164          // println(err2.Error())
   165          assert.Equal(t, err2 == nil, err == nil)
   166          // assert.Equal(t, len(data), d.i)
   167          assert.Equal(t, obj2, obj)
   168      })
   169  
   170      t.Run("map", func(t *testing.T) {
   171          var obj, obj2 = &map[int]int{}, &map[int]int{}
   172          var data = `{"true" : { },"1":1,"2" : true,"3":3}`
   173          d := NewDecoder(data)
   174          err := d.Decode(obj)
   175          err2 := json.Unmarshal([]byte(data), obj2)
   176          assert.Equal(t, err2 == nil, err == nil)
   177          // assert.Equal(t, len(data), d.i)
   178          assert.Equal(t, obj2, obj)
   179      })
   180      t.Run("map error", func(t *testing.T) {
   181          var obj, obj2 = &map[int]int{}, &map[int]int{}
   182          var data = `{"true" : { ],"1":1,"2" : true,"3":3}`
   183          d := NewDecoder(data)
   184          err := d.Decode(obj)
   185          err2 := json.Unmarshal([]byte(data), obj2)
   186          println(err.Error())
   187          println(err2.Error())
   188          assert.Equal(t, err2 == nil, err == nil)
   189          // assert.Equal(t, len(data), d.i)
   190          // assert.Equal(t, obj2, obj)
   191      })
   192  }
   193  
   194  func TestDecodeCorrupt(t *testing.T) {
   195      var ds = []string{
   196          `{,}`,
   197          `{,"a"}`,
   198          `{"a":}`,
   199          `{"a":1,}`,
   200          `{"a":1,"b"}`,
   201          `{"a":1,"b":}`,
   202          `{,"a":1 "b":2}`,
   203          `{"a",:1 "b":2}`,
   204          `{"a":,1 "b":2}`,
   205          `{"a":1 "b",:2}`,
   206          `{"a":1 "b":,2}`,
   207          `{"a":1 "b":2,}`,
   208          `{"a":1 "b":2}`,
   209          `[,]`,
   210          `[,1]`,
   211          `[1,]`,
   212          `[,1,2]`,
   213          `[1,2,]`,
   214      }
   215      for _, d := range ds {
   216          var o interface{}
   217          _, err := decode(d, &o, false)
   218          if err == nil {
   219              t.Fatalf("%#v", d)
   220          }
   221          if !strings.Contains(err.Error(), "invalid char"){
   222              t.Fatal(err.Error())
   223          }
   224      }
   225  }
   226  
   227  func TestDecodeOption(t *testing.T) {
   228      var s string
   229      var d *Decoder
   230      var out interface{}
   231      var out2 struct {}
   232      var err error
   233  
   234      s = "123"
   235      d = NewDecoder(s)
   236      d.SetOptions(OptionUseNumber);
   237      err = d.Decode(&out)
   238      assert.NoError(t, err)
   239      assert.Equal(t, out.(json.Number), json.Number("123"))
   240  
   241      d = NewDecoder(s)
   242      err = d.Decode(&out)
   243      assert.NoError(t, err)
   244      assert.Equal(t, out.(float64), float64(123))
   245  
   246      s = `{"un": 123}`
   247      d = NewDecoder(s)
   248      d.SetOptions(OptionDisableUnknown);
   249      err = d.Decode(&out2)
   250      assert.Error(t, err)
   251  
   252      d = NewDecoder(s)
   253      err = d.Decode(&out2)
   254      assert.NoError(t, err)
   255  }
   256  
   257  func decode(s string, v interface{}, copy bool) (int, error) {
   258      d := NewDecoder(s)
   259      if copy {
   260          d.CopyString()
   261      }
   262      err := d.Decode(v)
   263      if err != nil {
   264          return 0, err
   265      }
   266      return len(s), err
   267  }
   268  
   269  func TestDecoder_Basic(t *testing.T) {
   270      var v int
   271      pos, err := decode("12345", &v, false)
   272      assert.NoError(t, err)
   273      assert.Equal(t, 5, pos)
   274      assert.Equal(t, 12345, v)
   275  }
   276  
   277  func TestDecoder_Generic(t *testing.T) {
   278      var v interface{}
   279      pos, err := decode(TwitterJson, &v, false)
   280      assert.NoError(t, err)
   281      assert.Equal(t, len(TwitterJson), pos)
   282  }
   283  
   284  func TestDecoder_Binding(t *testing.T) {
   285      var v TwitterStruct
   286      pos, err := decode(TwitterJson, &v, false)
   287      assert.NoError(t, err)
   288      assert.Equal(t, len(TwitterJson), pos)
   289      assert.Equal(t, _BindingValue, v, 0)
   290      spew.Dump(v)
   291  }
   292  
   293  
   294  func TestDecoder_MapWithIndirectElement(t *testing.T) {
   295      var v map[string]struct { A [129]byte }
   296      _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false)
   297      require.NoError(t, err)
   298      assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A)
   299  }
   300  
   301  func BenchmarkDecoder_Generic_Sonic(b *testing.B) {
   302      var w interface{}
   303      _, _ = decode(TwitterJson, &w, true)
   304      b.SetBytes(int64(len(TwitterJson)))
   305      b.ResetTimer()
   306      for i := 0; i < b.N; i++ {
   307          var v interface{}
   308          _, _ = decode(TwitterJson, &v, true)
   309      }
   310  }
   311  
   312  func BenchmarkDecoder_Generic_Sonic_Fast(b *testing.B) {
   313      var w interface{}
   314      _, _ = decode(TwitterJson, &w, false)
   315      b.SetBytes(int64(len(TwitterJson)))
   316      b.ResetTimer()
   317      for i := 0; i < b.N; i++ {
   318          var v interface{}
   319          _, _ = decode(TwitterJson, &v, false)
   320      }
   321  }
   322  
   323  func BenchmarkDecoder_Generic_StdLib(b *testing.B) {
   324      var w interface{}
   325      m := []byte(TwitterJson)
   326      _ = json.Unmarshal(m, &w)
   327      b.SetBytes(int64(len(TwitterJson)))
   328      b.ResetTimer()
   329      for i := 0; i < b.N; i++ {
   330          var v interface{}
   331          _ = json.Unmarshal(m, &v)
   332      }
   333  }
   334  
   335  func BenchmarkDecoder_Binding_Sonic(b *testing.B) {
   336      var w TwitterStruct
   337      _, _ = decode(TwitterJson, &w, true)
   338      b.SetBytes(int64(len(TwitterJson)))
   339      b.ResetTimer()
   340      for i := 0; i < b.N; i++ {
   341          var v TwitterStruct
   342          _, _ = decode(TwitterJson, &v, true)
   343      }
   344  }
   345  
   346  func BenchmarkDecoder_Binding_Sonic_Fast(b *testing.B) {
   347      var w TwitterStruct
   348      _, _ = decode(TwitterJson, &w, false)
   349      b.SetBytes(int64(len(TwitterJson)))
   350      b.ResetTimer()
   351      for i := 0; i < b.N; i++ {
   352          var v TwitterStruct
   353          _, _ = decode(TwitterJson, &v, false)
   354      }
   355  }
   356  
   357  func BenchmarkDecoder_Binding_StdLib(b *testing.B) {
   358      var w TwitterStruct
   359      m := []byte(TwitterJson)
   360      _ = json.Unmarshal(m, &w)
   361      b.SetBytes(int64(len(TwitterJson)))
   362      b.ResetTimer()
   363      for i := 0; i < b.N; i++ {
   364          var v TwitterStruct
   365          _ = json.Unmarshal(m, &v)
   366      }
   367  }
   368  
   369  func BenchmarkDecoder_Parallel_Generic_Sonic(b *testing.B) {
   370      var w interface{}
   371      _, _ = decode(TwitterJson, &w, true)
   372      b.SetBytes(int64(len(TwitterJson)))
   373      b.ResetTimer()
   374      b.RunParallel(func(pb *testing.PB) {
   375          for pb.Next() {
   376              var v interface{}
   377              _, _ = decode(TwitterJson, &v, true)
   378          }
   379      })
   380  }
   381  
   382  func BenchmarkDecoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
   383      var w interface{}
   384      _, _ = decode(TwitterJson, &w, false)
   385      b.SetBytes(int64(len(TwitterJson)))
   386      b.ResetTimer()
   387      b.RunParallel(func(pb *testing.PB) {
   388          for pb.Next() {
   389              var v interface{}
   390              _, _ = decode(TwitterJson, &v, false)
   391          }
   392      })
   393  }
   394  
   395  func BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) {
   396      var w interface{}
   397      m := []byte(TwitterJson)
   398      _ = json.Unmarshal(m, &w)
   399      b.SetBytes(int64(len(TwitterJson)))
   400      b.ResetTimer()
   401      b.RunParallel(func(pb *testing.PB) {
   402          for pb.Next() {
   403              var v interface{}
   404              _ = json.Unmarshal(m, &v)
   405          }
   406      })
   407  }
   408  
   409  func BenchmarkDecoder_Parallel_Binding_Sonic(b *testing.B) {
   410      var w TwitterStruct
   411      _, _ = decode(TwitterJson, &w, true)
   412      b.SetBytes(int64(len(TwitterJson)))
   413      b.ResetTimer()
   414      b.RunParallel(func(pb *testing.PB) {
   415          for pb.Next() {
   416              var v TwitterStruct
   417              _, _ = decode(TwitterJson, &v, true)
   418          }
   419      })
   420  }
   421  
   422  func BenchmarkDecoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
   423      var w TwitterStruct
   424      _, _ = decode(TwitterJson, &w, false)
   425      b.SetBytes(int64(len(TwitterJson)))
   426      b.ResetTimer()
   427      b.RunParallel(func(pb *testing.PB) {
   428          for pb.Next() {
   429              var v TwitterStruct
   430              _, _ = decode(TwitterJson, &v, false)
   431          }
   432      })
   433  }
   434  
   435  func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {
   436      var w TwitterStruct
   437      m := []byte(TwitterJson)
   438      _ = json.Unmarshal(m, &w)
   439      b.SetBytes(int64(len(TwitterJson)))
   440      b.ResetTimer()
   441      b.RunParallel(func(pb *testing.PB) {
   442          for pb.Next() {
   443              var v TwitterStruct
   444              _ = json.Unmarshal(m, &v)
   445          }
   446      })
   447  }
   448
View as plain text