stream_test.go

Documentation: github.com/bytedance/sonic/internal/decoder

     1  /*
     2   * Copyright 2021 ByteDance Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package decoder
    18  
    19  import (
    20      `bytes`
    21      `encoding/json`
    22      `io`
    23      `io/ioutil`
    24      `strings`
    25      `testing`
    26  
    27      `github.com/bytedance/sonic/option`
    28      `github.com/stretchr/testify/assert`
    29      `github.com/stretchr/testify/require`
    30  )
    31  
    32  var (
    33      DefaultBufferSize = option.DefaultDecoderBufferSize
    34      _Single_JSON = `{`+`"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}` + strings.Repeat(" ", int(DefaultBufferSize)) + `{`
    35      _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}` + strings.Repeat(" ", int(DefaultBufferSize)) + `{"11111":"` + strings.Repeat("2", int(DefaultBufferSize)) + `"}`     
    36      _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}{ } {"11111":"` + 
    37      strings.Repeat("2", int(DefaultBufferSize))+`"} b {}`
    38      _SMALL_JSON = `{"a":"b"} [1] {"c":"d"} [2]`
    39  )
    40  
    41  func TestStreamError(t *testing.T) {
    42      var qs = []string{
    43          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"`,
    44          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"}`,
    45          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`""}`,
    46          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":}`,
    47          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":]`,
    48          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x`,
    49          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x}`,
    50          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x]`,
    51          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t`,
    52          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t}`,
    53          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":true]`,
    54          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f`,
    55          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f}`,
    56          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":false]`,
    57          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n`,
    58          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n}`,
    59          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":null]`,
    60          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"`,
    61          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a`,
    62          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a}`,
    63          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"`,
    64          `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"]`,
    65      }
    66  
    67      for i, q := range qs {
    68          var qq = []byte(q[:int(DefaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(DefaultBufferSize):])
    69          var obj interface{}
    70          require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj))
    71      }
    72  }
    73  
    74  func TestDecodeEmpty(t *testing.T) {
    75      var str = ``
    76      var r1 = strings.NewReader(str)
    77      var v1 interface{}
    78      var d1 = json.NewDecoder(r1)
    79      var r2 = strings.NewReader(str)
    80      var v2 interface{}
    81      var d2 = NewStreamDecoder(r2)
    82      es1 := d1.Decode(&v1)
    83      ee1 := d2.Decode(&v2)
    84      assert.Equal(t, es1, ee1)
    85      assert.Equal(t, v1, v2)
    86  }
    87  
    88  func TestDecodeRecurse(t *testing.T) {
    89      var str = _Single_JSON
    90      var r1 = bytes.NewBufferString(str)
    91      var v1 interface{}
    92      var d1 = json.NewDecoder(r1)
    93      var r2 = bytes.NewBufferString(str)
    94      var v2 interface{}
    95      var d2 = NewStreamDecoder(r2)
    96  
    97      require.Equal(t, d1.More(), d2.More())
    98      es1 := d1.Decode(&v1)
    99      ee1 := d2.Decode(&v2)
   100      assert.Equal(t, es1, ee1)
   101      assert.Equal(t, v1, v2)
   102  
   103      require.Equal(t, d1.More(), d2.More())
   104      r1.WriteString(str[1:])
   105      r2.WriteString(str[1:])
   106  
   107      require.Equal(t, d1.More(), d2.More())
   108      es1 = d1.Decode(&v1)
   109      ee1 = d2.Decode(&v2)
   110      assert.Equal(t, es1, ee1)
   111      println(es1)
   112      assert.Equal(t, v1, v2)
   113      require.Equal(t, d1.More(), d2.More())
   114  }
   115  
   116  type HaltReader struct {
   117      halts map[int]bool
   118      buf string
   119      p int
   120  }
   121  
   122  func NewHaltReader(buf string, halts map[int]bool) *HaltReader {
   123      return &HaltReader{
   124          halts: halts,
   125          buf: buf,
   126          p: 0,
   127      }
   128  }
   129  
   130  func (self *HaltReader) Read(p []byte) (int, error) {
   131      t := 0
   132      for ; t < len(p); {
   133          if self.p >= len(self.buf) {
   134              return t, io.EOF
   135          }
   136          if b, ok := self.halts[self.p]; b {
   137              self.halts[self.p] = false
   138              return t, nil
   139          } else if ok {
   140              delete(self.halts, self.p)
   141              return 0, nil
   142          }
   143          p[t] = self.buf[self.p]
   144          self.p++
   145          t++
   146      }
   147      return t, nil
   148  }
   149  
   150  func (self *HaltReader) Reset(buf string) {
   151      self.p = 0
   152      self.buf = buf
   153  }
   154  
   155  var testHalts = func () map[int]bool {
   156      return map[int]bool{
   157          1: true,
   158          10:true,
   159          20: true}
   160  }
   161  
   162  func TestBuffered(t *testing.T) {
   163      var str = _Triple_JSON
   164      var r1 = NewHaltReader(str, testHalts())
   165      var v1 map[string]interface{}
   166      var d1 = json.NewDecoder(r1)
   167    
   168      var r2 = NewHaltReader(str, testHalts())
   169      var v2 map[string]interface{}
   170      var d2 = NewStreamDecoder(r2)
   171  
   172      require.Equal(t, d1.More(), d2.More())
   173      require.Nil(t, d1.Decode(&v1))
   174      require.Nil(t, d2.Decode(&v2))
   175      left1, err1 := ioutil.ReadAll(d1.Buffered())
   176      require.Nil(t, err1)
   177      left2, err2 := ioutil.ReadAll(d2.Buffered())
   178      require.Nil(t, err2)
   179      require.Equal(t, d1.InputOffset(), d2.InputOffset())
   180      min := len(left1)
   181      if min > len(left2) {
   182          min = len(left2)
   183      }
   184      require.Equal(t, left1[:min], left2[:min])
   185  
   186      require.Equal(t, d1.More(), d2.More())
   187      es4 := d1.Decode(&v1)
   188      ee4 := d2.Decode(&v2)
   189      assert.Equal(t, es4, ee4)
   190      println(str[d1.InputOffset()-5:d1.InputOffset()+5])
   191      assert.Equal(t, d1.InputOffset(), d2.InputOffset()-1)
   192  
   193      require.Equal(t, d1.More(), d2.More())
   194      es2 := d1.Decode(&v1)
   195      ee2 := d2.Decode(&v2)
   196      assert.Equal(t, es2, ee2)
   197      println(str[d1.InputOffset()-5:d1.InputOffset()+5])
   198      assert.Equal(t, d1.InputOffset(), d2.InputOffset()-1)
   199  }
   200  
   201  func BenchmarkDecodeStream_Std(b *testing.B) {
   202      b.Run("single", func (b *testing.B) {
   203          var str = _Single_JSON
   204          var r1 = bytes.NewBufferString(str)
   205          dc := json.NewDecoder(r1)
   206          for i:=0; i<b.N; i++ {
   207              var v1 map[string]interface{}
   208              e := dc.Decode(&v1)
   209              if e != nil {
   210                  b.Fatal(e)
   211              }
   212              r1.WriteString(str[1:])
   213          }
   214      })
   215  
   216      b.Run("double", func (b *testing.B) {
   217          var str = _Double_JSON
   218          for i:=0; i<b.N; i++ {
   219              var r1 = strings.NewReader(str)
   220              var v1 map[string]interface{}
   221              dc := json.NewDecoder(r1)
   222              _ = dc.Decode(&v1)
   223              if dc.More() {
   224                  _ = dc.Decode(&v1)
   225              }
   226          }
   227      })
   228  
   229      b.Run("4x", func (b *testing.B) {
   230          var str = _Double_JSON + strings.Repeat(" ", int(DefaultBufferSize-10)) + _Double_JSON
   231          b.ResetTimer()
   232          for i:=0; i<b.N; i++ {
   233              var r1 = strings.NewReader(str)
   234              var v1 map[string]interface{}
   235              dc := json.NewDecoder(r1)
   236              for dc.More() {
   237                  e := dc.Decode(&v1)
   238                  if e != nil {
   239                      b.Fatal(e)
   240                  }
   241              }
   242          }
   243      })
   244      
   245      b.Run("halt", func (b *testing.B) {
   246          var str = _Double_JSON
   247          for i:=0; i<b.N; i++ {
   248              var r1 = NewHaltReader(str, testHalts())
   249              var v1 map[string]interface{}
   250              dc := json.NewDecoder(r1)
   251              _ = dc.Decode(&v1)
   252          }
   253      })
   254  
   255      b.Run("small", func (b *testing.B) {
   256          var str = _SMALL_JSON
   257          for i:=0; i<b.N; i++ {
   258              var r1 = strings.NewReader(str)
   259              var v1 interface{}
   260              dc := json.NewDecoder(r1)
   261              for dc.More() {
   262                  e := dc.Decode(&v1)
   263                  if e != nil {
   264                      b.Fatal(e)
   265                  }
   266              }
   267          }
   268      })
   269  }
   270  
   271  // func BenchmarkDecodeError_Sonic(b *testing.B) {
   272  //     var str = `\b测试1234`
   273  //     for i:=0; i<b.N; i++ {
   274  //         var v1 map[string]interface{}
   275  //         _ = NewDecoder(str).Decode(&v1)
   276  //     }
   277  // }
   278  
   279  func BenchmarkDecodeStream_Sonic(b *testing.B) {
   280      b.Run("single", func (b *testing.B) {
   281          var str = _Single_JSON
   282          var r1 = bytes.NewBufferString(str)
   283          dc := NewStreamDecoder(r1)
   284          for i:=0; i<b.N; i++ {
   285              var v1 map[string]interface{}
   286              e := dc.Decode(&v1)
   287              if e != nil {
   288                  b.Fatal(e)
   289              }
   290              r1.WriteString(str[1:])
   291          }
   292      })
   293  
   294      b.Run("double", func (b *testing.B) {
   295          var str = _Double_JSON
   296          for i:=0; i<b.N; i++ {
   297              var r1 = strings.NewReader(str)
   298              var v1 map[string]interface{}
   299              dc := NewStreamDecoder(r1)
   300              _ = dc.Decode(&v1)
   301              if dc.More() {
   302                  _ = dc.Decode(&v1)
   303              }
   304          }
   305      })
   306  
   307      b.Run("4x", func (b *testing.B) {
   308          var str = _Double_JSON + strings.Repeat(" ", int(DefaultBufferSize-10)) + _Double_JSON
   309          b.ResetTimer()
   310          for i:=0; i<b.N; i++ {
   311              // println("loop")
   312              var r1 = strings.NewReader(str)
   313              var v1 map[string]interface{}
   314              dc := NewStreamDecoder(r1)
   315              for dc.More() {
   316                  e := dc.Decode(&v1)
   317                  if e != nil {
   318                      b.Fatal(e)
   319                  }
   320              }
   321          }
   322      })
   323  
   324      b.Run("halt", func (b *testing.B) {
   325          var str = _Double_JSON
   326          for i:=0; i<b.N; i++ {
   327              var r1 = NewHaltReader(str, testHalts())
   328              var v1 map[string]interface{}
   329              dc := NewStreamDecoder(r1)
   330              _ = dc.Decode(&v1)
   331          }
   332      })
   333  
   334      b.Run("small", func (b *testing.B) {
   335          var str = _SMALL_JSON
   336          for i:=0; i<b.N; i++ {
   337              // println("one loop")
   338              var r1 = strings.NewReader(str)
   339              var v1 interface{}
   340              dc := NewStreamDecoder(r1)
   341              for dc.More() {
   342                  e := dc.Decode(&v1)
   343                  if e != nil {
   344                      b.Fatal(e)
   345                  }
   346              }
   347          }
   348      })
   349  }
View as plain text