1
16
17 package encoder
18
19 import (
20 `bytes`
21 `encoding`
22 `encoding/json`
23 `runtime`
24 `runtime/debug`
25 `strconv`
26 `sync`
27 `testing`
28 `time`
29
30 `github.com/bytedance/sonic/internal/rt`
31 `github.com/stretchr/testify/require`
32 )
33
34 func TestMain(m *testing.M) {
35 go func () {
36 if !debugAsyncGC {
37 return
38 }
39 println("Begin GC looping...")
40 for {
41 runtime.GC()
42 debug.FreeOSMemory()
43 }
44 println("stop GC looping!")
45 }()
46 time.Sleep(time.Millisecond)
47 m.Run()
48 }
49
50 func TestGC(t *testing.T) {
51 if debugSyncGC {
52 return
53 }
54 out, err := Encode(_GenericValue, 0)
55 if err != nil {
56 t.Fatal(err)
57 }
58 n := len(out)
59 wg := &sync.WaitGroup{}
60 N := 10000
61 for i:=0; i<N; i++ {
62 wg.Add(1)
63 go func (wg *sync.WaitGroup, size int) {
64 defer wg.Done()
65 out, err := Encode(_GenericValue, 0)
66 if err != nil {
67 t.Fatal(err)
68 }
69 if len(out) != size {
70 t.Fatal(len(out), size)
71 }
72 runtime.GC()
73 }(wg, n)
74 }
75 wg.Wait()
76 }
77
78 type sample struct {
79 M map[string]interface{}
80 S []interface{}
81 A [0]interface{}
82 MP *map[string]interface{}
83 SP *[]interface{}
84 AP *[0]interface{}
85 }
86
87 func TestOptionSliceOrMapNoNull(t *testing.T) {
88 obj := sample{}
89 out, err := Encode(obj, NoNullSliceOrMap)
90 if err != nil {
91 t.Fatal(err)
92 }
93 require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
94
95 obj2 := sample{}
96 out, err = Encode(obj2, 0)
97 if err != nil {
98 t.Fatal(err)
99 }
100 require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
101 }
102
103 func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
104 b.Run("true", func (b *testing.B) {
105 obj := sample{}
106 _, err := Encode(obj, NoNullSliceOrMap)
107 if err != nil {
108 b.Fatal(err)
109 }
110 b.ResetTimer()
111 for i:=0;i<b.N;i++{
112 _, _ = Encode(obj, NoNullSliceOrMap)
113 }
114 })
115
116 b.Run("false", func (b *testing.B) {
117 obj2 := sample{}
118 _, err := Encode(obj2, 0)
119 if err != nil {
120 b.Fatal(err)
121 }
122 for i:=0;i<b.N;i++{
123 _, _ = Encode(obj2, 0)
124 }
125 })
126 }
127
128 func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
129 require.Equal(t, exp, fn(arg))
130 }
131
132 func TestEncoder_String(t *testing.T) {
133 runEncoderTest(t, Quote, `""` , "")
134 runEncoderTest(t, Quote, `"hello, world"` , "hello, world")
135 runEncoderTest(t, Quote, `"hello啊啊啊aa"` , "hello啊啊啊aa")
136 runEncoderTest(t, Quote, `"hello\\\"world"` , "hello\\\"world")
137 runEncoderTest(t, Quote, `"hello\n\tworld"` , "hello\n\tworld")
138 runEncoderTest(t, Quote, `"hello\u0000\u0001world"` , "hello\x00\x01world")
139 runEncoderTest(t, Quote, `"hello\u0000\u0001world"` , "hello\x00\x01world")
140 runEncoderTest(t, Quote, `"Cartoonist, Illustrator, and T-Shirt connoisseur"` , "Cartoonist, Illustrator, and T-Shirt connoisseur")
141 }
142
143 type StringStruct struct {
144 X *int `json:"x,string,omitempty"`
145 Y []int `json:"y"`
146 Z json.Number `json:"z,string"`
147 W string `json:"w,string"`
148 }
149
150 func TestEncoder_FieldStringize(t *testing.T) {
151 x := 12345
152 v := StringStruct{X: &x, Y: []int{1, 2, 3}, Z: "4567456", W: "asdf"}
153 r, e := Encode(v, 0)
154 require.NoError(t, e)
155 println(string(r))
156 }
157
158 func TestEncodeErrorAndScratchBuf(t *testing.T) {
159 var obj = map[string]interface{}{
160 "a": json.RawMessage(" [} "),
161 }
162 buf := make([]byte, 0, 10)
163 _ = EncodeInto(&buf, obj, 0)
164 if len(buf) < 0 || len(buf) > 10 {
165 println(buf)
166 t.Fatal()
167 }
168 }
169
170 type MarshalerImpl struct {
171 X int
172 }
173
174 func (self *MarshalerImpl) MarshalJSON() ([]byte, error) {
175 ret := []byte(strconv.Itoa(self.X))
176 return append(ret, " "...), nil
177 }
178
179 type MarshalerStruct struct {
180 V MarshalerImpl
181 }
182
183 func TestEncoder_Marshaler(t *testing.T) {
184 v := MarshalerStruct{V: MarshalerImpl{X: 12345}}
185 ret, err := Encode(&v, 0)
186 require.NoError(t, err)
187 require.Equal(t, `{"V":12345 }`, string(ret))
188 ret, err = Encode(v, 0)
189 require.NoError(t, err)
190 require.Equal(t, `{"V":{"X":12345}}`, string(ret))
191
192 ret2, err2 := Encode(&v, 0)
193 require.NoError(t, err2)
194 require.Equal(t, `{"V":12345 }`, string(ret2))
195 ret3, err3 := Encode(v, CompactMarshaler)
196 require.NoError(t, err3)
197 require.Equal(t, `{"V":{"X":12345}}`, string(ret3))
198 }
199
200 type MarshalerErrorStruct struct {
201 V MarshalerImpl
202 }
203
204 func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) {
205 return []byte(`[""] {`), nil
206 }
207
208 func TestMarshalerError(t *testing.T) {
209 v := MarshalerErrorStruct{}
210 ret, err := Encode(&v, 0)
211 require.EqualError(t, err, `invalid Marshaler output json syntax at 5: "[\"\"] {"`)
212 require.Equal(t, []byte(nil), ret)
213 }
214
215 type RawMessageStruct struct {
216 X json.RawMessage
217 }
218
219 func TestEncoder_RawMessage(t *testing.T) {
220 rms := RawMessageStruct{
221 X: json.RawMessage("123456 "),
222 }
223 ret, err := Encode(&rms, 0)
224 require.NoError(t, err)
225 require.Equal(t, `{"X":123456 }`, string(ret))
226
227 ret, err = Encode(&rms, CompactMarshaler)
228 require.NoError(t, err)
229 require.Equal(t, `{"X":123456}`, string(ret))
230 }
231
232 type TextMarshalerImpl struct {
233 X string
234 }
235
236 func (self *TextMarshalerImpl) MarshalText() ([]byte, error) {
237 return []byte(self.X), nil
238 }
239
240 type TextMarshalerImplV struct {
241 X string
242 }
243
244 func (self TextMarshalerImplV) MarshalText() ([]byte, error) {
245 return []byte(self.X), nil
246 }
247
248 type TextMarshalerStruct struct {
249 V TextMarshalerImpl
250 }
251
252 func TestEncoder_TextMarshaler(t *testing.T) {
253 v := TextMarshalerStruct{V: TextMarshalerImpl{X: (`{"a"}`)}}
254 ret, err := Encode(&v, 0)
255 require.NoError(t, err)
256 require.Equal(t, `{"V":"{\"a\"}"}`, string(ret))
257 ret, err = Encode(v, 0)
258 require.NoError(t, err)
259 require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret))
260
261 ret2, err2 := Encode(&v, NoQuoteTextMarshaler)
262 require.NoError(t, err2)
263 require.Equal(t, `{"V":{"a"}}`, string(ret2))
264 ret3, err3 := Encode(v, NoQuoteTextMarshaler)
265 require.NoError(t, err3)
266 require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret3))
267 }
268
269 func TestTextMarshalTextKey_SortKeys(t *testing.T) {
270 v := map[*TextMarshalerImpl]string{
271 {"b"}: "b",
272 {"c"}: "c",
273 {"a"}: "a",
274 }
275 ret, err := Encode(v, SortMapKeys)
276 require.NoError(t, err)
277 require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
278
279 v2 := map[TextMarshalerImplV]string{
280 {"b"}: "b",
281 {"c"}: "c",
282 {"a"}: "a",
283 }
284 ret, err = Encode(v2, SortMapKeys)
285 require.NoError(t, err)
286 require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
287
288 v3 := map[encoding.TextMarshaler]string{
289 TextMarshalerImplV{"b"}: "b",
290 &TextMarshalerImpl{"c"}: "c",
291 TextMarshalerImplV{"a"}: "a",
292 }
293 ret, err = Encode(v3, SortMapKeys)
294 require.NoError(t, err)
295 require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
296 }
297
298 func TestEncoder_Marshal_EscapeHTML(t *testing.T) {
299 v := map[string]TextMarshalerImpl{"&&":{"<>"}}
300 ret, err := Encode(v, EscapeHTML)
301 require.NoError(t, err)
302 require.Equal(t, `{"\u0026\u0026":{"X":"\u003c\u003e"}}`, string(ret))
303 ret, err = Encode(v, 0)
304 require.NoError(t, err)
305 require.Equal(t, `{"&&":{"X":"<>"}}`, string(ret))
306
307
308
309 m := map[string]string{"test": "“123”"}
310 ret, err = Encode(m, EscapeHTML)
311 require.Equal(t, string(ret), `{"test":"“123”"}`)
312 require.NoError(t, err)
313
314 m = map[string]string{"K": "\u2028\u2028\xe2"}
315 ret, err = Encode(m, EscapeHTML)
316 require.Equal(t, string(ret), "{\"K\":\"\\u2028\\u2028\xe2\"}")
317 require.NoError(t, err)
318 }
319
320 func TestEncoder_EscapeHTML(t *testing.T) {
321
322 test := []string{
323 "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&",
324 "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}",
325 }
326 for _, s := range(test) {
327 data := []byte(s)
328 sdst := HTMLEscape(nil, data)
329 var dst bytes.Buffer
330 json.HTMLEscape(&dst, data)
331 require.Equal(t, string(sdst), dst.String())
332 }
333 }
334
335 func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) {
336 buf1, err1 := Encode(&_BindingValue, SortMapKeys | EscapeHTML)
337 require.NoError(t, err1)
338 buf2, err2 :=json.Marshal(&_BindingValue)
339 require.NoError(t, err2)
340 require.Equal(t, buf1, buf2)
341 }
342
343 var _GenericValue interface{}
344 var _BindingValue TwitterStruct
345
346 func init() {
347 _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue)
348 _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
349 }
350
351 func TestEncoder_Generic(t *testing.T) {
352 v, e := Encode(_GenericValue, 0)
353 require.NoError(t, e)
354 println(string(v))
355 }
356
357 func TestEncoder_Binding(t *testing.T) {
358 v, e := Encode(_BindingValue, 0)
359 require.NoError(t, e)
360 println(string(v))
361 }
362
363 func TestEncoder_MapSortKey(t *testing.T) {
364 m := map[string]string {
365 "C": "third",
366 "D": "forth",
367 "A": "first",
368 "F": "sixth",
369 "E": "fifth",
370 "B": "second",
371 }
372 v, e := Encode(m, SortMapKeys)
373 require.NoError(t, e)
374 require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v))
375 }
376
377 func BenchmarkEncoder_Generic_Sonic(b *testing.B) {
378 _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
379 b.SetBytes(int64(len(TwitterJson)))
380 b.ResetTimer()
381 for i := 0; i < b.N; i++ {
382 _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
383 }
384 }
385
386 func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) {
387 _, _ = Encode(_GenericValue, 0)
388 b.SetBytes(int64(len(TwitterJson)))
389 b.ResetTimer()
390 for i := 0; i < b.N; i++ {
391 _, _ = Encode(_GenericValue, 0)
392 }
393 }
394
395 func BenchmarkEncoder_Generic_StdLib(b *testing.B) {
396 _, _ = json.Marshal(_GenericValue)
397 b.SetBytes(int64(len(TwitterJson)))
398 b.ResetTimer()
399 for i := 0; i < b.N; i++ {
400 _, _ = json.Marshal(_GenericValue)
401 }
402 }
403
404 func BenchmarkEncoder_Binding_Sonic(b *testing.B) {
405 _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
406 b.SetBytes(int64(len(TwitterJson)))
407 b.ResetTimer()
408 for i := 0; i < b.N; i++ {
409 _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
410 }
411 }
412
413 func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) {
414 _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
415 b.SetBytes(int64(len(TwitterJson)))
416 b.ResetTimer()
417 for i := 0; i < b.N; i++ {
418 _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
419 }
420 }
421
422 func BenchmarkEncoder_Binding_StdLib(b *testing.B) {
423 _, _ = json.Marshal(&_BindingValue)
424 b.SetBytes(int64(len(TwitterJson)))
425 b.ResetTimer()
426 for i := 0; i < b.N; i++ {
427 _, _ = json.Marshal(&_BindingValue)
428 }
429 }
430
431 func BenchmarkEncoder_Parallel_Generic_Sonic(b *testing.B) {
432 _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
433 b.SetBytes(int64(len(TwitterJson)))
434 b.ResetTimer()
435 b.RunParallel(func(pb *testing.PB) {
436 for pb.Next() {
437 _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
438 }
439 })
440 }
441
442 func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
443 _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
444 b.SetBytes(int64(len(TwitterJson)))
445 b.ResetTimer()
446 b.RunParallel(func(pb *testing.PB) {
447 for pb.Next() {
448 _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
449 }
450 })
451 }
452
453 func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) {
454 _, _ = json.Marshal(_GenericValue)
455 b.SetBytes(int64(len(TwitterJson)))
456 b.ResetTimer()
457 b.RunParallel(func(pb *testing.PB) {
458 for pb.Next() {
459 _, _ = json.Marshal(_GenericValue)
460 }
461 })
462 }
463
464 func BenchmarkEncoder_Parallel_Binding_Sonic(b *testing.B) {
465 _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
466 b.SetBytes(int64(len(TwitterJson)))
467 b.ResetTimer()
468 b.RunParallel(func(pb *testing.PB) {
469 for pb.Next() {
470 _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
471 }
472 })
473 }
474
475 func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
476 _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
477 b.SetBytes(int64(len(TwitterJson)))
478 b.ResetTimer()
479 b.RunParallel(func(pb *testing.PB) {
480 for pb.Next() {
481 _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
482 }
483 })
484 }
485
486 func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) {
487 _, _ = json.Marshal(&_BindingValue)
488 b.SetBytes(int64(len(TwitterJson)))
489 b.ResetTimer()
490 b.RunParallel(func(pb *testing.PB) {
491 for pb.Next() {
492 _, _ = json.Marshal(&_BindingValue)
493 }
494 })
495 }
496
497 func BenchmarkHTMLEscape_Sonic(b *testing.B) {
498 jsonByte := []byte(TwitterJson)
499 b.SetBytes(int64(len(TwitterJson)))
500 b.ResetTimer()
501 var buf []byte
502 for i := 0; i < b.N; i++ {
503 buf = HTMLEscape(nil, jsonByte)
504 }
505 _ = buf
506 }
507
508 func BenchmarkHTMLEscape_StdLib(b *testing.B) {
509 jsonByte := []byte(TwitterJson)
510 b.SetBytes(int64(len(TwitterJson)))
511 b.ResetTimer()
512 var buf []byte
513 for i := 0; i < b.N; i++ {
514 out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson) * 6 / 5))
515 json.HTMLEscape(out, jsonByte)
516 buf = out.Bytes()
517 }
518 _ = buf
519 }
520
521
522 func BenchmarkValidate_Sonic(b *testing.B) {
523 var data = rt.Str2Mem(TwitterJson)
524 ok, s := Valid(data)
525 if !ok {
526 b.Fatal(s)
527 }
528 b.SetBytes(int64(len(TwitterJson)))
529 b.ResetTimer()
530 for i:=0; i<b.N; i++ {
531 _, _ = Valid(data)
532 }
533 }
534
535 func BenchmarkValidate_Std(b *testing.B) {
536 var data = rt.Str2Mem(TwitterJson)
537 if !json.Valid(data) {
538 b.Fatal()
539 }
540 b.SetBytes(int64(len(TwitterJson)))
541 b.ResetTimer()
542 for i:=0; i<b.N; i++ {
543 _ = json.Valid(data)
544 }
545 }
546
547 func BenchmarkCompact_Std(b *testing.B) {
548 var data = rt.Str2Mem(TwitterJson)
549 var dst = bytes.NewBuffer(nil)
550 if err := json.Compact(dst, data); err != nil {
551 b.Fatal(err)
552 }
553 b.SetBytes(int64(len(TwitterJson)))
554 b.ResetTimer()
555 for i:=0; i<b.N; i++ {
556 dst.Reset()
557 _ = json.Compact(dst, data)
558 }
559 }
560
561 type f64Bench struct {
562 name string
563 float float64
564 }
565 func BenchmarkEncode_Float64(b *testing.B) {
566 var bench = []f64Bench{
567 {"Zero", 0},
568 {"ShortDecimal", 1000},
569 {"Decimal", 33909},
570 {"Float", 339.7784},
571 {"Exp", -5.09e75},
572 {"NegExp", -5.11e-95},
573 {"LongExp", 1.234567890123456e-78},
574 {"Big", 123456789123456789123456789},
575
576 }
577 maxUint := "18446744073709551615"
578 for i := 1; i <= len(maxUint); i++ {
579 name := strconv.FormatInt(int64(i), 10) + "-Digs"
580 num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
581 bench = append(bench, f64Bench{name, float64(num)})
582 }
583 for _, c := range bench {
584 libs := []struct {
585 name string
586 test func(*testing.B)
587 }{{
588 name: "StdLib",
589 test: func(b *testing.B) { _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
590 }, {
591 name: "Sonic",
592 test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
593 }}
594 for _, lib := range libs {
595 name := lib.name + "_" + c.name
596 b.Run(name, lib.test)
597 }
598 }
599 }
600
601 type f32Bench struct {
602 name string
603 float float32
604 }
605 func BenchmarkEncode_Float32(b *testing.B) {
606 var bench = []f32Bench{
607 {"Zero", 0},
608 {"ShortDecimal", 1000},
609 {"Decimal", 33909},
610 {"ExactFraction", 3.375},
611 {"Point", 339.7784},
612 {"Exp", -5.09e25},
613 {"NegExp", -5.11e-25},
614 {"Shortest", 1.234567e-8},
615 }
616
617 maxUint := "18446744073709551615"
618 for i := 1; i <= len(maxUint); i++ {
619 name := strconv.FormatInt(int64(i), 10) + "-Digs"
620 num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
621 bench = append(bench, f32Bench{name, float32(num)})
622 }
623 for _, c := range bench {
624 libs := []struct {
625 name string
626 test func(*testing.B)
627 }{{
628 name: "StdLib",
629 test: func(b *testing.B) { _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
630 }, {
631 name: "Sonic",
632 test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
633 }}
634 for _, lib := range libs {
635 name := lib.name + "_" + c.name
636 b.Run(name, lib.test)
637 }
638 }
639 }
View as plain text