/* * Copyright 2021 ByteDance Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package encoder import ( `bytes` `encoding` `encoding/json` `runtime` `runtime/debug` `strconv` `sync` `testing` `time` `github.com/bytedance/sonic/internal/rt` `github.com/stretchr/testify/require` ) func TestMain(m *testing.M) { go func () { if !debugAsyncGC { return } println("Begin GC looping...") for { runtime.GC() debug.FreeOSMemory() } println("stop GC looping!") }() time.Sleep(time.Millisecond) m.Run() } func TestGC(t *testing.T) { if debugSyncGC { return } out, err := Encode(_GenericValue, 0) if err != nil { t.Fatal(err) } n := len(out) wg := &sync.WaitGroup{} N := 10000 for i:=0; i 10 { t.Fatal() } } type MarshalerImpl struct { X int } func (self *MarshalerImpl) MarshalJSON() ([]byte, error) { ret := []byte(strconv.Itoa(self.X)) return append(ret, " "...), nil } type MarshalerStruct struct { V MarshalerImpl } type MarshalerErrorStruct struct { V MarshalerImpl } func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) { return []byte(`[""] {`), nil } type RawMessageStruct struct { X json.RawMessage } type TextMarshalerImpl struct { X string } func (self *TextMarshalerImpl) MarshalText() ([]byte, error) { return []byte(self.X), nil } type TextMarshalerImplV struct { X string } func (self TextMarshalerImplV) MarshalText() ([]byte, error) { return []byte(self.X), nil } type TextMarshalerStruct struct { V TextMarshalerImpl } func TestTextMarshalTextKey_SortKeys(t *testing.T) { v := map[*TextMarshalerImpl]string{ {"b"}: "b", {"c"}: "c", {"a"}: "a", } ret, err := Encode(v, SortMapKeys) require.NoError(t, err) require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) v2 := map[TextMarshalerImplV]string{ {"b"}: "b", {"c"}: "c", {"a"}: "a", } ret, err = Encode(v2, SortMapKeys) require.NoError(t, err) require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) v3 := map[encoding.TextMarshaler]string{ TextMarshalerImplV{"b"}: "b", &TextMarshalerImpl{"c"}: "c", TextMarshalerImplV{"a"}: "a", } ret, err = Encode(v3, SortMapKeys) require.NoError(t, err) require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) } func TestEncoder_EscapeHTML(t *testing.T) { // test data from libfuzzer test := []string{ "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&", "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}", } for _, s := range(test) { data := []byte(s) sdst := HTMLEscape(nil, data) var dst bytes.Buffer json.HTMLEscape(&dst, data) require.Equal(t, string(sdst), dst.String()) } } func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) { buf1, err1 := Encode(&_BindingValue, SortMapKeys | EscapeHTML) require.NoError(t, err1) buf2, err2 :=json.Marshal(&_BindingValue) require.NoError(t, err2) require.Equal(t, buf1, buf2) } var _GenericValue interface{} var _BindingValue TwitterStruct func init() { _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue) _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) } func TestEncoder_Generic(t *testing.T) { v, e := Encode(_GenericValue, 0) require.NoError(t, e) println(string(v)) } func TestEncoder_Binding(t *testing.T) { v, e := Encode(_BindingValue, 0) require.NoError(t, e) println(string(v)) } func TestEncoder_MapSortKey(t *testing.T) { m := map[string]string { "C": "third", "D": "forth", "A": "first", "F": "sixth", "E": "fifth", "B": "second", } v, e := Encode(m, SortMapKeys) require.NoError(t, e) require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v)) } func BenchmarkEncoder_Generic_Sonic(b *testing.B) { _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) } } func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) { _, _ = Encode(_GenericValue, 0) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = Encode(_GenericValue, 0) } } func BenchmarkEncoder_Generic_StdLib(b *testing.B) { _, _ = json.Marshal(_GenericValue) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = json.Marshal(_GenericValue) } } func BenchmarkEncoder_Binding_Sonic(b *testing.B) { _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) } } func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) { _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) } } func BenchmarkEncoder_Binding_StdLib(b *testing.B) { _, _ = json.Marshal(&_BindingValue) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i := 0; i < b.N; i++ { _, _ = json.Marshal(&_BindingValue) } } func BenchmarkEncoder_Parallel_Generic_Sonic(b *testing.B) { _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) } }) } func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) { _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) } }) } func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) { _, _ = json.Marshal(_GenericValue) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = json.Marshal(_GenericValue) } }) } func BenchmarkEncoder_Parallel_Binding_Sonic(b *testing.B) { _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) } }) } func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) { _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) } }) } func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) { _, _ = json.Marshal(&_BindingValue) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _ = json.Marshal(&_BindingValue) } }) } func BenchmarkHTMLEscape_Sonic(b *testing.B) { jsonByte := []byte(TwitterJson) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() var buf []byte for i := 0; i < b.N; i++ { buf = HTMLEscape(nil, jsonByte) } _ = buf } func BenchmarkHTMLEscape_StdLib(b *testing.B) { jsonByte := []byte(TwitterJson) b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() var buf []byte for i := 0; i < b.N; i++ { out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson) * 6 / 5)) json.HTMLEscape(out, jsonByte) buf = out.Bytes() } _ = buf } func BenchmarkValidate_Sonic(b *testing.B) { var data = rt.Str2Mem(TwitterJson) ok, s := Valid(data) if !ok { b.Fatal(s) } b.SetBytes(int64(len(TwitterJson))) b.ResetTimer() for i:=0; i