// +build amd64,go1.16,!go1.23 /* * Copyright 2021 ByteDance Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package sonic import ( `bytes` `encoding/hex` `encoding/json` `fmt` `math/rand` `reflect` `strings` `testing` `time` `github.com/davecgh/go-spew/spew` `github.com/stretchr/testify/assert` `github.com/bytedance/sonic/ast` ) func Parse(src string) (*ast.Node, error) { node, err := ast.NewParser(src).Parse() if err != 0 { return &node, fmt.Errorf("parsing error: %v", err) } return &node, nil } func assertCond(cond bool) { if !cond { panic("assertCond failed") } } func TestExampleSearch(t *testing.T) { data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `) node, e := Get(data, "test", 0) x, _ := node.Bool() if e != nil || x != true { t.Fatalf("node: %v, err: %v", node, e) } node, e = Get(data, "test", 1) a, _ := node.Float64() if e != nil || a != 0.1 { t.Fatalf("node: %v, err: %v", node, e) } node, e = Get(data, "test", 2) b, _ := node.String() if e != nil || b != "abc" { t.Fatalf("node: %v, err: %v", node, e) } node, e = Get(data, "test", 3) arr, _ := node.Array() if e != nil || arr[0] != "h" { t.Fatalf("node: %v, err: %v", node, e) } node, e = Get(data, "test", 4, "a") c, _ := node.String() if e != nil || c != "bc" { t.Fatalf("node: %v, err: %v", node, e) } } func TestExampleSearchEscapedKey(t *testing.T) { data := []byte(` { "xx" : [] , "yy" :{ }, "test\"" : [ true , 0.1 , "abc", [ "h" ], { "a\u0008": {}, "b\\\\": null, "\u2028\u2028": "\u2028\u2029", "\u0026":2, "0":1 } ], ",,,,,,,,,,(,,15": ",", ",,,,,,,,,,(,,,16": "a,]}", ",,,,,,,,,,(,,,,17": 1, ",,,,,,,,,,(,,,,,,,,,,,,,,,,(,,,,34": "c" } `) type getTest struct{ path []interface{} expect interface{} } tests := []getTest { {[]interface{}{"test\"", 0}, true}, {[]interface{}{"test\"", 1}, 0.1}, {[]interface{}{"test\"", 2}, "abc"}, {[]interface{}{"test\"", 3}, []interface{}{"h"}}, {[]interface{}{"test\"", 4, "a\u0008"}, map[string]interface{}{}}, {[]interface{}{"test\"", 4, "b\\\\"}, nil}, {[]interface{}{"test\"", 4, "\u2028\u2028"}, "\u2028\u2029"}, {[]interface{}{"test\"", 4, "0"}, float64(1)}, {[]interface{}{",,,,,,,,,,(,,15"}, ","}, {[]interface{}{",,,,,,,,,,(,,,16"}, "a,]}"}, {[]interface{}{",,,,,,,,,,(,,,,17"}, float64(1)}, {[]interface{}{",,,,,,,,,,(,,,,,,,,,,,,,,,,(,,,,34"}, "c"}, } for _, test := range(tests) { node, err := Get(data, test.path...) assert.NoErrorf(t, err, "get return errors") got, err := node.Interface() assert.NoErrorf(t, err, "get convert errors") assert.Equalf(t, test.expect, got, "get result is wrong from path %#v", test.path) } } func TestExampleSearchErr(t *testing.T) { data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `) node, e := Get(data, "zz") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "xx", 4) if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "yy", "a") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "test", 4, "x") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) } func TestExampleSearchEscapedKeyError(t *testing.T) { data := []byte(` { "xx" : [] ,"yy" :{ }, "x\u0008" : [] ,"y\\\"y" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `) node, e := Get(data, "zz") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "x\u0008", 4) if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "yy", "a") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) node, e = Get(data, "test", 4, "x") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } node, e = Get(data, "y\\\"y", 4, "x") if e == nil { t.Fatalf("node: %v, err: %v", node, e) } fmt.Println(e) } func TestRandomData(t *testing.T) { var lstr string defer func() { if v := recover(); v != nil { println("'" + hex.Dump([]byte(lstr)) + "'") println(lstr) panic(v) } }() data := []byte(`"�-mp�`) _, err := ast.NewParser(string(data)).Parse() if err != 0 { fmt.Println(hex.Dump(data)) fmt.Println(string(data)) } rand.Seed(time.Now().UnixNano()) b := make([]byte, 200) for i := 0; i < 1000000; i++ { n, ee := rand.Read(b[:rand.Int()%len(b)]) if ee != nil { t.Fatalf("get random bytes failed: %v,", ee) return } lstr = string(b[:n]) _, _ = ast.NewParser(lstr).Parse() } } func TestRandomValidStrings(t *testing.T) { rand.Seed(time.Now().UnixNano()) b := make([]byte, 200) for i := 0; i < 1000; i++ { n, err := rand.Read(b[:rand.Int()%len(b)]) if err != nil { t.Fatal("get random data failed:", err) } sm, err := json.Marshal(string(b[:n])) if err != nil { t.Fatal("marshal data failed:",err) } var su string if err := json.Unmarshal(sm, &su); err != nil { t.Fatal("unmarshal data failed:",err) } token, err := GetFromString(`{"str":`+string(sm)+`}`, "str") if err != nil { spew.Dump(string(sm)) t.Fatal("search data failed:",err) } x, _ := token.Interface() st, ok := x.(string) if !ok { t.Fatalf("type mismatch, exp: %v, got: %v", su, x) } if st != su { t.Fatalf("string mismatch, exp: %v, got: %v", su, x) } } } func TestEmoji(t *testing.T) { var input = []byte(`{"utf8":"Example emoji, KO: \ud83d\udd13, \ud83c\udfc3 ` + `OK: \u2764\ufe0f "}`) value, err := Get(input, "utf8") if err != nil { t.Fatal(err) } var v map[string]interface{} if err := json.Unmarshal(input, &v); err != nil { t.Fatal(err) } s, _ := v["utf8"].(string) x, _ := value.String() if x != s { t.Fatalf("expected '%v', got '%v'", s, x) } } func testEscapePath(t *testing.T, json, expect string, path ...interface{}) { n, e := Get([]byte(json), path...) if e != nil { t.Fatal(e) } x, _ := n.String() if x != expect { x, _ := n.Interface() t.Fatalf("expected '%v', got '%v'", expect, x) } } func TestEscapePath(t *testing.T) { data := `{ "test":{ "*":"valZ", "*v":"val0", "keyv*":"val1", "key*v":"val2", "keyv?":"val3", "key?v":"val4", "keyv.":"val5", "key.v":"val6", "keyk*":{"key?":"val7"} } }` testEscapePath(t, data, "valZ", "test", "*") testEscapePath(t, data, "val0", "test", "*v") testEscapePath(t, data, "val1", "test", "keyv*") testEscapePath(t, data, "val2", "test", "key*v") testEscapePath(t, data, "val3", "test", "keyv?") testEscapePath(t, data, "val4", "test", "key?v") testEscapePath(t, data, "val5", "test", "keyv.") testEscapePath(t, data, "val6", "test", "key.v") testEscapePath(t, data, "val7", "test", "keyk*", "key?") } func TestParseAny(t *testing.T) { n, e := Parse("100") assertCond(e == nil) if n == nil { panic("n is nil") } x, _ := n.Float64() assertCond(x == 100) n, e = Parse("true") assertCond(e == nil) if n == nil { panic("n is nil") } a, _ := n.Bool() assertCond(a) n, e = Parse("false") assertCond(e == nil) if n == nil { panic("n is nil") } b, _ := n.Bool() assertCond(b == false) n, e = Parse("yikes") assertCond(e != nil) } func TestTime(t *testing.T) { data := []byte(`{ "code": 0, "msg": "", "data": { "sz002024": { "qfqday": [ [ "2014-01-02", "8.93", "9.03", "9.17", "8.88", "621143.00" ], [ "2014-01-03", "9.03", "9.30", "9.47", "8.98", "1624438.00" ] ] } } }`) var num []string n, e := Get(data, "data", "sz002024", "qfqday", 0) if e != nil { t.Fatal(e) } arr, _ := n.Array() for _, v := range arr { s := v.(string) num = append(num, s) } if fmt.Sprintf("%v", num) != "[2014-01-02 8.93 9.03 9.17 8.88 621143.00]" { t.Fatalf("invalid result") } } var exampleJSON = `{ "widget": { "debug": "on", "window": { "title": "Sample Konfabulator Widget", "name": "main_window", "width": 500, "height": 500 }, "image": { "src": "Images/Sun.png", "hOffset": 250, "vOffset": 250, "alignment": "center" }, "text": { "data": "Click Here", "size": 36, "style": "bold", "vOffset": 100, "alignment": "center", "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" } } }` func TestUnmarshalMap(t *testing.T) { n, err := Parse(exampleJSON) if err != nil || n == nil { t.Fatal(err) } m1, _ := n.Map() var m2 map[string]interface{} if err := json.Unmarshal([]byte(exampleJSON), &m2); err != nil { t.Fatal(err) } b1, err := json.Marshal(m1) if err != nil { t.Fatal(err) } b2, err := json.Marshal(m2) if err != nil { t.Fatal(err) } if !bytes.Equal(b1, b2) { t.Fatalf("b1 != b2\n b1: %v\nb2: %v", string(b1), string(b2)) } } func GetMany(src2 string, path ...string) (ret []string) { src := []byte(src2) for _, p := range path { pathes := strings.Split(p, ".") if len(pathes) == 0 { panic(fmt.Sprintf("invalid path: %v", p)) } ps := make([]interface{}, 0, len(pathes)) for _, p := range pathes { ps = append(ps, p) } n, e := Get(src, ps...) if e != nil { ret = append(ret, "") continue } x, _ := n.Interface() ret = append(ret, fmt.Sprintf("%v", x)) } return } func get(str string, path string) *ast.Node { src := []byte(str) pathes := strings.Split(path, ".") if len(pathes) == 0 { panic(fmt.Sprintf("invalid path: %v", path)) } ps := make([]interface{}, 0, len(pathes)) for _, p := range pathes { ps = append(ps, p) } n, e := Get(src, ps...) if e != nil { return nil } return &n } func TestSingleArrayValue(t *testing.T) { var data = []byte(`{"key": "value","key2":[1,2,3,4,"A"]}`) array, _ := get(string(data), "key2").Array() if len(array) != 5 { t.Fatalf("got '%v', expected '%v'", len(array), 5) } _, e := Get(data, "key3") if e == nil { t.Fatalf("got '%v', expected '%v'", e, nil) } } var manyJSON = ` { "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"hello":"world" }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}, "position":{"type":"Point","coordinates":[-115.24,33.09]}, "loves":["world peace"], "name":{"last":"Anderson","first":"Nancy"}, "age":31, "x":{"a":"emptya","b":"emptyb"}, "name_last":"Yellow", "name_first":"Cat", }` func TestManyBasic(t *testing.T) { testMany := func(shouldFallback bool, expect string, paths ...string) { println() rs := GetMany( manyJSON, paths..., ) // if len(rs) != len(paths) { // t.Fatalf("expected %v, got %v", len(paths), len(rs)) // } var results = "[" + strings.Join(rs, " ") + "]" if results != expect { fmt.Printf("%v\n", paths) t.Fatalf("expected %v, got %v", expect, results) } } testMany(false, "[Point]", "position.type") testMany(false, `[emptya [world peace] 31]`, "x.a", "loves", "age") testMany(false, `[[world peace]]`, "loves") testMany(false, `[map[first:Nancy last:Anderson] Nancy]`, "name", "name.first") testMany(true, `[]`, strings.Repeat("a.", 40)+"hello") res := get(manyJSON, strings.Repeat("a.", 48)+"a") assertCond(res != nil) x, _ := res.Interface() testMany(true, "["+fmt.Sprint(x)+"]", strings.Repeat("a.", 48)+"a") // these should fallback testMany(true, `[Cat Nancy]`, "name_first", "name.first") testMany(true, `[world]`, strings.Repeat("a.", 70)+"hello") } func testMany(t *testing.T, json string, paths, expected []string) { testManyAny(t, json, paths, expected) testManyAny(t, json, paths, expected) } func testManyAny(t *testing.T, json string, paths, expected []string) { var result []string for i := 0; i < 2; i++ { var which string if i == 0 { which = "Get" result = nil for j := 0; j < len(expected); j++ { x, _ := get(json, paths[j]).Interface() result = append(result, fmt.Sprintf("%v", x)) } } else if i == 1 { which = "GetMany" result = GetMany(json, paths...) } if result == nil { panic("result is nil") } for j := 0; j < len(expected); j++ { if result[j] != expected[j] { t.Fatalf("Using key '%s' for '%s'\nexpected '%v', got '%v'", paths[j], which, expected[j], result[j]) } } } } func TestNested(t *testing.T) { data := `{ "name": "FirstName", "name1": "FirstName1", ` + `"address": "address1", "addressDetails": "address2", }` paths := []string{"name", "name1", "address", "addressDetails"} expected := []string{"FirstName", "FirstName1", "address1", "address2"} t.Run("SingleMany", func(t *testing.T) { testMany(t, data, paths, expected) }) } func TestMultiLevelFields(t *testing.T) { data := `{ "Level1Field1":3, "Level1Field4":4, "Level1Field2":{ "Level2Field1":[ "value1", "value2" ], "Level2Field2":{ "Level3Field1":[ { "key1":"value1" } ] } } }` paths := []string{"Level1Field1", "Level1Field2.Level2Field1", "Level1Field2.Level2Field2.Level3Field1", "Level1Field4"} expected := []string{"3", `[value1 value2]`, `[map[key1:value1]]`, "4"} t.Run("SingleMany", func(t *testing.T) { testMany(t, data, paths, expected) }) } func TestRandomMany(t *testing.T) { var lstr string defer func() { if v := recover(); v != nil { println("'" + hex.EncodeToString([]byte(lstr)) + "'") println("'" + lstr + "'") panic(v) } }() rand.Seed(time.Now().UnixNano()) b := make([]byte, 512) for i := 0; i < 5000; i++ { n, err := rand.Read(b[:rand.Int()%len(b)]) if err != nil { t.Fatal(err) } lstr = string(b[:n]) paths := make([]string, rand.Int()%64) for i := range paths { var b []byte n := rand.Int() % 5 for j := 0; j < n; j++ { if j > 0 { b = append(b, '.') } nn := rand.Int() % 10 for k := 0; k < nn; k++ { b = append(b, 'a'+byte(rand.Int()%26)) } } paths[i] = string(b) } GetMany(lstr, paths...) } } func TestGetMany(t *testing.T) { data := `{"bar": {"id": 99, "mybar": "my mybar" }, "foo": ` + `{"myfoo": [605]}}` paths := []string{"foo.myfoo", "bar.id", "bar.mybar", "bar.mybarx"} expected := []string{"[605]", "99", "my mybar", ""} results := GetMany(data, paths...) if len(expected) != len(results) { t.Fatalf("expected %v, got %v", len(expected), len(results)) } for i, path := range paths { if results[i] != expected[i] { t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i], results[i], path) } } } func TestGetMany2(t *testing.T) { data := `{"bar": {"id": 99, "xyz": "my xyz"}, "foo": {"myfoo": [605]}}` paths := []string{"foo.myfoo", "bar.id", "bar.xyz", "bar.abc"} expected := []string{"[605]", "99", "my xyz", ""} results := GetMany(data, paths...) if len(expected) != len(results) { t.Fatalf("expected %v, got %v", len(expected), len(results)) } for i, path := range paths { if results[i] != expected[i] { t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i], results[i], path) } } } func TestNullArray(t *testing.T) { n, _ := get(`{"data":null}`, "data").Interface() if n != nil { t.Fatalf("expected '%v', got '%v'", nil, n) } n = get(`{}`, "data") if reflect.DeepEqual(n, nil) { t.Fatalf("expected '%v', got '%v'", nil, n) } n = get(`{"data":[]}`, "data") if reflect.DeepEqual(n, &ast.Node{}) { t.Fatalf("expected '%v', got '%v'", nil, n) } arr, _ := get(`{"data":[null]}`, "data").Array() n = len(arr) if n != 1 { t.Fatalf("expected '%v', got '%v'", 1, n) } } func TestGetMany3(t *testing.T) { var r []string data := `{"MarketName":null,"Nounce":6115}` r = GetMany(data, "Nounce", "Buys", "Sells", "Fills") if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) } r = GetMany(data, "Nounce", "Buys", "Sells") if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) } r = GetMany(data, "Nounce") if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) } } func TestGetMany4(t *testing.T) { data := `{"one": {"two": 2, "three": 3}, "four": 4, "five": 5}` results := GetMany(data, "four", "five", "one.two", "one.six") expected := []string{"4", "5", "2", ""} for i, r := range results { if r != expected[i] { t.Fatalf("expected %v, got %v", expected[i], r) } } } func TestGetNotExist(t *testing.T) { var dataStr = `{"m1":{"m2":3}}` ret, err := GetFromString(dataStr, "not_exist", "m3") if err == nil || ret.Exists() { t.Fatal("Get exist!") } if ret.Type() != ast.V_NONE { t.Fatal(ret.Type()) } ret, err = GetFromString(dataStr) if !ret.IsRaw() || ret.Type() != ast.V_OBJECT { t.Fatal(ret.Type()) } v11 := ret.Get("not_exist") if v11.Exists() { t.Fatal() } v2 := ret.GetByPath("m1", "m2") if !v2.Exists() || !v2.IsRaw() { t.Fatal(ret.Type()) } x, _ := v2.Int64() if x != 3 { t.Fatal(x) } }