cbor_test.go

Documentation: github.com/ugorji/go/codec

     1  // Copyright (c) 2012-2020 Ugorji Nwoke. All rights reserved.
     2  // Use of this source code is governed by a MIT license found in the LICENSE file.
     3  
     4  package codec
     5  
     6  import (
     7  	"bufio"
     8  	"bytes"
     9  	"encoding/hex"
    10  	"math"
    11  	"os"
    12  	"reflect"
    13  	"regexp"
    14  	"strings"
    15  	"testing"
    16  )
    17  
    18  func TestCborIndefiniteLength(t *testing.T) {
    19  	var h Handle = testCborH
    20  	defer testSetup(t, &h)()
    21  	bh := testBasicHandle(h)
    22  	defer func(oldMapType reflect.Type) {
    23  		bh.MapType = oldMapType
    24  	}(bh.MapType)
    25  	bh.MapType = testMapStrIntfTyp
    26  	// var (
    27  	// 	M1 map[string][]byte
    28  	// 	M2 map[uint64]bool
    29  	// 	L1 []interface{}
    30  	// 	S1 []string
    31  	// 	B1 []byte
    32  	// )
    33  	var v, vv interface{}
    34  	// define it (v), encode it using indefinite lengths, decode it (vv), compare v to vv
    35  	v = map[string]interface{}{
    36  		"one-byte-key":   []byte{1, 2, 3, 4, 5, 6},
    37  		"two-string-key": "two-value",
    38  		"three-list-key": []interface{}{true, false, uint64(1), int64(-1)},
    39  	}
    40  	var buf bytes.Buffer
    41  	// buf.Reset()
    42  	e := NewEncoder(&buf, h)
    43  	buf.WriteByte(cborBdIndefiniteMap)
    44  	//----
    45  	buf.WriteByte(cborBdIndefiniteString)
    46  	e.MustEncode("one-")
    47  	e.MustEncode("byte-")
    48  	e.MustEncode("key")
    49  	buf.WriteByte(cborBdBreak)
    50  
    51  	buf.WriteByte(cborBdIndefiniteBytes)
    52  	e.MustEncode([]byte{1, 2, 3})
    53  	e.MustEncode([]byte{4, 5, 6})
    54  	buf.WriteByte(cborBdBreak)
    55  
    56  	//----
    57  	buf.WriteByte(cborBdIndefiniteString)
    58  	e.MustEncode("two-")
    59  	e.MustEncode("string-")
    60  	e.MustEncode("key")
    61  	buf.WriteByte(cborBdBreak)
    62  
    63  	buf.WriteByte(cborBdIndefiniteString)
    64  	e.MustEncode("two-")
    65  	e.MustEncode("value")
    66  	buf.WriteByte(cborBdBreak)
    67  
    68  	//----
    69  	buf.WriteByte(cborBdIndefiniteString)
    70  	e.MustEncode("three-")
    71  	e.MustEncode("list-")
    72  	e.MustEncode("key")
    73  	buf.WriteByte(cborBdBreak)
    74  
    75  	buf.WriteByte(cborBdIndefiniteArray)
    76  	e.MustEncode(true)
    77  	e.MustEncode(false)
    78  	e.MustEncode(uint64(1))
    79  	e.MustEncode(int64(-1))
    80  	buf.WriteByte(cborBdBreak)
    81  
    82  	buf.WriteByte(cborBdBreak) // close map
    83  
    84  	NewDecoderBytes(buf.Bytes(), h).MustDecode(&vv)
    85  	if err := deepEqual(v, vv); err != nil {
    86  		t.Logf("-------- Before and After marshal do not match: Error: %v", err)
    87  		if testVerbose {
    88  			t.Logf("    ....... GOLDEN:  (%T) %#v", v, v)
    89  			t.Logf("    ....... DECODED: (%T) %#v", vv, vv)
    90  		}
    91  		t.FailNow()
    92  	}
    93  }
    94  
    95  // "If any item between the indefinite-length string indicator (0b010_11111 or 0b011_11111) and the
    96  // "break" stop code is not a definite-length string item of the same major type, the string is not
    97  // well-formed."
    98  func TestCborIndefiniteLengthStringChunksCannotMixTypes(t *testing.T) {
    99  	if !testRecoverPanicToErr {
   100  		t.Skip(testSkipIfNotRecoverPanicToErrMsg)
   101  	}
   102  	var h Handle = testCborH
   103  	defer testSetup(t, &h)()
   104  
   105  	for _, in := range [][]byte{
   106  		{cborBdIndefiniteString, 0x40, cborBdBreak}, // byte string chunk in indefinite length text string
   107  		{cborBdIndefiniteBytes, 0x60, cborBdBreak},  // text string chunk in indefinite length byte string
   108  	} {
   109  		var out string
   110  		err := NewDecoderBytes(in, h).Decode(&out)
   111  		if err == nil {
   112  			t.Errorf("expected error but decoded 0x%x to: %q", in, out)
   113  		}
   114  	}
   115  }
   116  
   117  // "If any definite-length text string inside an indefinite-length text string is invalid, the
   118  // indefinite-length text string is invalid. Note that this implies that the UTF-8 bytes of a single
   119  // Unicode code point (scalar value) cannot be spread between chunks: a new chunk of a text string
   120  // can only be started at a code point boundary."
   121  func TestCborIndefiniteLengthTextStringChunksAreUTF8(t *testing.T) {
   122  	if !testRecoverPanicToErr {
   123  		t.Skip(testSkipIfNotRecoverPanicToErrMsg)
   124  	}
   125  	var h Handle = testCborH
   126  	defer testSetup(t, &h)()
   127  
   128  	bh := testBasicHandle(h)
   129  	defer func(oldValidateUnicode bool) {
   130  		bh.ValidateUnicode = oldValidateUnicode
   131  	}(bh.ValidateUnicode)
   132  	bh.ValidateUnicode = true
   133  
   134  	var out string
   135  	in := []byte{cborBdIndefiniteString, 0x61, 0xc2, 0x61, 0xa3, cborBdBreak}
   136  	err := NewDecoderBytes(in, h).Decode(&out)
   137  	if err == nil {
   138  		t.Errorf("expected error but decoded to: %q", out)
   139  	}
   140  }
   141  
   142  type testCborGolden struct {
   143  	Base64     string      `codec:"cbor"`
   144  	Hex        string      `codec:"hex"`
   145  	Roundtrip  bool        `codec:"roundtrip"`
   146  	Decoded    interface{} `codec:"decoded"`
   147  	Diagnostic string      `codec:"diagnostic"`
   148  	Skip       bool        `codec:"skip"`
   149  }
   150  
   151  // Some tests are skipped because they include numbers outside the range of int64/uint64
   152  func TestCborGoldens(t *testing.T) {
   153  	var h Handle = testCborH
   154  	defer testSetup(t, &h)()
   155  	bh := testBasicHandle(h)
   156  	defer func(oldMapType reflect.Type) {
   157  		bh.MapType = oldMapType
   158  	}(bh.MapType)
   159  	bh.MapType = testMapStrIntfTyp
   160  
   161  	// decode test-cbor-goldens.json into a list of []*testCborGolden
   162  	// for each one,
   163  	// - decode hex into []byte bs
   164  	// - decode bs into interface{} v
   165  	// - compare both using deepequal
   166  	// - for any miss, record it
   167  	var gs []*testCborGolden
   168  	f, err := os.Open("test-cbor-goldens.json")
   169  	if err != nil {
   170  		t.Logf("error opening test-cbor-goldens.json: %v", err)
   171  		t.FailNow()
   172  	}
   173  	defer f.Close()
   174  	jh := new(JsonHandle)
   175  	jh.MapType = testMapStrIntfTyp
   176  	// d := NewDecoder(f, jh)
   177  	d := NewDecoder(bufio.NewReader(f), jh)
   178  	// err = d.Decode(&gs)
   179  	d.MustDecode(&gs)
   180  	if err != nil {
   181  		t.Logf("error json decoding test-cbor-goldens.json: %v", err)
   182  		t.FailNow()
   183  	}
   184  
   185  	tagregex := regexp.MustCompile(`[\d]+\(.+?\)`)
   186  	hexregex := regexp.MustCompile(`h'([0-9a-fA-F]*)'`)
   187  	for i, g := range gs {
   188  		// fmt.Printf("%v, skip: %v, isTag: %v, %s\n", i, g.Skip, tagregex.MatchString(g.Diagnostic), g.Diagnostic)
   189  		// skip tags or simple or those with prefix, as we can't verify them.
   190  		if g.Skip || strings.HasPrefix(g.Diagnostic, "simple(") || tagregex.MatchString(g.Diagnostic) {
   191  			// fmt.Printf("%v: skipped\n", i)
   192  			if testVerbose {
   193  				t.Logf("[%v] skipping because skip=true OR unsupported simple value or Tag Value", i)
   194  			}
   195  			continue
   196  		}
   197  		// println("++++++++++++", i, "g.Diagnostic", g.Diagnostic)
   198  		if hexregex.MatchString(g.Diagnostic) {
   199  			// println(i, "g.Diagnostic matched hex")
   200  			if s2 := g.Diagnostic[2 : len(g.Diagnostic)-1]; s2 == "" {
   201  				g.Decoded = zeroByteSlice
   202  			} else if bs2, err2 := hex.DecodeString(s2); err2 == nil {
   203  				g.Decoded = bs2
   204  			}
   205  			// fmt.Printf("%v: hex: %v\n", i, g.Decoded)
   206  		}
   207  		bs, err := hex.DecodeString(g.Hex)
   208  		if err != nil {
   209  			t.Logf("[%v] error hex decoding %s [%v]: %v", i, g.Hex, g.Hex, err)
   210  			t.FailNow()
   211  		}
   212  		var v interface{}
   213  		NewDecoderBytes(bs, h).MustDecode(&v)
   214  		if _, ok := v.(RawExt); ok {
   215  			continue
   216  		}
   217  		// check the diagnostics to compare
   218  		switch g.Diagnostic {
   219  		case "Infinity":
   220  			b := math.IsInf(v.(float64), 1)
   221  			testCborError(t, i, math.Inf(1), v, nil, &b)
   222  		case "-Infinity":
   223  			b := math.IsInf(v.(float64), -1)
   224  			testCborError(t, i, math.Inf(-1), v, nil, &b)
   225  		case "NaN":
   226  			// println(i, "checking NaN")
   227  			b := math.IsNaN(v.(float64))
   228  			testCborError(t, i, math.NaN(), v, nil, &b)
   229  		case "undefined":
   230  			b := v == nil
   231  			testCborError(t, i, nil, v, nil, &b)
   232  		default:
   233  			v0 := g.Decoded
   234  			// testCborCoerceJsonNumber(reflect.ValueOf(&v0))
   235  			testCborError(t, i, v0, v, deepEqual(v0, v), nil)
   236  		}
   237  	}
   238  }
   239  
   240  func testCborError(t *testing.T, i int, v0, v1 interface{}, err error, equal *bool) {
   241  	if err == nil && equal == nil {
   242  		// fmt.Printf("%v testCborError passed (err and equal nil)\n", i)
   243  		return
   244  	}
   245  	if err != nil {
   246  		t.Logf("[%v] deepEqual error: %v", i, err)
   247  		if testVerbose {
   248  			t.Logf("    ....... GOLDEN:  (%T) %#v", v0, v0)
   249  			t.Logf("    ....... DECODED: (%T) %#v", v1, v1)
   250  		}
   251  		t.FailNow()
   252  	}
   253  	if equal != nil && !*equal {
   254  		t.Logf("[%v] values not equal", i)
   255  		if testVerbose {
   256  			t.Logf("    ....... GOLDEN:  (%T) %#v", v0, v0)
   257  			t.Logf("    ....... DECODED: (%T) %#v", v1, v1)
   258  		}
   259  		t.FailNow()
   260  	}
   261  	// fmt.Printf("%v testCborError passed (checks passed)\n", i)
   262  }
   263  
   264  func TestCborHalfFloat(t *testing.T) {
   265  	var h Handle = testCborH
   266  	defer testSetup(t, &h)()
   267  	m := map[uint16]float64{
   268  		// using examples from
   269  		// https://en.wikipedia.org/wiki/Half-precision_floating-point_format
   270  		0x3c00: 1,
   271  		0x3c01: 1 + math.Pow(2, -10),
   272  		0xc000: -2,
   273  		0x7bff: 65504,
   274  		0x0400: math.Pow(2, -14),
   275  		0x03ff: math.Pow(2, -14) - math.Pow(2, -24),
   276  		0x0001: math.Pow(2, -24),
   277  		0x0000: 0,
   278  		0x8000: -0.0,
   279  	}
   280  	var ba [3]byte
   281  	ba[0] = cborBdFloat16
   282  	var res float64
   283  	for k, v := range m {
   284  		res = 0
   285  		bigenstd.PutUint16(ba[1:], k)
   286  		testUnmarshalErr(&res, ba[:3], h, t, "-")
   287  		if res == v {
   288  			if testVerbose {
   289  				t.Logf("equal floats: from %x %b, %v", k, k, v)
   290  			}
   291  		} else {
   292  			t.Logf("unequal floats: from %x %b, %v != %v", k, k, res, v)
   293  			t.FailNow()
   294  		}
   295  	}
   296  }
   297  
   298  func TestCborSkipTags(t *testing.T) {
   299  	defer testSetup(t, nil)()
   300  	type Tcbortags struct {
   301  		A string
   302  		M map[string]interface{}
   303  		// A []interface{}
   304  	}
   305  	var b8 [8]byte
   306  	var w bytesEncAppender
   307  	w.b = []byte{}
   308  
   309  	// To make it easier,
   310  	//    - use tags between math.MaxUint8 and math.MaxUint16 (incl SelfDesc)
   311  	//    - use 1 char strings for key names
   312  	//    - use 3-6 char strings for map keys
   313  	//    - use integers that fit in 2 bytes (between 0x20 and 0xff)
   314  
   315  	var tags = [...]uint64{math.MaxUint8 * 2, math.MaxUint8 * 8, 55799, math.MaxUint16 / 2}
   316  	var tagIdx int
   317  	var doAddTag bool
   318  	addTagFn8To16 := func() {
   319  		if !doAddTag {
   320  			return
   321  		}
   322  		// writes a tag between MaxUint8 and MaxUint16 (culled from cborEncDriver.encUint)
   323  		w.writen1(cborBaseTag + 0x19)
   324  		// bigenHelper.writeUint16
   325  		bigenstd.PutUint16(b8[:2], uint16(tags[tagIdx%len(tags)]))
   326  		w.writeb(b8[:2])
   327  		tagIdx++
   328  	}
   329  
   330  	var v Tcbortags
   331  	v.A = "cbor"
   332  	v.M = make(map[string]interface{})
   333  	v.M["111"] = uint64(111)
   334  	v.M["111.11"] = 111.11
   335  	v.M["true"] = true
   336  	// v.A = append(v.A, 222, 22.22, "true")
   337  
   338  	// make stream manually (interspacing tags around it)
   339  	// WriteMapStart - e.encLen(cborBaseMap, length) - encUint(length, bd)
   340  	// EncodeStringEnc - e.encStringBytesS(cborBaseString, v)
   341  
   342  	fnEncode := func() {
   343  		w.b = w.b[:0]
   344  		addTagFn8To16()
   345  		// write v (Tcbortags, with 3 fields = map with 3 entries)
   346  		w.writen1(2 + cborBaseMap) // 3 fields = 3 entries
   347  		// write v.A
   348  		var s = "A"
   349  		w.writen1(byte(len(s)) + cborBaseString)
   350  		w.writestr(s)
   351  		w.writen1(byte(len(v.A)) + cborBaseString)
   352  		w.writestr(v.A)
   353  		//w.writen1(0)
   354  
   355  		addTagFn8To16()
   356  		s = "M"
   357  		w.writen1(byte(len(s)) + cborBaseString)
   358  		w.writestr(s)
   359  
   360  		addTagFn8To16()
   361  		w.writen1(byte(len(v.M)) + cborBaseMap)
   362  
   363  		addTagFn8To16()
   364  		s = "111"
   365  		w.writen1(byte(len(s)) + cborBaseString)
   366  		w.writestr(s)
   367  		w.writen2(cborBaseUint+0x18, uint8(111))
   368  
   369  		addTagFn8To16()
   370  		s = "111.11"
   371  		w.writen1(byte(len(s)) + cborBaseString)
   372  		w.writestr(s)
   373  		w.writen1(cborBdFloat64)
   374  		bigenstd.PutUint64(b8[:8], math.Float64bits(111.11))
   375  		w.writeb(b8[:8])
   376  
   377  		addTagFn8To16()
   378  		s = "true"
   379  		w.writen1(byte(len(s)) + cborBaseString)
   380  		w.writestr(s)
   381  		w.writen1(cborBdTrue)
   382  	}
   383  
   384  	var h CborHandle
   385  	h.SkipUnexpectedTags = true
   386  	h.Canonical = true
   387  
   388  	var gold []byte
   389  	NewEncoderBytes(&gold, &h).MustEncode(v)
   390  	// xdebug2f("encoded:    gold: %v", gold)
   391  
   392  	// w.b is the encoded bytes
   393  	var v2 Tcbortags
   394  	doAddTag = false
   395  	fnEncode()
   396  	// xdebug2f("manual:  no-tags: %v", w.b)
   397  
   398  	testDeepEqualErr(gold, w.b, t, "cbor-skip-tags--bytes---")
   399  	NewDecoderBytes(w.b, &h).MustDecode(&v2)
   400  	testDeepEqualErr(v, v2, t, "cbor-skip-tags--no-tags-")
   401  
   402  	var v3 Tcbortags
   403  	doAddTag = true
   404  	fnEncode()
   405  	// xdebug2f("manual: has-tags: %v", w.b)
   406  	NewDecoderBytes(w.b, &h).MustDecode(&v3)
   407  	testDeepEqualErr(v, v2, t, "cbor-skip-tags--has-tags")
   408  
   409  	// Github 300 - tests naked path
   410  	{
   411  		expected := []interface{}{"x", uint64(0x0)}
   412  		toDecode := []byte{0x82, 0x61, 0x78, 0x00}
   413  
   414  		var raw interface{}
   415  
   416  		NewDecoderBytes(toDecode, &h).MustDecode(&raw)
   417  		testDeepEqualErr(expected, raw, t, "cbor-skip-tags--gh-300---no-skips")
   418  
   419  		toDecode = []byte{0xd9, 0xd9, 0xf7, 0x82, 0x61, 0x78, 0x00}
   420  		raw = nil
   421  		NewDecoderBytes(toDecode, &h).MustDecode(&raw)
   422  		testDeepEqualErr(expected, raw, t, "cbor-skip-tags--gh-300--has-skips")
   423  	}
   424  }
   425  
   426  func TestCborMalformed(t *testing.T) {
   427  	if !testRecoverPanicToErr {
   428  		t.Skip(testSkipIfNotRecoverPanicToErrMsg)
   429  	}
   430  	var h Handle = testCborH
   431  	defer testSetup(t, &h)()
   432  	var bad = [][]byte{
   433  		[]byte("\x9b\x00\x00000000"),
   434  		[]byte("\x9b\x00\x00\x81112233"),
   435  	}
   436  
   437  	var out interface{}
   438  	for _, v := range bad {
   439  		out = nil
   440  		err := testUnmarshal(&out, v, h)
   441  		if err == nil {
   442  			t.Logf("missing expected error decoding malformed cbor")
   443  			t.FailNow()
   444  		}
   445  	}
   446  }
   447
View as plain text