transform_test.go

Documentation: golang.org/x/text/unicode/norm

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package norm
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  
    11  	"golang.org/x/text/transform"
    12  )
    13  
    14  func TestTransform(t *testing.T) {
    15  	tests := []struct {
    16  		f       Form
    17  		in, out string
    18  		eof     bool
    19  		dstSize int
    20  		err     error
    21  	}{
    22  		{NFC, "ab", "ab", true, 2, nil},
    23  		{NFC, "qx", "qx", true, 2, nil},
    24  		{NFD, "qx", "qx", true, 2, nil},
    25  		{NFC, "", "", true, 1, nil},
    26  		{NFD, "", "", true, 1, nil},
    27  		{NFC, "", "", false, 1, nil},
    28  		{NFD, "", "", false, 1, nil},
    29  
    30  		// Normalized segment does not fit in destination.
    31  		{NFD, "ö", "", true, 1, transform.ErrShortDst},
    32  		{NFD, "ö", "", true, 2, transform.ErrShortDst},
    33  
    34  		// As an artifact of the algorithm, only full segments are written.
    35  		// This is not strictly required, and some bytes could be written.
    36  		// In practice, for Transform to not block, the destination buffer
    37  		// should be at least MaxSegmentSize to work anyway and these edge
    38  		// conditions will be relatively rare.
    39  		{NFC, "ab", "", true, 1, transform.ErrShortDst},
    40  		// This is even true for inert runes.
    41  		{NFC, "qx", "", true, 1, transform.ErrShortDst},
    42  		{NFC, "a\u0300abc", "\u00e0a", true, 4, transform.ErrShortDst},
    43  
    44  		// We cannot write a segment if successive runes could still change the result.
    45  		{NFD, "ö", "", false, 3, transform.ErrShortSrc},
    46  		{NFC, "a\u0300", "", false, 4, transform.ErrShortSrc},
    47  		{NFD, "a\u0300", "", false, 4, transform.ErrShortSrc},
    48  		{NFC, "ö", "", false, 3, transform.ErrShortSrc},
    49  
    50  		{NFD, "abc", "", false, 1, transform.ErrShortDst},
    51  		{NFC, "abc", "", false, 1, transform.ErrShortDst},
    52  		{NFC, "abc", "a", false, 2, transform.ErrShortDst},
    53  		{NFD, "éfff", "", false, 2, transform.ErrShortDst},
    54  		{NFC, "e\u0301fffff", "\u00e9fff", false, 6, transform.ErrShortDst},
    55  		{NFD, "ééééé", "e\u0301e\u0301e\u0301", false, 15, transform.ErrShortDst},
    56  
    57  		{NFC, "a\u0300", "", true, 1, transform.ErrShortDst},
    58  		// Theoretically could fit, but won't due to simplified checks.
    59  		{NFC, "a\u0300", "", true, 2, transform.ErrShortDst},
    60  		{NFC, "a\u0300", "", true, 3, transform.ErrShortDst},
    61  		{NFC, "a\u0300", "\u00e0", true, 4, nil},
    62  
    63  		{NFD, "öa\u0300", "o\u0308", false, 8, transform.ErrShortSrc},
    64  		{NFD, "öa\u0300ö", "o\u0308a\u0300", true, 8, transform.ErrShortDst},
    65  		{NFD, "öa\u0300ö", "o\u0308a\u0300", false, 12, transform.ErrShortSrc},
    66  
    67  		// Illegal input is copied verbatim.
    68  		{NFD, "\xbd\xb2=\xbc ", "\xbd\xb2=\xbc ", true, 8, nil},
    69  	}
    70  	b := make([]byte, 100)
    71  	for i, tt := range tests {
    72  		t.Run(fmt.Sprintf("%d:%s", i, tt.in), func(t *testing.T) {
    73  			nDst, _, err := tt.f.Transform(b[:tt.dstSize], []byte(tt.in), tt.eof)
    74  			out := string(b[:nDst])
    75  			if out != tt.out || err != tt.err {
    76  				t.Errorf("was %+q (%v); want %+q (%v)", out, err, tt.out, tt.err)
    77  			}
    78  			if want := tt.f.String(tt.in)[:nDst]; want != out {
    79  				t.Errorf("incorrect normalization: was %+q; want %+q", out, want)
    80  			}
    81  		})
    82  	}
    83  }
    84  
    85  var transBufSizes = []int{
    86  	MaxTransformChunkSize,
    87  	3 * MaxTransformChunkSize / 2,
    88  	2 * MaxTransformChunkSize,
    89  	3 * MaxTransformChunkSize,
    90  	100 * MaxTransformChunkSize,
    91  }
    92  
    93  func doTransNorm(f Form, buf []byte, b []byte) []byte {
    94  	acc := []byte{}
    95  	for p := 0; p < len(b); {
    96  		nd, ns, _ := f.Transform(buf[:], b[p:], true)
    97  		p += ns
    98  		acc = append(acc, buf[:nd]...)
    99  	}
   100  	return acc
   101  }
   102  
   103  func TestTransformNorm(t *testing.T) {
   104  	for _, sz := range transBufSizes {
   105  		buf := make([]byte, sz)
   106  		runNormTests(t, fmt.Sprintf("Transform:%d", sz), func(f Form, out []byte, s string) []byte {
   107  			return doTransNorm(f, buf, append(out, s...))
   108  		})
   109  	}
   110  }
   111
View as plain text