table_test.go

Documentation: golang.org/x/text/collate

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package collate
     6  
     7  import (
     8  	"testing"
     9  
    10  	"golang.org/x/text/collate/build"
    11  	"golang.org/x/text/internal/colltab"
    12  	"golang.org/x/text/unicode/norm"
    13  )
    14  
    15  type ColElems []Weights
    16  
    17  type input struct {
    18  	str string
    19  	ces [][]int
    20  }
    21  
    22  type check struct {
    23  	in  string
    24  	n   int
    25  	out ColElems
    26  }
    27  
    28  type tableTest struct {
    29  	in  []input
    30  	chk []check
    31  }
    32  
    33  func w(ce ...int) Weights {
    34  	return W(ce...)
    35  }
    36  
    37  var defaults = w(0)
    38  
    39  func pt(p, t int) []int {
    40  	return []int{p, defaults.Secondary, t}
    41  }
    42  
    43  func makeTable(in []input) (*Collator, error) {
    44  	b := build.NewBuilder()
    45  	for _, r := range in {
    46  		if e := b.Add([]rune(r.str), r.ces, nil); e != nil {
    47  			panic(e)
    48  		}
    49  	}
    50  	t, err := b.Build()
    51  	if err != nil {
    52  		return nil, err
    53  	}
    54  	return NewFromTable(t), nil
    55  }
    56  
    57  // modSeq holds a seqeunce of modifiers in increasing order of CCC long enough
    58  // to cause a segment overflow if not handled correctly. The last rune in this
    59  // list has a CCC of 214.
    60  var modSeq = []rune{
    61  	0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, 0x05B8, 0x05B9, 0x05BB,
    62  	0x05BC, 0x05BD, 0x05BF, 0x05C1, 0x05C2, 0xFB1E, 0x064B, 0x064C, 0x064D, 0x064E,
    63  	0x064F, 0x0650, 0x0651, 0x0652, 0x0670, 0x0711, 0x0C55, 0x0C56, 0x0E38, 0x0E48,
    64  	0x0EB8, 0x0EC8, 0x0F71, 0x0F72, 0x0F74, 0x0321, 0x1DCE,
    65  }
    66  
    67  var mods []input
    68  var modW = func() ColElems {
    69  	ws := ColElems{}
    70  	for _, r := range modSeq {
    71  		rune := norm.NFC.PropertiesString(string(r))
    72  		ws = append(ws, w(0, int(rune.CCC())))
    73  		mods = append(mods, input{string(r), [][]int{{0, int(rune.CCC())}}})
    74  	}
    75  	return ws
    76  }()
    77  
    78  var appendNextTests = []tableTest{
    79  	{ // test getWeights
    80  		[]input{
    81  			{"a", [][]int{{100}}},
    82  			{"b", [][]int{{105}}},
    83  			{"c", [][]int{{110}}},
    84  			{"ß", [][]int{{120}}},
    85  		},
    86  		[]check{
    87  			{"a", 1, ColElems{w(100)}},
    88  			{"b", 1, ColElems{w(105)}},
    89  			{"c", 1, ColElems{w(110)}},
    90  			{"d", 1, ColElems{w(0x50064)}},
    91  			{"ab", 1, ColElems{w(100)}},
    92  			{"bc", 1, ColElems{w(105)}},
    93  			{"dd", 1, ColElems{w(0x50064)}},
    94  			{"ß", 2, ColElems{w(120)}},
    95  		},
    96  	},
    97  	{ // test expansion
    98  		[]input{
    99  			{"u", [][]int{{100}}},
   100  			{"U", [][]int{{100}, {0, 25}}},
   101  			{"w", [][]int{{100}, {100}}},
   102  			{"W", [][]int{{100}, {0, 25}, {100}, {0, 25}}},
   103  		},
   104  		[]check{
   105  			{"u", 1, ColElems{w(100)}},
   106  			{"U", 1, ColElems{w(100), w(0, 25)}},
   107  			{"w", 1, ColElems{w(100), w(100)}},
   108  			{"W", 1, ColElems{w(100), w(0, 25), w(100), w(0, 25)}},
   109  		},
   110  	},
   111  	{ // test decompose
   112  		[]input{
   113  			{"D", [][]int{pt(104, 8)}},
   114  			{"z", [][]int{pt(130, 8)}},
   115  			{"\u030C", [][]int{{0, 40}}},                               // Caron
   116  			{"\u01C5", [][]int{pt(104, 9), pt(130, 4), {0, 40, 0x1F}}}, // ǅ = D+z+caron
   117  		},
   118  		[]check{
   119  			{"\u01C5", 2, ColElems{w(pt(104, 9)...), w(pt(130, 4)...), w(0, 40, 0x1F)}},
   120  		},
   121  	},
   122  	{ // test basic contraction
   123  		[]input{
   124  			{"a", [][]int{{100}}},
   125  			{"ab", [][]int{{101}}},
   126  			{"aab", [][]int{{101}, {101}}},
   127  			{"abc", [][]int{{102}}},
   128  			{"b", [][]int{{200}}},
   129  			{"c", [][]int{{300}}},
   130  			{"d", [][]int{{400}}},
   131  		},
   132  		[]check{
   133  			{"a", 1, ColElems{w(100)}},
   134  			{"aa", 1, ColElems{w(100)}},
   135  			{"aac", 1, ColElems{w(100)}},
   136  			{"d", 1, ColElems{w(400)}},
   137  			{"ab", 2, ColElems{w(101)}},
   138  			{"abb", 2, ColElems{w(101)}},
   139  			{"aab", 3, ColElems{w(101), w(101)}},
   140  			{"aaba", 3, ColElems{w(101), w(101)}},
   141  			{"abc", 3, ColElems{w(102)}},
   142  			{"abcd", 3, ColElems{w(102)}},
   143  		},
   144  	},
   145  	{ // test discontinuous contraction
   146  		append(mods, []input{
   147  			// modifiers; secondary weight equals ccc
   148  			{"\u0316", [][]int{{0, 220}}},
   149  			{"\u0317", [][]int{{0, 220}, {0, 220}}},
   150  			{"\u302D", [][]int{{0, 222}}},
   151  			{"\u302E", [][]int{{0, 225}}}, // used as starter
   152  			{"\u302F", [][]int{{0, 224}}}, // used as starter
   153  			{"\u18A9", [][]int{{0, 228}}},
   154  			{"\u0300", [][]int{{0, 230}}},
   155  			{"\u0301", [][]int{{0, 230}}},
   156  			{"\u0315", [][]int{{0, 232}}},
   157  			{"\u031A", [][]int{{0, 232}}},
   158  			{"\u035C", [][]int{{0, 233}}},
   159  			{"\u035F", [][]int{{0, 233}}},
   160  			{"\u035D", [][]int{{0, 234}}},
   161  			{"\u035E", [][]int{{0, 234}}},
   162  			{"\u0345", [][]int{{0, 240}}},
   163  
   164  			// starters
   165  			{"a", [][]int{{100}}},
   166  			{"b", [][]int{{200}}},
   167  			{"c", [][]int{{300}}},
   168  			{"\u03B1", [][]int{{900}}},
   169  			{"\x01", [][]int{{0, 0, 0, 0}}},
   170  
   171  			// contractions
   172  			{"a\u0300", [][]int{{101}}},
   173  			{"a\u0301", [][]int{{102}}},
   174  			{"a\u035E", [][]int{{110}}},
   175  			{"a\u035Eb\u035E", [][]int{{115}}},
   176  			{"ac\u035Eaca\u035E", [][]int{{116}}},
   177  			{"a\u035Db\u035D", [][]int{{117}}},
   178  			{"a\u0301\u035Db", [][]int{{120}}},
   179  			{"a\u0301\u035F", [][]int{{121}}},
   180  			{"a\u0301\u035Fb", [][]int{{119}}},
   181  			{"\u03B1\u0345", [][]int{{901}, {902}}},
   182  			{"\u302E\u302F", [][]int{{0, 131}, {0, 131}}},
   183  			{"\u302F\u18A9", [][]int{{0, 130}}},
   184  		}...),
   185  		[]check{
   186  			{"a\x01\u0300", 1, ColElems{w(100)}},
   187  			{"ab", 1, ColElems{w(100)}},                              // closing segment
   188  			{"a\u0316\u0300b", 5, ColElems{w(101), w(0, 220)}},       // closing segment
   189  			{"a\u0316\u0300", 5, ColElems{w(101), w(0, 220)}},        // no closing segment
   190  			{"a\u0316\u0300\u035Cb", 5, ColElems{w(101), w(0, 220)}}, // completes before segment end
   191  			{"a\u0316\u0300\u035C", 5, ColElems{w(101), w(0, 220)}},  // completes before segment end
   192  
   193  			{"a\u0316\u0301b", 5, ColElems{w(102), w(0, 220)}},       // closing segment
   194  			{"a\u0316\u0301", 5, ColElems{w(102), w(0, 220)}},        // no closing segment
   195  			{"a\u0316\u0301\u035Cb", 5, ColElems{w(102), w(0, 220)}}, // completes before segment end
   196  			{"a\u0316\u0301\u035C", 5, ColElems{w(102), w(0, 220)}},  // completes before segment end
   197  
   198  			// match blocked by modifier with same ccc
   199  			{"a\u0301\u0315\u031A\u035Fb", 3, ColElems{w(102)}},
   200  
   201  			// multiple gaps
   202  			{"a\u0301\u035Db", 6, ColElems{w(120)}},
   203  			{"a\u0301\u035F", 5, ColElems{w(121)}},
   204  			{"a\u0301\u035Fb", 6, ColElems{w(119)}},
   205  			{"a\u0316\u0301\u035F", 7, ColElems{w(121), w(0, 220)}},
   206  			{"a\u0301\u0315\u035Fb", 7, ColElems{w(121), w(0, 232)}},
   207  			{"a\u0316\u0301\u0315\u035Db", 5, ColElems{w(102), w(0, 220)}},
   208  			{"a\u0316\u0301\u0315\u035F", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
   209  			{"a\u0316\u0301\u0315\u035Fb", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
   210  			{"a\u0316\u0301\u0315\u035F\u035D", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
   211  			{"a\u0316\u0301\u0315\u035F\u035Db", 9, ColElems{w(121), w(0, 220), w(0, 232)}},
   212  
   213  			// handling of segment overflow
   214  			{ // just fits within segment
   215  				"a" + string(modSeq[:30]) + "\u0301",
   216  				3 + len(string(modSeq[:30])),
   217  				append(ColElems{w(102)}, modW[:30]...),
   218  			},
   219  			{"a" + string(modSeq[:31]) + "\u0301", 1, ColElems{w(100)}}, // overflow
   220  			{"a" + string(modSeq) + "\u0301", 1, ColElems{w(100)}},
   221  			{ // just fits within segment with two interstitial runes
   222  				"a" + string(modSeq[:28]) + "\u0301\u0315\u035F",
   223  				7 + len(string(modSeq[:28])),
   224  				append(append(ColElems{w(121)}, modW[:28]...), w(0, 232)),
   225  			},
   226  			{ // second half does not fit within segment
   227  				"a" + string(modSeq[:29]) + "\u0301\u0315\u035F",
   228  				3 + len(string(modSeq[:29])),
   229  				append(ColElems{w(102)}, modW[:29]...),
   230  			},
   231  
   232  			// discontinuity can only occur in last normalization segment
   233  			{"a\u035Eb\u035E", 6, ColElems{w(115)}},
   234  			{"a\u0316\u035Eb\u035E", 5, ColElems{w(110), w(0, 220)}},
   235  			{"a\u035Db\u035D", 6, ColElems{w(117)}},
   236  			{"a\u0316\u035Db\u035D", 1, ColElems{w(100)}},
   237  			{"a\u035Eb\u0316\u035E", 8, ColElems{w(115), w(0, 220)}},
   238  			{"a\u035Db\u0316\u035D", 8, ColElems{w(117), w(0, 220)}},
   239  			{"ac\u035Eaca\u035E", 9, ColElems{w(116)}},
   240  			{"a\u0316c\u035Eaca\u035E", 1, ColElems{w(100)}},
   241  			{"ac\u035Eac\u0316a\u035E", 1, ColElems{w(100)}},
   242  
   243  			// expanding contraction
   244  			{"\u03B1\u0345", 4, ColElems{w(901), w(902)}},
   245  
   246  			// Theoretical possibilities
   247  			// contraction within a gap
   248  			{"a\u302F\u18A9\u0301", 9, ColElems{w(102), w(0, 130)}},
   249  			// expansion within a gap
   250  			{"a\u0317\u0301", 5, ColElems{w(102), w(0, 220), w(0, 220)}},
   251  			// repeating CCC blocks last modifier
   252  			{"a\u302E\u302F\u0301", 1, ColElems{w(100)}},
   253  			// The trailing combining characters (with lower CCC) should block the first one.
   254  			// TODO: make the following pass.
   255  			// {"a\u035E\u0316\u0316", 1, ColElems{w(100)}},
   256  			{"a\u035F\u035Eb", 5, ColElems{w(110), w(0, 233)}},
   257  			// Last combiner should match after normalization.
   258  			// TODO: make the following pass.
   259  			// {"a\u035D\u0301", 3, ColElems{w(102), w(0, 234)}},
   260  			// The first combiner is blocking the second one as they have the same CCC.
   261  			{"a\u035D\u035Eb", 1, ColElems{w(100)}},
   262  		},
   263  	},
   264  }
   265  
   266  func TestAppendNext(t *testing.T) {
   267  	for i, tt := range appendNextTests {
   268  		c, err := makeTable(tt.in)
   269  		if err != nil {
   270  			t.Errorf("%d: error creating table: %v", i, err)
   271  			continue
   272  		}
   273  		for j, chk := range tt.chk {
   274  			ws, n := c.t.AppendNext(nil, []byte(chk.in))
   275  			if n != chk.n {
   276  				t.Errorf("%d:%d: bytes consumed was %d; want %d", i, j, n, chk.n)
   277  			}
   278  			out := convertFromWeights(chk.out)
   279  			if len(ws) != len(out) {
   280  				t.Errorf("%d:%d: len(ws) was %d; want %d (%X vs %X)\n%X", i, j, len(ws), len(out), ws, out, chk.in)
   281  				continue
   282  			}
   283  			for k, w := range ws {
   284  				w, _ = colltab.MakeElem(w.Primary(), w.Secondary(), int(w.Tertiary()), 0)
   285  				if w != out[k] {
   286  					t.Errorf("%d:%d: Weights %d was %X; want %X", i, j, k, w, out[k])
   287  				}
   288  			}
   289  		}
   290  	}
   291  }
   292
View as plain text