maketables.go

Documentation: golang.org/x/text/encoding/simplifiedchinese

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  
     7  package main
     8  
     9  // This program generates tables.go:
    10  //	go run maketables.go | gofmt > tables.go
    11  
    12  import (
    13  	"bufio"
    14  	"fmt"
    15  	"log"
    16  	"net/http"
    17  	"sort"
    18  	"strings"
    19  )
    20  
    21  func main() {
    22  	fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
    23  	fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n")
    24  	fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n")
    25  
    26  	printGB18030()
    27  	printGBK()
    28  }
    29  
    30  func printGB18030() {
    31  	res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt")
    32  	if err != nil {
    33  		log.Fatalf("Get: %v", err)
    34  	}
    35  	defer res.Body.Close()
    36  
    37  	fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n")
    38  	fmt.Printf("var gb18030 = [...][2]uint16{\n")
    39  	scanner := bufio.NewScanner(res.Body)
    40  	for scanner.Scan() {
    41  		s := strings.TrimSpace(scanner.Text())
    42  		if s == "" || s[0] == '#' {
    43  			continue
    44  		}
    45  		x, y := uint32(0), uint32(0)
    46  		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
    47  			log.Fatalf("could not parse %q", s)
    48  		}
    49  		if x < 0x10000 && y < 0x10000 {
    50  			fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y)
    51  		}
    52  	}
    53  	fmt.Printf("}\n\n")
    54  }
    55  
    56  func printGBK() {
    57  	res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt")
    58  	if err != nil {
    59  		log.Fatalf("Get: %v", err)
    60  	}
    61  	defer res.Body.Close()
    62  
    63  	mapping := [65536]uint16{}
    64  	reverse := [65536]uint16{}
    65  
    66  	scanner := bufio.NewScanner(res.Body)
    67  	for scanner.Scan() {
    68  		s := strings.TrimSpace(scanner.Text())
    69  		if s == "" || s[0] == '#' {
    70  			continue
    71  		}
    72  		x, y := uint16(0), uint16(0)
    73  		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
    74  			log.Fatalf("could not parse %q", s)
    75  		}
    76  		if x < 0 || 126*190 <= x {
    77  			log.Fatalf("GBK code %d is out of range", x)
    78  		}
    79  		mapping[x] = y
    80  		if reverse[y] == 0 {
    81  			c0, c1 := x/190, x%190
    82  			if c1 >= 0x3f {
    83  				c1++
    84  			}
    85  			reverse[y] = (0x81+c0)<<8 | (0x40 + c1)
    86  		}
    87  	}
    88  	if err := scanner.Err(); err != nil {
    89  		log.Fatalf("scanner error: %v", err)
    90  	}
    91  
    92  	fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n")
    93  	fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n")
    94  	fmt.Printf("var decode = [...]uint16{\n")
    95  	for i, v := range mapping {
    96  		if v != 0 {
    97  			fmt.Printf("\t%d: 0x%04X,\n", i, v)
    98  		}
    99  	}
   100  	fmt.Printf("}\n\n")
   101  
   102  	// Any run of at least separation continuous zero entries in the reverse map will
   103  	// be a separate encode table.
   104  	const separation = 1024
   105  
   106  	intervals := []interval(nil)
   107  	low, high := -1, -1
   108  	for i, v := range reverse {
   109  		if v == 0 {
   110  			continue
   111  		}
   112  		if low < 0 {
   113  			low = i
   114  		} else if i-high >= separation {
   115  			if high >= 0 {
   116  				intervals = append(intervals, interval{low, high})
   117  			}
   118  			low = i
   119  		}
   120  		high = i + 1
   121  	}
   122  	if high >= 0 {
   123  		intervals = append(intervals, interval{low, high})
   124  	}
   125  	sort.Sort(byDecreasingLength(intervals))
   126  
   127  	fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
   128  	fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n")
   129  	fmt.Printf("// sorted by decreasing length.\n")
   130  	for i, v := range intervals {
   131  		fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
   132  	}
   133  	fmt.Printf("\n")
   134  
   135  	for i, v := range intervals {
   136  		fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
   137  		fmt.Printf("var encode%d = [...]uint16{\n", i)
   138  		for j := v.low; j < v.high; j++ {
   139  			x := reverse[j]
   140  			if x == 0 {
   141  				continue
   142  			}
   143  			fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
   144  		}
   145  		fmt.Printf("}\n\n")
   146  	}
   147  }
   148  
   149  // interval is a half-open interval [low, high).
   150  type interval struct {
   151  	low, high int
   152  }
   153  
   154  func (i interval) len() int { return i.high - i.low }
   155  
   156  // byDecreasingLength sorts intervals by decreasing length.
   157  type byDecreasingLength []interval
   158  
   159  func (b byDecreasingLength) Len() int           { return len(b) }
   160  func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
   161  func (b byDecreasingLength) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
   162
View as plain text