maketables.go

Documentation: golang.org/x/text/encoding/korean

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  
     7  package main
     8  
     9  // This program generates tables.go:
    10  //	go run maketables.go | gofmt > tables.go
    11  
    12  import (
    13  	"bufio"
    14  	"fmt"
    15  	"log"
    16  	"net/http"
    17  	"sort"
    18  	"strings"
    19  )
    20  
    21  func main() {
    22  	fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
    23  	fmt.Printf("// Package korean provides Korean encodings such as EUC-KR.\n")
    24  	fmt.Printf(`package korean // import "golang.org/x/text/encoding/korean"` + "\n\n")
    25  
    26  	res, err := http.Get("http://encoding.spec.whatwg.org/index-euc-kr.txt")
    27  	if err != nil {
    28  		log.Fatalf("Get: %v", err)
    29  	}
    30  	defer res.Body.Close()
    31  
    32  	mapping := [65536]uint16{}
    33  	reverse := [65536]uint16{}
    34  
    35  	scanner := bufio.NewScanner(res.Body)
    36  	for scanner.Scan() {
    37  		s := strings.TrimSpace(scanner.Text())
    38  		if s == "" || s[0] == '#' {
    39  			continue
    40  		}
    41  		x, y := uint16(0), uint16(0)
    42  		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
    43  			log.Fatalf("could not parse %q", s)
    44  		}
    45  		if x < 0 || 178*(0xc7-0x81)+(0xfe-0xc7)*94+(0xff-0xa1) <= x {
    46  			log.Fatalf("EUC-KR code %d is out of range", x)
    47  		}
    48  		mapping[x] = y
    49  		if reverse[y] == 0 {
    50  			c0, c1 := uint16(0), uint16(0)
    51  			if x < 178*(0xc7-0x81) {
    52  				c0 = uint16(x/178) + 0x81
    53  				c1 = uint16(x % 178)
    54  				switch {
    55  				case c1 < 1*26:
    56  					c1 += 0x41
    57  				case c1 < 2*26:
    58  					c1 += 0x47
    59  				default:
    60  					c1 += 0x4d
    61  				}
    62  			} else {
    63  				x -= 178 * (0xc7 - 0x81)
    64  				c0 = uint16(x/94) + 0xc7
    65  				c1 = uint16(x%94) + 0xa1
    66  			}
    67  			reverse[y] = c0<<8 | c1
    68  		}
    69  	}
    70  	if err := scanner.Err(); err != nil {
    71  		log.Fatalf("scanner error: %v", err)
    72  	}
    73  
    74  	fmt.Printf("// decode is the decoding table from EUC-KR code to Unicode.\n")
    75  	fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-euc-kr.txt\n")
    76  	fmt.Printf("var decode = [...]uint16{\n")
    77  	for i, v := range mapping {
    78  		if v != 0 {
    79  			fmt.Printf("\t%d: 0x%04X,\n", i, v)
    80  		}
    81  	}
    82  	fmt.Printf("}\n\n")
    83  
    84  	// Any run of at least separation continuous zero entries in the reverse map will
    85  	// be a separate encode table.
    86  	const separation = 1024
    87  
    88  	intervals := []interval(nil)
    89  	low, high := -1, -1
    90  	for i, v := range reverse {
    91  		if v == 0 {
    92  			continue
    93  		}
    94  		if low < 0 {
    95  			low = i
    96  		} else if i-high >= separation {
    97  			if high >= 0 {
    98  				intervals = append(intervals, interval{low, high})
    99  			}
   100  			low = i
   101  		}
   102  		high = i + 1
   103  	}
   104  	if high >= 0 {
   105  		intervals = append(intervals, interval{low, high})
   106  	}
   107  	sort.Sort(byDecreasingLength(intervals))
   108  
   109  	fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
   110  	fmt.Printf("// encodeX are the encoding tables from Unicode to EUC-KR code,\n")
   111  	fmt.Printf("// sorted by decreasing length.\n")
   112  	for i, v := range intervals {
   113  		fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
   114  	}
   115  	fmt.Printf("\n")
   116  
   117  	for i, v := range intervals {
   118  		fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
   119  		fmt.Printf("var encode%d = [...]uint16{\n", i)
   120  		for j := v.low; j < v.high; j++ {
   121  			x := reverse[j]
   122  			if x == 0 {
   123  				continue
   124  			}
   125  			fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
   126  		}
   127  		fmt.Printf("}\n\n")
   128  	}
   129  }
   130  
   131  // interval is a half-open interval [low, high).
   132  type interval struct {
   133  	low, high int
   134  }
   135  
   136  func (i interval) len() int { return i.high - i.low }
   137  
   138  // byDecreasingLength sorts intervals by decreasing length.
   139  type byDecreasingLength []interval
   140  
   141  func (b byDecreasingLength) Len() int           { return len(b) }
   142  func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
   143  func (b byDecreasingLength) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
   144
View as plain text