144 lines
4.6 KiB
Python
Executable File
144 lines
4.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""
|
|
Unicode case folding database conversion utility
|
|
|
|
Parses the database and generates a C++ function which implements the case
|
|
folding algorithm. The database entries are of the form:
|
|
|
|
<code>; <status>; <mapping>; # <name>
|
|
|
|
<status> can be one of four characters:
|
|
C - Common mappings
|
|
S - mappings for Simple case folding
|
|
F - mappings for Full case folding
|
|
T - special case for Turkish I characters
|
|
|
|
Right now this generates a function which implements simple case folding (C+S
|
|
entries).
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import sys
|
|
import re
|
|
try:
|
|
from urllib.request import urlopen
|
|
except ImportError:
|
|
from urllib2 import urlopen
|
|
|
|
|
|
# This variable will body of the mappings function
|
|
body = ""
|
|
|
|
# Reads file line-by-line, extracts Common and Simple case fold mappings and
|
|
# returns a (from_char, to_char, from_name) tuple.
|
|
def mappings(f):
|
|
previous_from = -1
|
|
expr = re.compile(r'^(.*); [CS]; (.*); # (.*)')
|
|
for line in f:
|
|
m = expr.match(line)
|
|
if not m: continue
|
|
from_char = int(m.group(1), 16)
|
|
to_char = int(m.group(2), 16)
|
|
from_name = m.group(3)
|
|
|
|
if from_char <= previous_from:
|
|
raise Exception("Duplicate or unsorted characters in input")
|
|
yield from_char, to_char, from_name
|
|
previous_from = from_char
|
|
|
|
# Computes the shift (to_char - from_char) in a mapping.
|
|
def shift(mapping):
|
|
return mapping[1] - mapping[0]
|
|
|
|
# Computes the stride (from_char2 - from_char1) of two mappings.
|
|
def stride2(mapping1, mapping2):
|
|
return mapping2[0] - mapping1[0]
|
|
|
|
# Computes the stride of a list of mappings. The list should have at least two
|
|
# mappings. All mappings in the list are assumed to have the same stride.
|
|
def stride(block):
|
|
return stride2(block[0], block[1])
|
|
|
|
|
|
# b is a list of mappings. All the mappings are assumed to have the same
|
|
# shift and the stride between adjecant mappings (if any) is constant.
|
|
def dump_block(b):
|
|
global body
|
|
|
|
if len(b) == 1:
|
|
# Special case for handling blocks of length 1. We don't even need to
|
|
# emit the "if (C < X) return C" check below as all characters in this
|
|
# range will be caught by the "C < X" check emitted by the first
|
|
# non-trivial block.
|
|
body += " // {2}\n if (C == {0:#06x})\n return {1:#06x};\n".format(*b[0])
|
|
return
|
|
|
|
first = b[0][0]
|
|
last = first + stride(b) * (len(b)-1)
|
|
modulo = first % stride(b)
|
|
|
|
# All characters before this block map to themselves.
|
|
body += " if (C < {0:#06x})\n return C;\n".format(first)
|
|
body += " // {0} characters\n".format(len(b))
|
|
|
|
# Generic pattern: check upper bound (lower bound is checked by the "if"
|
|
# above) and modulo of C, return C+shift.
|
|
pattern = " if (C <= {0:#06x} && C % {1} == {2})\n return C + {3};\n"
|
|
|
|
if stride(b) == 2 and shift(b[0]) == 1 and modulo == 0:
|
|
# Special case:
|
|
# We can elide the modulo-check because the expression "C|1" will map
|
|
# the intervening characters to themselves.
|
|
pattern = " if (C <= {0:#06x})\n return C | 1;\n"
|
|
elif stride(b) == 1:
|
|
# Another special case: X % 1 is always zero, so don't emit the
|
|
# modulo-check.
|
|
pattern = " if (C <= {0:#06x})\n return C + {3};\n"
|
|
|
|
body += pattern.format(last, stride(b), modulo, shift(b[0]))
|
|
|
|
current_block = []
|
|
f = urlopen(sys.argv[1])
|
|
for m in mappings(f):
|
|
if len(current_block) == 0:
|
|
current_block.append(m)
|
|
continue
|
|
|
|
if shift(current_block[0]) != shift(m):
|
|
# Incompatible shift, start a new block.
|
|
dump_block(current_block)
|
|
current_block = [m]
|
|
continue
|
|
|
|
if len(current_block) == 1 or stride(current_block) == stride2(current_block[-1], m):
|
|
current_block.append(m)
|
|
continue
|
|
|
|
# Incompatible stride, start a new block.
|
|
dump_block(current_block)
|
|
current_block = [m]
|
|
f.close()
|
|
|
|
dump_block(current_block)
|
|
|
|
print('//===---------- Support/UnicodeCaseFold.cpp -------------------------------===//')
|
|
print('//')
|
|
print('// This file was generated by utils/unicode-case-fold.py from the Unicode')
|
|
print('// case folding database at')
|
|
print('// ', sys.argv[1])
|
|
print('//')
|
|
print('// To regenerate this file, run:')
|
|
print('// utils/unicode-case-fold.py \\')
|
|
print('// "{}" \\'.format(sys.argv[1]))
|
|
print('// > lib/Support/UnicodeCaseFold.cpp')
|
|
print('//')
|
|
print('//===----------------------------------------------------------------------===//')
|
|
print('')
|
|
print('#include "llvm/Support/Unicode.h"')
|
|
print('')
|
|
print("int llvm::sys::unicode::foldCharSimple(int C) {")
|
|
print(body)
|
|
print(" return C;")
|
|
print("}")
|