#!/usr/bin/python\r
-# -*- coding: <encoding name> -*-\r
+# -*- coding: ascii -*-\r
# Unicode Data Converter\r
#\r
# This file is part of the Public Domain C Library (PDCLib).\r
\r
Download the data from\r
ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt\r
+\r
+We do some simple "run" compression, because characters in the Unicode Data file\r
+tend to come in groups with the same properties.\r
"""\r
import os\r
\r
BIT_LOWER = 64\r
BIT_UPPER = 128\r
BIT_DIGIT = 256\r
+BIT_XDIGT = 512\r
\r
# Category to bitfield mapping\r
categories = {\r
'Sc': BIT_GRAPH, # Currency symbol\r
'Sk': BIT_GRAPH, # Non-letterlike modifier symbol\r
'So': BIT_GRAPH, # Other symbol\r
- 'Zs': BIT_SPACE | BIT_GRAPH | BIT_BLANK, # Non-zero-width space character\r
- 'Zl': BIT_SPACE | BIT_GRAPH, # Line separator\r
- 'Zp': BIT_SPACE | BIT_GRAPH, # Paragraph separator\r
+ 'Zs': BIT_SPACE, # Non-zero-width space character\r
+ 'Zl': BIT_SPACE, # Line separator\r
+ 'Zp': BIT_SPACE, # Paragraph separator\r
'Cc': BIT_CNTRL, # C0/C1 control codes\r
}\r
\r
+# Characters with special properties\r
+special = {\r
+ # Blank characters\r
+ 0x0020: BIT_SPACE | BIT_BLANK, # space\r
+ 0x0009: BIT_SPACE | BIT_BLANK, # tab\r
+\r
+ # Digits\r
+ 0x0030: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0031: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0032: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0033: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0034: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0035: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0036: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0037: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0038: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+ 0x0039: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
+\r
+ # A-F (hex uppercase)\r
+ 0x0041: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+ 0x0042: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+ 0x0043: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+ 0x0044: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+ 0x0045: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+ 0x0046: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_UPPER,\r
+\r
+\r
+ # a-f (hex lowercase)\r
+ 0x0061: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+ 0x0062: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+ 0x0063: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+ 0x0064: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+ 0x0065: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+ 0x0066: BIT_XDIGT | BIT_ALPHA | BIT_GRAPH | BIT_LOWER,\r
+}\r
+\r
+class Group:\r
+ def __init__(self, start, flags, upper_delta, lower_delta):\r
+ self.start = start\r
+ self.flags = flags\r
+ self.upper_delta = upper_delta\r
+ self.lower_delta = lower_delta\r
+ self.chars = []\r
+\r
+ def add_char(self, num, label):\r
+ self.chars.append((num, label))\r
+\r
+ def write_to_file(self, f):\r
+ for char in self.chars:\r
+ f.write("// %x %s\n" % char)\r
+ f.write(" { 0x%X, \t0x%X, \t0x%X, \t%d, \t%d },\n" %\r
+ (self.start, len(self.chars), self.flags, self.lower_delta, self.upper_delta))\r
+\r
+ def next(self):\r
+ return self.start + len(self.chars)\r
+\r
+groups = []\r
+\r
+def add_char(num, upper, lower, bits, label):\r
+ upper_delta = upper - num\r
+ lower_delta = lower - num\r
+\r
+ if len(groups) != 0:\r
+ cur = groups[-1]\r
+ if num == cur.next() and cur.flags == bits and \\r
+ cur.upper_delta == upper_delta and \\r
+ cur.lower_delta == lower_delta:\r
+ cur.add_char(num, label)\r
+ return\r
+\r
+ g = Group(num, bits, upper_delta, lower_delta)\r
+ g.add_char(num, label)\r
+ groups.append(g)\r
+\r
in_file = open('UnicodeData.txt', 'r')\r
out_file = open('_PDCLIB_unicodedata.c', 'w')\r
try:\r
+ for line in in_file:\r
+ (num_hex, name, category, combining_class, bidi_class, decomposition,\r
+ numeric_type, numeric_digit, numeric_value, mirrored, u1name, iso_com, \r
+ upper_case_hex, lower_case_hex, title_case_hex) = line.split(";")\r
+\r
+ num = int(num_hex, 16)\r
+ upper_case = int(upper_case_hex, 16) if len(upper_case_hex) else num\r
+ lower_case = int(lower_case_hex, 16) if len(lower_case_hex) else num\r
+ bits = special.get(num, categories.get(category, 0))\r
+\r
+ if upper_case == 0 and lower_case == 0 and bits == 0:\r
+ continue\r
+\r
+ add_char(num, upper_case, lower_case, bits, name)\r
+\r
out_file.write("""\r
/* Unicode Character Information ** AUTOMATICALLY GENERATED FILE **\r
*\r
* in Exhibit 1 of the Unicode Terms of Use, found at\r
* http://www.unicode.org/copyright.html#Exhibit1\r
*/\r
+ #ifndef REGTEST\r
#include <_PDCLIB_locale.h>\r
\r
- _PDCLIB_wctype_t _PDCLIB_wctype[] = {\r
-// { value,\tflags,\tlower,\tupper\t}, // name\r
+const _PDCLIB_wcinfo_t _PDCLIB_wcinfo[] = {\r
+// { value, \tlength, \tflags,\tlower,\tupper\t}, // name\r
""")\r
- for line in in_file:\r
- (num_hex, name, category, combining_class, bidi_class, decomposition,\r
- numeric_type, numeric_digit, numeric_value, mirrored, u1name, iso_com, \r
- upper_case_hex, lower_case_hex, title_case_hex) = line.split(";")\r
-\r
- num = int(num_hex, 16)\r
- upper_case = int(upper_case_hex, 16) if len(upper_case_hex) else num\r
- lower_case = int(lower_case_hex, 16) if len(lower_case_hex) else num\r
- bits = categories.get(category, 0)\r
+ for g in groups:\r
+ g.write_to_file(out_file)\r
+ out_file.write('};\n\n')\r
+ out_file.write("""\r
+const size_t _PDCLIB_wcinfo_size = sizeof(_PDCLIB_wcinfo) / sizeof(_PDCLIB_wcinfo[0]);\r
+#endif\r
\r
- if upper_case == 0 and lower_case == 0 and bits == 0:\r
- continue\r
+#ifdef TEST\r
+#include <_PDCLIB_test.h>\r
+int main( void )\r
+{\r
+ return TEST_RESULTS;\r
+}\r
+#endif\r
\r
- out_file.write(" { 0x%X,\t0x%X,\t0x%X,\t0x%X }, // %s\n" % (\r
- num, bits, lower_case, upper_case, name))\r
- out_file.write('};\n\n')\r
- out_file.write('size_t _PDCLIB_wctype_size = sizeof(_PDCLIB_wctype) / sizeof(_PDCLIB_wctype[0]);\n\n')\r
+""")\r
except:\r
in_file.close()\r
out_file.close()\r
projects / pdclib.old / blobdiff