2 # -*- coding: <encoding name> -*-
\r
3 # Unicode Data Converter
\r
5 # This file is part of the Public Domain C Library (PDCLib).
\r
6 # Permission is granted to use, modify, and / or redistribute at will.
\r
8 Converts the character information provdied by Unicode in the UnicodeData.txt
\r
9 file from the Unicode character database into a table for use by PDCLib.
\r
11 Usage: Download the UnicodeData.txt file to the same directory as this script
\r
12 and then run it. Both Python 2 and 3 are supported.
\r
14 Download the data from
\r
15 ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt
\r
19 # MUST BE KEPT SYNCHRONIZED WITH _PDCLIB_locale.h
\r
30 # Category to bitfield mapping
\r
32 'Lu': BIT_ALPHA | BIT_GRAPH | BIT_UPPER, # Uppercase
\r
33 'Ll': BIT_ALPHA | BIT_GRAPH | BIT_LOWER, # Lowercase
\r
34 'Lt': BIT_ALPHA | BIT_GRAPH | BIT_UPPER, # Title case. Upper?
\r
35 'Lm': BIT_ALPHA | BIT_GRAPH, # Modifier. Case?
\r
36 'Lo': BIT_ALPHA | BIT_GRAPH, # "Other" letter (e.g. Ideograph)
\r
37 'Nd': BIT_DIGIT | BIT_GRAPH, # Decimal digit
\r
38 'Nl': BIT_GRAPH, # Letter-like numeric character
\r
39 'No': BIT_GRAPH, # Other numeric
\r
40 'Pc': BIT_PUNCT | BIT_GRAPH, # Connecting punctuation
\r
41 'Pd': BIT_PUNCT | BIT_GRAPH, # Dash punctuation
\r
42 'Ps': BIT_PUNCT | BIT_GRAPH, # Opening punctuation
\r
43 'Pe': BIT_PUNCT | BIT_GRAPH, # Closing punctuation
\r
44 'Pi': BIT_PUNCT | BIT_GRAPH, # Opening quote
\r
45 'Pf': BIT_PUNCT | BIT_GRAPH, # Closing quote
\r
46 'Po': BIT_PUNCT | BIT_GRAPH, # Other punctuation
\r
47 'Sm': BIT_GRAPH, # Mathematical symbol
\r
48 'Sc': BIT_GRAPH, # Currency symbol
\r
49 'Sk': BIT_GRAPH, # Non-letterlike modifier symbol
\r
50 'So': BIT_GRAPH, # Other symbol
\r
51 'Zs': BIT_SPACE | BIT_GRAPH | BIT_BLANK, # Non-zero-width space character
\r
52 'Zl': BIT_SPACE | BIT_GRAPH, # Line separator
\r
53 'Zp': BIT_SPACE | BIT_GRAPH, # Paragraph separator
\r
54 'Cc': BIT_CNTRL, # C0/C1 control codes
\r
57 in_file = open('UnicodeData.txt', 'r')
\r
58 out_file = open('_PDCLIB_unicodedata.c', 'w')
\r
61 /* Unicode Character Information ** AUTOMATICALLY GENERATED FILE **
\r
63 * This file is part of the PDCLib public domain C Library, but is automatically
\r
64 * generated from the Unicode character data information file found at
\r
65 * ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt
\r
67 * As a result, the licensing that applies to that file also applies to this
\r
68 * file. The licensing which applies to the Unicode character data can be found
\r
69 * in Exhibit 1 of the Unicode Terms of Use, found at
\r
70 * http://www.unicode.org/copyright.html#Exhibit1
\r
73 #include <_PDCLIB_locale.h>
\r
75 _PDCLIB_wcinfo_t _PDCLIB_wcinfo[] = {
\r
76 // { value,\tflags,\tlower,\tupper\t}, // name
\r
78 for line in in_file:
\r
79 (num_hex, name, category, combining_class, bidi_class, decomposition,
\r
80 numeric_type, numeric_digit, numeric_value, mirrored, u1name, iso_com,
\r
81 upper_case_hex, lower_case_hex, title_case_hex) = line.split(";")
\r
83 num = int(num_hex, 16)
\r
84 upper_case = int(upper_case_hex, 16) if len(upper_case_hex) else num
\r
85 lower_case = int(lower_case_hex, 16) if len(lower_case_hex) else num
\r
86 bits = categories.get(category, 0)
\r
88 if upper_case == 0 and lower_case == 0 and bits == 0:
\r
91 out_file.write(" { 0x%X,\t0x%X,\t0x%X,\t0x%X }, // %s\n" % (
\r
92 num, bits, lower_case, upper_case, name))
\r
93 out_file.write('};\n\n')
\r
95 size_t _PDCLIB_wcinfo_size = sizeof(_PDCLIB_wcinfo) / sizeof(_PDCLIB_wcinfo[0]);
\r
99 #include <_PDCLIB_test.h>
\r
102 return TEST_RESULTS;
\r
110 os.remove('_PDCLIB_unicodedata.c')
\r