PDCLIB-3 correct classification of space characters

diff --git a/functions/locale/UnicodeData.py b/functions/locale/UnicodeData.py
index e31ec2e29f02fb2b5022a23a7cb0918273b20e40..6fe74cb8d90ee263a1681274bf2faa5fc2255548 100644 (file)
--- a/functions/locale/UnicodeData.py
+++ b/functions/locale/UnicodeData.py
@@ -49,14 +49,18 @@ categories = {
     'Sc': BIT_GRAPH,                            # Currency symbol\r
     'Sk': BIT_GRAPH,                            # Non-letterlike modifier symbol\r
     'So': BIT_GRAPH,                            # Other symbol\r
-    'Zs': BIT_SPACE | BIT_GRAPH | BIT_BLANK,    # Non-zero-width space character\r
-    'Zl': BIT_SPACE | BIT_GRAPH,                # Line separator\r
-    'Zp': BIT_SPACE | BIT_GRAPH,                # Paragraph separator\r
+    'Zs': BIT_SPACE,                            # Non-zero-width space character\r
+    'Zl': BIT_SPACE,                            # Line separator\r
+    'Zp': BIT_SPACE,                            # Paragraph separator\r
     'Cc': BIT_CNTRL,                            # C0/C1 control codes\r
 }\r
 \r
 # Characters with special properties\r
 special = {\r
+    # Blank characters\r
+    0x0020: BIT_SPACE | BIT_BLANK, # space\r
+    0x0009: BIT_SPACE | BIT_BLANK, # tab\r
+\r
     # Digits\r
     0x0030: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r
     0x0031: BIT_XDIGT | BIT_DIGIT | BIT_GRAPH,\r

diff --git a/functions/locale/_PDCLIB_unicodedata.c b/functions/locale/_PDCLIB_unicodedata.c
index 928226ce48e6dc95110f46b562191bde522b857d..dbfc02a8674bd687cbcf24b349b1fe8a906ebf70 100644 (file)
--- a/functions/locale/_PDCLIB_unicodedata.c
+++ b/functions/locale/_PDCLIB_unicodedata.c
@@ -24,7 +24,7 @@
     { 0x6,     0x4,    0x6,    0x6 }, // <control>\r
     { 0x7,     0x4,    0x7,    0x7 }, // <control>\r
     { 0x8,     0x4,    0x8,    0x8 }, // <control>\r
-    { 0x9,     0x4,    0x9,    0x9 }, // <control>\r
+    { 0x9,     0x22,   0x9,    0x9 }, // <control>\r
     { 0xA,     0x4,    0xA,    0xA }, // <control>\r
     { 0xB,     0x4,    0xB,    0xB }, // <control>\r
     { 0xC,     0x4,    0xC,    0xC }, // <control>\r
@@ -47,7 +47,7 @@
     { 0x1D,    0x4,    0x1D,   0x1D }, // <control>\r
     { 0x1E,    0x4,    0x1E,   0x1E }, // <control>\r
     { 0x1F,    0x4,    0x1F,   0x1F }, // <control>\r
-    { 0x20,    0x2A,   0x20,   0x20 }, // SPACE\r
+    { 0x20,    0x22,   0x20,   0x20 }, // SPACE\r
     { 0x21,    0x18,   0x21,   0x21 }, // EXCLAMATION MARK\r
     { 0x22,    0x18,   0x22,   0x22 }, // QUOTATION MARK\r
     { 0x23,    0x18,   0x23,   0x23 }, // NUMBER SIGN\r
@@ -175,7 +175,7 @@
     { 0x9D,    0x4,    0x9D,   0x9D }, // <control>\r
     { 0x9E,    0x4,    0x9E,   0x9E }, // <control>\r
     { 0x9F,    0x4,    0x9F,   0x9F }, // <control>\r
-    { 0xA0,    0x2A,   0xA0,   0xA0 }, // NO-BREAK SPACE\r
+    { 0xA0,    0x20,   0xA0,   0xA0 }, // NO-BREAK SPACE\r
     { 0xA1,    0x18,   0xA1,   0xA1 }, // INVERTED EXCLAMATION MARK\r
     { 0xA2,    0x8,    0xA2,   0xA2 }, // CENT SIGN\r
     { 0xA3,    0x8,    0xA3,   0xA3 }, // POUND SIGN\r
@@ -4994,7 +4994,7 @@
     { 0x167D,  0x9,    0x167D, 0x167D }, // CANADIAN SYLLABICS WOODS-CREE THWAA\r
     { 0x167E,  0x9,    0x167E, 0x167E }, // CANADIAN SYLLABICS WOODS-CREE FINAL TH\r
     { 0x167F,  0x9,    0x167F, 0x167F }, // CANADIAN SYLLABICS BLACKFOOT W\r
-    { 0x1680,  0x2A,   0x1680, 0x1680 }, // OGHAM SPACE MARK\r
+    { 0x1680,  0x20,   0x1680, 0x1680 }, // OGHAM SPACE MARK\r
     { 0x1681,  0x9,    0x1681, 0x1681 }, // OGHAM LETTER BEITH\r
     { 0x1682,  0x9,    0x1682, 0x1682 }, // OGHAM LETTER LUIS\r
     { 0x1683,  0x9,    0x1683, 0x1683 }, // OGHAM LETTER FEARN\r
@@ -5313,7 +5313,7 @@
     { 0x180B,  0x0,    0x180B, 0x180B }, // MONGOLIAN FREE VARIATION SELECTOR ONE\r
     { 0x180C,  0x0,    0x180C, 0x180C }, // MONGOLIAN FREE VARIATION SELECTOR TWO\r
     { 0x180D,  0x0,    0x180D, 0x180D }, // MONGOLIAN FREE VARIATION SELECTOR THREE\r
-    { 0x180E,  0x2A,   0x180E, 0x180E }, // MONGOLIAN VOWEL SEPARATOR\r
+    { 0x180E,  0x20,   0x180E, 0x180E }, // MONGOLIAN VOWEL SEPARATOR\r
     { 0x1810,  0x108,  0x1810, 0x1810 }, // MONGOLIAN DIGIT ZERO\r
     { 0x1811,  0x108,  0x1811, 0x1811 }, // MONGOLIAN DIGIT ONE\r
     { 0x1812,  0x108,  0x1812, 0x1812 }, // MONGOLIAN DIGIT TWO\r
@@ -7032,17 +7032,17 @@
     { 0x1FFC,  0x89,   0x1FF3, 0x1FFC }, // GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI\r
     { 0x1FFD,  0x8,    0x1FFD, 0x1FFD }, // GREEK OXIA\r
     { 0x1FFE,  0x8,    0x1FFE, 0x1FFE }, // GREEK DASIA\r
-    { 0x2000,  0x2A,   0x2000, 0x2000 }, // EN QUAD\r
-    { 0x2001,  0x2A,   0x2001, 0x2001 }, // EM QUAD\r
-    { 0x2002,  0x2A,   0x2002, 0x2002 }, // EN SPACE\r
-    { 0x2003,  0x2A,   0x2003, 0x2003 }, // EM SPACE\r
-    { 0x2004,  0x2A,   0x2004, 0x2004 }, // THREE-PER-EM SPACE\r
-    { 0x2005,  0x2A,   0x2005, 0x2005 }, // FOUR-PER-EM SPACE\r
-    { 0x2006,  0x2A,   0x2006, 0x2006 }, // SIX-PER-EM SPACE\r
-    { 0x2007,  0x2A,   0x2007, 0x2007 }, // FIGURE SPACE\r
-    { 0x2008,  0x2A,   0x2008, 0x2008 }, // PUNCTUATION SPACE\r
-    { 0x2009,  0x2A,   0x2009, 0x2009 }, // THIN SPACE\r
-    { 0x200A,  0x2A,   0x200A, 0x200A }, // HAIR SPACE\r
+    { 0x2000,  0x20,   0x2000, 0x2000 }, // EN QUAD\r
+    { 0x2001,  0x20,   0x2001, 0x2001 }, // EM QUAD\r
+    { 0x2002,  0x20,   0x2002, 0x2002 }, // EN SPACE\r
+    { 0x2003,  0x20,   0x2003, 0x2003 }, // EM SPACE\r
+    { 0x2004,  0x20,   0x2004, 0x2004 }, // THREE-PER-EM SPACE\r
+    { 0x2005,  0x20,   0x2005, 0x2005 }, // FOUR-PER-EM SPACE\r
+    { 0x2006,  0x20,   0x2006, 0x2006 }, // SIX-PER-EM SPACE\r
+    { 0x2007,  0x20,   0x2007, 0x2007 }, // FIGURE SPACE\r
+    { 0x2008,  0x20,   0x2008, 0x2008 }, // PUNCTUATION SPACE\r
+    { 0x2009,  0x20,   0x2009, 0x2009 }, // THIN SPACE\r
+    { 0x200A,  0x20,   0x200A, 0x200A }, // HAIR SPACE\r
     { 0x200B,  0x0,    0x200B, 0x200B }, // ZERO WIDTH SPACE\r
     { 0x200C,  0x0,    0x200C, 0x200C }, // ZERO WIDTH NON-JOINER\r
     { 0x200D,  0x0,    0x200D, 0x200D }, // ZERO WIDTH JOINER\r
@@ -7072,14 +7072,14 @@
     { 0x2025,  0x18,   0x2025, 0x2025 }, // TWO DOT LEADER\r
     { 0x2026,  0x18,   0x2026, 0x2026 }, // HORIZONTAL ELLIPSIS\r
     { 0x2027,  0x18,   0x2027, 0x2027 }, // HYPHENATION POINT\r
-    { 0x2028,  0x28,   0x2028, 0x2028 }, // LINE SEPARATOR\r
-    { 0x2029,  0x28,   0x2029, 0x2029 }, // PARAGRAPH SEPARATOR\r
+    { 0x2028,  0x20,   0x2028, 0x2028 }, // LINE SEPARATOR\r
+    { 0x2029,  0x20,   0x2029, 0x2029 }, // PARAGRAPH SEPARATOR\r
     { 0x202A,  0x0,    0x202A, 0x202A }, // LEFT-TO-RIGHT EMBEDDING\r
     { 0x202B,  0x0,    0x202B, 0x202B }, // RIGHT-TO-LEFT EMBEDDING\r
     { 0x202C,  0x0,    0x202C, 0x202C }, // POP DIRECTIONAL FORMATTING\r
     { 0x202D,  0x0,    0x202D, 0x202D }, // LEFT-TO-RIGHT OVERRIDE\r
     { 0x202E,  0x0,    0x202E, 0x202E }, // RIGHT-TO-LEFT OVERRIDE\r
-    { 0x202F,  0x2A,   0x202F, 0x202F }, // NARROW NO-BREAK SPACE\r
+    { 0x202F,  0x20,   0x202F, 0x202F }, // NARROW NO-BREAK SPACE\r
     { 0x2030,  0x18,   0x2030, 0x2030 }, // PER MILLE SIGN\r
     { 0x2031,  0x18,   0x2031, 0x2031 }, // PER TEN THOUSAND SIGN\r
     { 0x2032,  0x18,   0x2032, 0x2032 }, // PRIME\r
@@ -7127,7 +7127,7 @@
     { 0x205C,  0x18,   0x205C, 0x205C }, // DOTTED CROSS\r
     { 0x205D,  0x18,   0x205D, 0x205D }, // TRICOLON\r
     { 0x205E,  0x18,   0x205E, 0x205E }, // VERTICAL FOUR DOTS\r
-    { 0x205F,  0x2A,   0x205F, 0x205F }, // MEDIUM MATHEMATICAL SPACE\r
+    { 0x205F,  0x20,   0x205F, 0x205F }, // MEDIUM MATHEMATICAL SPACE\r
     { 0x2060,  0x0,    0x2060, 0x2060 }, // WORD JOINER\r
     { 0x2061,  0x0,    0x2061, 0x2061 }, // FUNCTION APPLICATION\r
     { 0x2062,  0x0,    0x2062, 0x2062 }, // INVISIBLE TIMES\r
@@ -10683,7 +10683,7 @@
     { 0x2FF9,  0x8,    0x2FF9, 0x2FF9 }, // IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM UPPER RIGHT\r
     { 0x2FFA,  0x8,    0x2FFA, 0x2FFA }, // IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER LEFT\r
     { 0x2FFB,  0x8,    0x2FFB, 0x2FFB }, // IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID\r
-    { 0x3000,  0x2A,   0x3000, 0x3000 }, // IDEOGRAPHIC SPACE\r
+    { 0x3000,  0x20,   0x3000, 0x3000 }, // IDEOGRAPHIC SPACE\r
     { 0x3001,  0x18,   0x3001, 0x3001 }, // IDEOGRAPHIC COMMA\r
     { 0x3002,  0x18,   0x3002, 0x3002 }, // IDEOGRAPHIC FULL STOP\r
     { 0x3003,  0x18,   0x3003, 0x3003 }, // DITTO MARK\r