+++ /dev/null
-/* UTF-8 codec
-
- This file is part of the Public Domain C Library (PDCLib).
- Permission is granted to use, modify, and / or redistribute at will.
-*/
-
-#ifndef REGTEST
-#include <stdbool.h>
-#include <stdint.h>
-#include <uchar.h>
-#include <assert.h>
-#include "_PDCLIB_encoding.h"
-
-/* Use of the mbstate:
- *
- * _StUC[0] is the current decoding state
- * _St32[1] is the character accumulated so far
- */
-
-static bool utf8_mbsinit( const mbstate_t *p_s )
-{ return p_s->_StUC[0] == 0; }
-
-enum {
- DecStart = 0,
-
- Dec2B2,
-
- Dec3B2,
- Dec3B3,
-
- Dec4B2,
- Dec4B3,
- Dec4B4
-};
-
-#define state (p_s->_StUC[0])
-#define accum (p_s->_St32[1])
-
-#define START_CONVERSION \
- bool result = true; \
-
-#define END_CONVERSION \
-end_conversion: \
- return result
-
-#define FINISH(_r) do { \
- result = (_r); \
- goto end_conversion; \
-} while(0)
-
-#define OUT32(_c) do { \
- if(p_outbuf) \
- (*((*p_outbuf)++)) = (_c); \
- (*p_outsz)--; \
- _PDCLIB_UNDEFINED(accum); \
- state = DecStart; \
-} while(0)
-
-#define CHECK_CONTINUATION \
- do { if((c & 0xC0) != 0x80) return false; } while(0)
-
-static bool utf8toc32(
- char32_t *restrict *restrict p_outbuf,
- size_t *restrict p_outsz,
- const char *restrict *restrict p_inbuf,
- size_t *restrict p_insz,
- mbstate_t *restrict p_s
-)
-{
- START_CONVERSION
- while(*p_outsz && *p_insz) {
- unsigned char c = **p_inbuf;
- char32_t c32;
- switch(state) {
- case DecStart:
- // 1 byte
- if(c <= 0x7F) {
- OUT32(c);
- } else if(c <= 0xDF) {
- accum = (c & 0x1F) << 6;
- state = Dec2B2;
- } else if(c <= 0xEF) {
- accum = (c & 0x0F) << 12;
- state = Dec3B2;
- } else if(c <= 0xF4) {
- accum = (c & 0x07) << 18;
- state = Dec4B2;
- } else {
- // 5+byte sequence illegal
- FINISH(false);
- }
- break;
-
- case Dec2B2:
- CHECK_CONTINUATION;
-
- c32 = accum | (c & 0x3F);
-
- // Overlong sequence (e.g. NUL injection)
- if(c32 <= 0x7F)
- FINISH(false);
-
- OUT32(c32);
- break;
-
- case Dec3B2:
- CHECK_CONTINUATION;
- accum |= (c & 0x3F) << 6;
- state = Dec3B3;
- break;
-
- case Dec3B3:
- CHECK_CONTINUATION;
-
- c32 = accum | (c & 0x3F);
-
- // Overlong
- if(c32 <= 0x07FF)
- FINISH(false);
-
- // Surrogate
- if(c32 >= 0xD800 && c32 <= 0xDFFF)
- FINISH(false);
-
- OUT32(c32);
- break;
-
- case Dec4B2:
- CHECK_CONTINUATION;
- accum |= (c & 0x3F) << 12;
- state = Dec4B3;
- break;
-
- case Dec4B3:
- CHECK_CONTINUATION;
- accum |= (c & 0x3F) << 6;
- state = Dec4B4;
- break;
-
- case Dec4B4:
- CHECK_CONTINUATION;
-
- c32 = accum | (c & 0x3F);
-
- // Overlong
- if(c32 <= 0xFFFF) FINISH(false);
-
- // Not in Unicode
- if(c32 > 0x10FFFF) FINISH(false);
-
- OUT32(c32);
- break;
-
- default:
- assert(!"Invalid state");
- }
-
- (*p_inbuf)++;
- (*p_insz)--;
- }
- END_CONVERSION;
-}
-
-enum {
- EncStart = 0,
- Enc1R,
- Enc2R,
- Enc3R,
-};
-
-static bool c32toutf8(
- char *restrict *restrict p_outbuf,
- size_t *restrict p_outsz,
- const char32_t *restrict *restrict p_inbuf,
- size_t *restrict p_insz,
- mbstate_t *restrict p_s
-)
-{
- START_CONVERSION
- while(*p_outsz) {
- unsigned char outc = 0;
- switch(state) {
- case Enc3R:
- outc = 0x80 | ((accum >> 12) & 0x3F);
- state = Enc2R;
- break;
-
- case Enc2R:
- outc = 0x80 | ((accum >> 6) & 0x3F);
- state = Enc1R;
- break;
-
- case Enc1R:
- outc = 0x80 | (accum & 0x3F);
- state = EncStart;
- _PDCLIB_UNDEFINED(accum);
- break;
-
- case EncStart:
- if(*p_insz == 0)
- FINISH(true);
-
- accum = **p_inbuf;
- (*p_inbuf)++;
- (*p_insz)--;
-
- if(accum <= 0x7F) {
- outc = accum;
- state = EncStart;
- _PDCLIB_UNDEFINED(accum);
- } else if(accum <= 0x7FF) {
- outc = 0xC0 | (accum >> 6);
- state = Enc1R;
- } else if(accum <= 0xFFFF) {
- outc = 0xE0 | (accum >> 12);
- state = Enc2R;
- } else if(accum <= 0x10FFFF) {
- outc = 0xF0 | (accum >> 18);
- state = Enc3R;
- } else {
- FINISH(false);
- }
- break;
- }
-
- if(p_outbuf) {
- **p_outbuf = outc;
- (*p_outbuf)++;
- }
- (*p_outsz)--;
- }
- END_CONVERSION;
-}
-
-const struct _PDCLIB_charcodec_t _PDCLIB_utf8_codec = {
- .__mbsinit = utf8_mbsinit,
- .__mbstoc32s = utf8toc32,
- .__c32stombs = c32toutf8,
- .__mb_max = 4,
-};
-
-#endif
-
-#ifdef TEST
-#include "_PDCLIB_test.h"
-
-int main( void )
-{
-#ifndef REGTEST
- // Valid conversion & back
-
- static const char* input = "abcde" "\xDF\xBF" "\xEF\xBF\xBF"
- "\xF4\x8F\xBF\xBF";
-
- char32_t c32out[8];
-
- char32_t *c32ptr = &c32out[0];
- size_t c32rem = 8;
- const char *chrptr = (char*) &input[0];
- size_t chrrem = strlen(input);
- mbstate_t mbs = { 0 };
-
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs));
- TESTCASE(c32rem == 0);
- TESTCASE(chrrem == 0);
- TESTCASE(c32ptr == &c32out[8]);
- TESTCASE(chrptr == &input[strlen(input)]);
- TESTCASE(c32out[0] == 'a' && c32out[1] == 'b' && c32out[2] == 'c' &&
- c32out[3] == 'd' && c32out[4] == 'e' && c32out[5] == 0x7FF &&
- c32out[6] == 0xFFFF && c32out[7] == 0x10FFFF);
-
- char chrout[strlen(input)];
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &chrout[0];
- chrrem = strlen(input);
- TESTCASE(c32toutf8(&chrptr, &chrrem, &c32ptr, &c32rem, &mbs));
- TESTCASE(c32rem == 0);
- TESTCASE(chrrem == 0);
- TESTCASE(c32ptr == &c32out[8]);
- TESTCASE(chrptr == &chrout[strlen(input)]);
- TESTCASE(memcmp(chrout, input, strlen(input)) == 0);
-
- // Multi-part conversion
- static const char* mpinput = "\xDF\xBF";
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &mpinput[0];
- chrrem = 1;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs));
- TESTCASE(c32ptr == &c32out[0]);
- TESTCASE(c32rem == 8);
- TESTCASE(chrptr == &mpinput[1]);
- TESTCASE(chrrem == 0);
- chrrem = 1;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs));
- TESTCASE(c32ptr == &c32out[1]);
- TESTCASE(c32rem == 7);
- TESTCASE(chrptr == &mpinput[2]);
- TESTCASE(chrrem == 0);
-
- // Invalid conversions
-
- // Overlong nuls
- const char* nul2 = "\xC0\x80";
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &nul2[0];
- chrrem = 2;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs) == false);
- memset(&mbs, 0, sizeof mbs);
- const char* nul3 = "\xE0\x80\x80";
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &nul3[0];
- chrrem = 3;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs) == false);
- memset(&mbs, 0, sizeof mbs);
- const char* nul4 = "\xF0\x80\x80\x80";
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &nul4[0];
- chrrem = 4;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs) == false);
-
- // Starting on a continuation
- const char* cont = "\x80";
- c32ptr = &c32out[0];
- c32rem = 8;
- chrptr = &cont[0];
- chrrem = 1;
- TESTCASE(utf8toc32(&c32ptr, &c32rem, &chrptr, &chrrem, &mbs) == false);
-#endif
- return TEST_RESULTS;
-}
-
-#endif
-
projects / pdclib / blobdiff