| /** |
| * @file unicode.cpp |
| * Detects, read and writes characters in the proper format. |
| * |
| * @author Ben Gardner |
| * @license GPL v2+ |
| */ |
| #include "uncrustify_types.h" |
| #include "prototypes.h" |
| #include "unc_ctype.h" |
| #include <cstring> |
| #include <cstdlib> |
| |
| |
| /** |
| * See if all characters are ASCII (0-127) |
| */ |
| static bool is_ascii(const vector<UINT8>& data, int& non_ascii_cnt, int& zero_cnt) |
| { |
| non_ascii_cnt = zero_cnt = 0; |
| for (int idx = 0; idx < (int)data.size(); idx++) |
| { |
| if (data[idx] & 0x80) |
| { |
| non_ascii_cnt++; |
| } |
| if (!data[idx]) |
| { |
| zero_cnt++; |
| } |
| } |
| return((non_ascii_cnt + zero_cnt) == 0); |
| } |
| |
| |
| /** |
| * Convert the array of bytes into an array of ints |
| */ |
| static bool decode_bytes(const vector<UINT8>& in_data, deque<int>& out_data) |
| { |
| out_data.resize(in_data.size()); |
| for (int idx = 0; idx < (int)in_data.size(); idx++) |
| { |
| out_data[idx] = in_data[idx]; |
| } |
| return true; |
| } |
| |
| |
| void encode_utf8(int ch, vector<UINT8>& res) |
| { |
| if (ch < 0) |
| { |
| /* illegal code - do not store */ |
| } |
| else if (ch < 0x80) |
| { |
| /* 0xxxxxxx */ |
| res.push_back(ch); |
| } |
| else if (ch < 0x0800) |
| { |
| /* 110xxxxx 10xxxxxx */ |
| res.push_back(0xC0 | (ch >> 6)); |
| res.push_back(0x80 | (ch & 0x3f)); |
| } |
| else if (ch < 0x10000) |
| { |
| /* 1110xxxx 10xxxxxx 10xxxxxx */ |
| res.push_back(0xE0 | (ch >> 12)); |
| res.push_back(0x80 | ((ch >> 6) & 0x3f)); |
| res.push_back(0x80 | (ch & 0x3f)); |
| } |
| else if (ch < 0x200000) |
| { |
| /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ |
| res.push_back(0xF0 | (ch >> 18)); |
| res.push_back(0x80 | ((ch >> 12) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 6) & 0x3f)); |
| res.push_back(0x80 | (ch & 0x3f)); |
| } |
| else if (ch < 0x4000000) |
| { |
| /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ |
| res.push_back(0xF8 | (ch >> 24)); |
| res.push_back(0x80 | ((ch >> 18) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 12) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 6) & 0x3f)); |
| res.push_back(0x80 | (ch & 0x3f)); |
| } |
| else /* (ch <= 0x7fffffff) */ |
| { |
| /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ |
| res.push_back(0xFC | (ch >> 30)); |
| res.push_back(0x80 | ((ch >> 24) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 18) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 12) & 0x3f)); |
| res.push_back(0x80 | ((ch >> 6) & 0x3f)); |
| res.push_back(0x80 | (ch & 0x3f)); |
| } |
| } |
| |
| |
| /** |
| * Decode UTF-8 sequences from in_data and put the chars in out_data. |
| * If there are any decoding errors, then return false. |
| */ |
| static bool decode_utf8(const vector<UINT8>& in_data, deque<int>& out_data) |
| { |
| int idx = 0; |
| int ch, tmp, cnt; |
| |
| out_data.clear(); |
| |
| /* check for UTF-8 BOM silliness and skip */ |
| if (in_data.size() >= 3) |
| { |
| if ((in_data[0] == 0xef) && |
| (in_data[1] == 0xbb) && |
| (in_data[2] == 0xbf)) |
| { |
| /* skip it */ |
| idx = 3; |
| } |
| } |
| |
| while (idx < (int)in_data.size()) |
| { |
| ch = in_data[idx++]; |
| if (ch < 0x80) /* 1-byte sequence */ |
| { |
| out_data.push_back(ch); |
| continue; |
| } |
| else if ((ch & 0xE0) == 0xC0) /* 2-byte sequence */ |
| { |
| ch &= 0x1F; |
| cnt = 1; |
| } |
| else if ((ch & 0xF0) == 0xE0) /* 3-byte sequence */ |
| { |
| ch &= 0x0F; |
| cnt = 2; |
| } |
| else if ((ch & 0xF8) == 0xF0) /* 4-byte sequence */ |
| { |
| ch &= 0x07; |
| cnt = 3; |
| } |
| else if ((ch & 0xFC) == 0xF8) /* 5-byte sequence */ |
| { |
| ch &= 0x03; |
| cnt = 4; |
| } |
| else if ((ch & 0xFE) == 0xFC) /* 6-byte sequence */ |
| { |
| ch &= 0x01; |
| cnt = 5; |
| } |
| else |
| { |
| /* invalid UTF-8 sequence */ |
| return false; |
| } |
| |
| while ((cnt-- > 0) && (idx < (int)in_data.size())) |
| { |
| tmp = in_data[idx++]; |
| if ((tmp & 0xC0) != 0x80) |
| { |
| /* invalid UTF-8 sequence */ |
| return false; |
| } |
| ch = (ch << 6) | (tmp & 0x3f); |
| } |
| if (cnt >= 0) |
| { |
| /* short UTF-8 sequence */ |
| return false; |
| } |
| out_data.push_back(ch); |
| } |
| return true; |
| } |
| |
| |
| /** |
| * Extract 2 bytes from the stream and increment idx by 2 |
| */ |
| static int get_word(const vector<UINT8>& in_data, int& idx, bool be) |
| { |
| int ch; |
| |
| if ((idx + 2) > (int)in_data.size()) |
| { |
| ch = -1; |
| } |
| else if (be) |
| { |
| ch = (in_data[idx] << 8) | in_data[idx + 1]; |
| } |
| else |
| { |
| ch = in_data[idx] | (in_data[idx + 1] << 8); |
| } |
| idx += 2; |
| return ch; |
| } |
| |
| |
| /** |
| * Deocde a UTF-16 sequence. |
| * Sets enc based on the BOM. |
| * Must have the BOM as the first two bytes. |
| */ |
| static bool decode_utf16(const vector<UINT8>& in_data, deque<int>& out_data, CharEncoding& enc) |
| { |
| out_data.clear(); |
| |
| if (in_data.size() & 1) |
| { |
| /* can't have and odd length */ |
| return false; |
| } |
| |
| if (in_data.size() < 2) |
| { |
| /* we require the BOM or at least 1 char */ |
| return false; |
| } |
| |
| int idx = 2; |
| if ((in_data[0] == 0xfe) && (in_data[1] == 0xff)) |
| { |
| enc = ENC_UTF16_BE; |
| } |
| else if ((in_data[0] == 0xff) && (in_data[1] == 0xfe)) |
| { |
| enc = ENC_UTF16_LE; |
| } |
| else |
| { |
| /* If we have a few words, we can take a guess, assuming the first few |
| * chars are ASCII */ |
| enc = ENC_ASCII; |
| idx = 0; |
| if (in_data.size() >= 6) |
| { |
| if ((in_data[0] == 0) && (in_data[2] == 0) && (in_data[4] == 0)) |
| { |
| enc = ENC_UTF16_BE; |
| } |
| else if ((in_data[1] == 0) && (in_data[3] == 0) && (in_data[5] == 0)) |
| { |
| enc = ENC_UTF16_LE; |
| } |
| } |
| if (enc == ENC_ASCII) |
| { |
| return false; |
| } |
| } |
| |
| bool be = (enc == ENC_UTF16_BE); |
| |
| while (idx < (int)in_data.size()) |
| { |
| int ch = get_word(in_data, idx, be); |
| if ((ch & 0xfc00) == 0xd800) |
| { |
| ch &= 0x3ff; |
| ch <<= 10; |
| int tmp = get_word(in_data, idx, be); |
| if ((tmp & 0xfc00) != 0xdc00) |
| { |
| return false; |
| } |
| ch |= (tmp & 0x3ff); |
| ch += 0x10000; |
| out_data.push_back(ch); |
| } |
| else if (((ch >= 0) && (ch < 0xD800)) || (ch >= 0xE000)) |
| { |
| out_data.push_back(ch); |
| } |
| else |
| { |
| /* invalid character */ |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| |
| /** |
| * Looks for the BOM of UTF-16 BE/LE and UTF-8. |
| * If found, set enc and return true. |
| * Sets enc to ENC_ASCII and returns false if not found. |
| */ |
| static bool decode_bom(const vector<UINT8>& in_data, CharEncoding& enc) |
| { |
| enc = ENC_ASCII; |
| if (in_data.size() >= 2) |
| { |
| if ((in_data[0] == 0xfe) && (in_data[1] == 0xff)) |
| { |
| enc = ENC_UTF16_BE; |
| return true; |
| } |
| else if ((in_data[0] == 0xff) && (in_data[1] == 0xfe)) |
| { |
| enc = ENC_UTF16_LE; |
| return true; |
| } |
| else if ((in_data.size() >= 3) && |
| (in_data[0] == 0xef) && |
| (in_data[1] == 0xbb) && |
| (in_data[2] == 0xbf)) |
| { |
| enc = ENC_UTF8; |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| |
| /** |
| * Figure out the encoding and convert to an int sequence |
| */ |
| bool decode_unicode(const vector<UINT8>& in_data, deque<int>& out_data, CharEncoding& enc, bool& has_bom) |
| { |
| /* check for a BOM */ |
| if (decode_bom(in_data, enc)) |
| { |
| has_bom = true; |
| if (enc == ENC_UTF8) |
| { |
| return decode_utf8(in_data, out_data); |
| } |
| else |
| { |
| return decode_utf16(in_data, out_data, enc); |
| } |
| } |
| has_bom = false; |
| |
| /* Check for simple ASCII */ |
| int non_ascii_cnt; |
| int zero_cnt; |
| if (is_ascii(in_data, non_ascii_cnt, zero_cnt)) |
| { |
| enc = ENC_ASCII; |
| return decode_bytes(in_data, out_data); |
| } |
| |
| /* There are alot of 0's in UTF-16 (~50%) */ |
| if ((zero_cnt > ((int)in_data.size() / 4)) && |
| (zero_cnt <= ((int)in_data.size() / 2))) |
| { |
| /* likely is UTF-16 */ |
| if (decode_utf16(in_data, out_data, enc)) |
| { |
| return true; |
| } |
| } |
| |
| if (decode_utf8(in_data, out_data)) |
| { |
| enc = ENC_UTF8; |
| return true; |
| } |
| |
| /* it is an unrecognized byte sequence */ |
| enc = ENC_BYTE; |
| return decode_bytes(in_data, out_data); |
| } |
| |
| |
| /** |
| * Write for ASCII and BYTE encoding |
| */ |
| static void write_byte(int ch) |
| { |
| if ((ch & 0xff) == ch) |
| { |
| if (cpd.fout) |
| { |
| fputc(ch, cpd.fout); |
| } |
| if (cpd.bout) |
| { |
| cpd.bout->push_back((UINT8)ch); |
| } |
| } |
| else |
| { |
| /* illegal code - do not store */ |
| } |
| } |
| |
| |
| /** |
| * Writes a single character to a file using UTF-8 encoding |
| */ |
| static void write_utf8(int ch) |
| { |
| vector<UINT8> vv; |
| vv.reserve(6); |
| |
| encode_utf8(ch, vv); |
| for (int idx = 0; idx < (int)vv.size(); idx++) |
| { |
| write_byte(vv[idx]); |
| } |
| } |
| |
| |
| static void write_utf16(int ch, bool be) |
| { |
| /* U+0000 to U+D7FF and U+E000 to U+FFFF */ |
| if (((ch >= 0) && (ch < 0xD800)) || ((ch >= 0xE000) && (ch < 0x10000))) |
| { |
| if (be) |
| { |
| write_byte(ch >> 8); |
| write_byte(ch & 0xff); |
| } |
| else |
| { |
| write_byte(ch & 0xff); |
| write_byte(ch >> 8); |
| } |
| } |
| else if ((ch >= 0x10000) && (ch < 0x110000)) |
| { |
| int v1 = ch - 0x10000; |
| int w1 = 0xD800 + (v1 >> 10); |
| int w2 = 0xDC00 + (v1 & 0x3ff); |
| if (be) |
| { |
| write_byte(w1 >> 8); |
| write_byte(w1 & 0xff); |
| write_byte(w2 >> 8); |
| write_byte(w2 & 0xff); |
| } |
| else |
| { |
| write_byte(w1 & 0xff); |
| write_byte(w1 >> 8); |
| write_byte(w2 & 0xff); |
| write_byte(w2 >> 8); |
| } |
| } |
| else |
| { |
| /* illegal code - do not store */ |
| } |
| } |
| |
| |
| void write_bom() |
| { |
| switch (cpd.enc) |
| { |
| case ENC_UTF8: |
| write_byte(0xef); |
| write_byte(0xbb); |
| write_byte(0xbf); |
| break; |
| |
| case ENC_UTF16_LE: |
| write_utf16(0xfeff, false); |
| break; |
| |
| case ENC_UTF16_BE: |
| write_utf16(0xfeff, true); |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| |
| /** |
| * @param ch the 31-bit char value |
| */ |
| void write_char(int ch) |
| { |
| if (ch >= 0) |
| { |
| switch (cpd.enc) |
| { |
| case ENC_BYTE: |
| write_byte(ch & 0xff); |
| break; |
| |
| case ENC_ASCII: |
| default: |
| write_byte(ch); |
| break; |
| |
| case ENC_UTF8: |
| write_utf8(ch); |
| break; |
| |
| case ENC_UTF16_LE: |
| write_utf16(ch, false); |
| break; |
| |
| case ENC_UTF16_BE: |
| write_utf16(ch, true); |
| break; |
| } |
| } |
| } |
| |
| |
| void write_string(const unc_text& text) |
| { |
| for (int idx = 0; idx < (int)text.size(); idx++) |
| { |
| write_char(text[idx]); |
| } |
| } |
