OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 6
Code Issues Packages Projects Releases Wiki Activity

Update Unicode to 15.0 

Unicode 15.0 adds 4,489 characters, for a total of 149,186 characters.
These additions include 2 new scripts along with 20 new emoji characters,
and 4,193 CJK ideographs.

This changes modify most existing tables including
 - XID_Start/XID_Continue in Clang
 - The character name database (used by \N{} in Clang)
 - The list of formattable/printable codepoints
 - The case folding algorithm (which we had not updated since Unicode 9)
 - The list of nonspacing/enclosing marks used by the column width
   computation algorithm. The rest of the column width algorithm
   is not updated.

Reviewed By: tahonermann

Differential Revision: https://reviews.llvm.org/D133807
This commit is contained in:
Corentin Jabot 2022-09-13 20:09:56 +02:00
parent 3b8ec0fde5
commit c932cef32a
9 changed files with 21164 additions and 20852 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
clang/docs/ReleaseNotes.rst
View File

@ -220,6 +220,9 @@ Non-comprehensive list of changes in this release
temporarily reversed with ``-Xclang temporarily reversed with ``-Xclang
-fno-modules-validate-textual-header-includes``, but this flag will be -fno-modules-validate-textual-header-includes``, but this flag will be
removed in a future Clang release. removed in a future Clang release.
- Unicode support has been updated to support Unicode 15.0.
New unicode codepoints are supported as appropriate in diagnostics,
C and C++ identifiers, and escape sequences.
New Compiler Flags New Compiler Flags
------------------ ------------------

273
clang/lib/Lex/UnicodeCharSets.h
View File

@ -10,7 +10,7 @@
#include "llvm/Support/UnicodeCharRanges.h" #include "llvm/Support/UnicodeCharRanges.h"
// Unicode 14 XID_Start // Unicode 15.0 XID_Start
static const llvm::sys::UnicodeCharRange XIDStartRanges[] = { static const llvm::sys::UnicodeCharRange XIDStartRanges[] = {
{0x0041, 0x005A}, {0x0061, 0x007A}, {0x00AA, 0x00AA}, {0x0041, 0x005A}, {0x0061, 0x007A}, {0x00AA, 0x00AA},
{0x00B5, 0x00B5}, {0x00BA, 0x00BA}, {0x00C0, 0x00D6}, {0x00B5, 0x00B5}, {0x00BA, 0x00BA}, {0x00C0, 0x00D6},
@ -170,69 +170,72 @@ static const llvm::sys::UnicodeCharRange XIDStartRanges[] = {
{0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172},
{0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4},
{0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211},
{0x11213, 0x1122B}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286},
{0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D},
{0x112B0, 0x112DE}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x11305, 0x1130C},
{0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330},
{0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D},
{0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434},
{0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF},
{0x114C7, 0x114C7}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE},
{0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644},
{0x116B8, 0x116B8}, {0x11700, 0x1171A}, {0x11740, 0x11746}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x11700, 0x1171A},
{0x11800, 0x1182B}, {0x118A0, 0x118DF}, {0x118FF, 0x11906}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118DF},
{0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913},
{0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F},
{0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0},
{0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00},
{0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50},
{0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8},
{0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40},
{0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09},
{0x11D46, 0x11D46}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65},
{0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98},
{0x11FB0, 0x11FB0}, {0x12000, 0x12399}, {0x12400, 0x1246E}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, {0x11F04, 0x11F10},
{0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342E}, {0x11F12, 0x11F33}, {0x11FB0, 0x11FB0}, {0x12000, 0x12399},
{0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0},
{0x16A70, 0x16ABE}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x13000, 0x1342F}, {0x13441, 0x13446}, {0x14400, 0x14646},
{0x16B40, 0x16B43}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A70, 0x16ABE},
{0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43},
{0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E7F},
{0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F},
{0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7},
{0x1B000, 0x1B122}, {0x1B150, 0x1B152}, {0x1B164, 0x1B167}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3},
{0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122},
{0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155},
{0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A},
{0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99},
{0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F},
{0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC},
{0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3},
{0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514},
{0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E},
{0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550},
{0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA},
{0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1DF00, 0x1DF1E}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734},
{0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788},
{0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB},
{0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D},
{0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E14E, 0x1E14E}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E14E, 0x1E14E},
{0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E7E0, 0x1E7E6}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E4D0, 0x1E4EB},
{0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE},
{0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943},
{0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F},
{0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27},
{0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39},
{0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47},
{0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F},
{0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57},
{0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D},
{0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64},
{0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77},
{0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89},
{0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9},
{0x20000, 0x2A6DF}, {0x2A700, 0x2B738}, {0x2B740, 0x2B81D}, {0x1EEAB, 0x1EEBB}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739},
{0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0},
{0x30000, 0x3134A}}; {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}};
// Unicode 14 XID_Continue, excluding XID_Start // Unicode 15.0 XID_Continue, excluding XID_Start
// The Unicode Property XID_Continue is a super set of XID_Start. // The Unicode Property XID_Continue is a super set of XID_Start.
// To save Space, the table below only contains the codepoints // To save Space, the table below only contains the codepoints
// that are not also in XID_Start. // that are not also in XID_Start.
@ -268,64 +271,65 @@ static const llvm::sys::UnicodeCharRange XIDContinueRanges[] = {
{0x0C66, 0x0C6F}, {0x0C81, 0x0C83}, {0x0CBC, 0x0CBC}, {0x0C66, 0x0C6F}, {0x0C81, 0x0C83}, {0x0CBC, 0x0CBC},
{0x0CBE, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, {0x0CBE, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD},
{0x0CD5, 0x0CD6}, {0x0CE2, 0x0CE3}, {0x0CE6, 0x0CEF}, {0x0CD5, 0x0CD6}, {0x0CE2, 0x0CE3}, {0x0CE6, 0x0CEF},
{0x0D00, 0x0D03}, {0x0D3B, 0x0D3C}, {0x0D3E, 0x0D44}, {0x0CF3, 0x0CF3}, {0x0D00, 0x0D03}, {0x0D3B, 0x0D3C},
{0x0D46, 0x0D48}, {0x0D4A, 0x0D4D}, {0x0D57, 0x0D57}, {0x0D3E, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4D},
{0x0D62, 0x0D63}, {0x0D66, 0x0D6F}, {0x0D81, 0x0D83}, {0x0D57, 0x0D57}, {0x0D62, 0x0D63}, {0x0D66, 0x0D6F},
{0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0D81, 0x0D83}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4},
{0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF3}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF},
{0x0E31, 0x0E31}, {0x0E33, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0DF2, 0x0DF3}, {0x0E31, 0x0E31}, {0x0E33, 0x0E3A},
{0x0E50, 0x0E59}, {0x0EB1, 0x0EB1}, {0x0EB3, 0x0EBC}, {0x0E47, 0x0E4E}, {0x0E50, 0x0E59}, {0x0EB1, 0x0EB1},
{0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, {0x0F18, 0x0F19}, {0x0EB3, 0x0EBC}, {0x0EC8, 0x0ECE}, {0x0ED0, 0x0ED9},
{0x0F20, 0x0F29}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37}, {0x0F18, 0x0F19}, {0x0F20, 0x0F29}, {0x0F35, 0x0F35},
{0x0F39, 0x0F39}, {0x0F3E, 0x0F3F}, {0x0F71, 0x0F84}, {0x0F37, 0x0F37}, {0x0F39, 0x0F39}, {0x0F3E, 0x0F3F},
{0x0F86, 0x0F87}, {0x0F8D, 0x0F97}, {0x0F99, 0x0FBC}, {0x0F71, 0x0F84}, {0x0F86, 0x0F87}, {0x0F8D, 0x0F97},
{0x0FC6, 0x0FC6}, {0x102B, 0x103E}, {0x1040, 0x1049}, {0x0F99, 0x0FBC}, {0x0FC6, 0x0FC6}, {0x102B, 0x103E},
{0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, {0x1040, 0x1049}, {0x1056, 0x1059}, {0x105E, 0x1060},
{0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, {0x1062, 0x1064}, {0x1067, 0x106D}, {0x1071, 0x1074},
{0x108F, 0x109D}, {0x135D, 0x135F}, {0x1369, 0x1371}, {0x1082, 0x108D}, {0x108F, 0x109D}, {0x135D, 0x135F},
{0x1712, 0x1715}, {0x1732, 0x1734}, {0x1752, 0x1753}, {0x1369, 0x1371}, {0x1712, 0x1715}, {0x1732, 0x1734},
{0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, {0x1752, 0x1753}, {0x1772, 0x1773}, {0x17B4, 0x17D3},
{0x17E0, 0x17E9}, {0x180B, 0x180D}, {0x180F, 0x1819}, {0x17DD, 0x17DD}, {0x17E0, 0x17E9}, {0x180B, 0x180D},
{0x18A9, 0x18A9}, {0x1920, 0x192B}, {0x1930, 0x193B}, {0x180F, 0x1819}, {0x18A9, 0x18A9}, {0x1920, 0x192B},
{0x1946, 0x194F}, {0x19D0, 0x19DA}, {0x1A17, 0x1A1B}, {0x1930, 0x193B}, {0x1946, 0x194F}, {0x19D0, 0x19DA},
{0x1A55, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A89}, {0x1A17, 0x1A1B}, {0x1A55, 0x1A5E}, {0x1A60, 0x1A7C},
{0x1A90, 0x1A99}, {0x1AB0, 0x1ABD}, {0x1ABF, 0x1ACE}, {0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, {0x1AB0, 0x1ABD},
{0x1B00, 0x1B04}, {0x1B34, 0x1B44}, {0x1B50, 0x1B59}, {0x1ABF, 0x1ACE}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44},
{0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, {0x1B50, 0x1B59}, {0x1B6B, 0x1B73}, {0x1B80, 0x1B82},
{0x1BB0, 0x1BB9}, {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1BA1, 0x1BAD}, {0x1BB0, 0x1BB9}, {0x1BE6, 0x1BF3},
{0x1C40, 0x1C49}, {0x1C50, 0x1C59}, {0x1CD0, 0x1CD2}, {0x1C24, 0x1C37}, {0x1C40, 0x1C49}, {0x1C50, 0x1C59},
{0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF4, 0x1CF4}, {0x1CD0, 0x1CD2}, {0x1CD4, 0x1CE8}, {0x1CED, 0x1CED},
{0x1CF7, 0x1CF9}, {0x1DC0, 0x1DFF}, {0x203F, 0x2040}, {0x1CF4, 0x1CF4}, {0x1CF7, 0x1CF9}, {0x1DC0, 0x1DFF},
{0x2054, 0x2054}, {0x20D0, 0x20DC}, {0x20E1, 0x20E1}, {0x203F, 0x2040}, {0x2054, 0x2054}, {0x20D0, 0x20DC},
{0x20E5, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, {0x20E1, 0x20E1}, {0x20E5, 0x20F0}, {0x2CEF, 0x2CF1},
{0x2DE0, 0x2DFF}, {0x302A, 0x302F}, {0x3099, 0x309A}, {0x2D7F, 0x2D7F}, {0x2DE0, 0x2DFF}, {0x302A, 0x302F},
{0xA620, 0xA629}, {0xA66F, 0xA66F}, {0xA674, 0xA67D}, {0x3099, 0x309A}, {0xA620, 0xA629}, {0xA66F, 0xA66F},
{0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1},
{0xA806, 0xA806}, {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA802, 0xA802}, {0xA806, 0xA806}, {0xA80B, 0xA80B},
{0xA82C, 0xA82C}, {0xA880, 0xA881}, {0xA8B4, 0xA8C5}, {0xA823, 0xA827}, {0xA82C, 0xA82C}, {0xA880, 0xA881},
{0xA8D0, 0xA8D9}, {0xA8E0, 0xA8F1}, {0xA8FF, 0xA909}, {0xA8B4, 0xA8C5}, {0xA8D0, 0xA8D9}, {0xA8E0, 0xA8F1},
{0xA926, 0xA92D}, {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA8FF, 0xA909}, {0xA926, 0xA92D}, {0xA947, 0xA953},
{0xA9B3, 0xA9C0}, {0xA9D0, 0xA9D9}, {0xA9E5, 0xA9E5}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, {0xA9D0, 0xA9D9},
{0xA9F0, 0xA9F9}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, {0xA9E5, 0xA9E5}, {0xA9F0, 0xA9F9}, {0xAA29, 0xAA36},
{0xAA4C, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA7B, 0xAA7D}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4D}, {0xAA50, 0xAA59},
{0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4},
{0xAABE, 0xAABF}, {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1},
{0xAAF5, 0xAAF6}, {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, {0xABE3, 0xABEA},
{0xABF0, 0xABF9}, {0xFB1E, 0xFB1E}, {0xFE00, 0xFE0F}, {0xABEC, 0xABED}, {0xABF0, 0xABF9}, {0xFB1E, 0xFB1E},
{0xFE20, 0xFE2F}, {0xFE33, 0xFE34}, {0xFE4D, 0xFE4F}, {0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0xFE33, 0xFE34},
{0xFF10, 0xFF19}, {0xFF3F, 0xFF3F}, {0xFF9E, 0xFF9F}, {0xFE4D, 0xFE4F}, {0xFF10, 0xFF19}, {0xFF3F, 0xFF3F},
{0x101FD, 0x101FD}, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0xFF9E, 0xFF9F}, {0x101FD, 0x101FD}, {0x102E0, 0x102E0},
{0x104A0, 0x104A9}, {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10376, 0x1037A}, {0x104A0, 0x104A9}, {0x10A01, 0x10A03},
{0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A},
{0x10AE5, 0x10AE6}, {0x10D24, 0x10D27}, {0x10D30, 0x10D39}, {0x10A3F, 0x10A3F}, {0x10AE5, 0x10AE6}, {0x10D24, 0x10D27},
{0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x10F82, 0x10F85}, {0x10D30, 0x10D39}, {0x10EAB, 0x10EAC}, {0x10EFD, 0x10EFF},
{0x11000, 0x11002}, {0x11038, 0x11046}, {0x11066, 0x11070}, {0x10F46, 0x10F50}, {0x10F82, 0x10F85}, {0x11000, 0x11002},
{0x11073, 0x11074}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, {0x11038, 0x11046}, {0x11066, 0x11070}, {0x11073, 0x11074},
{0x110C2, 0x110C2}, {0x110F0, 0x110F9}, {0x11100, 0x11102}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, {0x110C2, 0x110C2},
{0x11127, 0x11134}, {0x11136, 0x1113F}, {0x11145, 0x11146}, {0x110F0, 0x110F9}, {0x11100, 0x11102}, {0x11127, 0x11134},
{0x11173, 0x11173}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x11136, 0x1113F}, {0x11145, 0x11146}, {0x11173, 0x11173},
{0x111C9, 0x111CC}, {0x111CE, 0x111D9}, {0x1122C, 0x11237}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111C9, 0x111CC},
{0x1123E, 0x1123E}, {0x112DF, 0x112EA}, {0x112F0, 0x112F9}, {0x111CE, 0x111D9}, {0x1122C, 0x11237}, {0x1123E, 0x1123E},
{0x11241, 0x11241}, {0x112DF, 0x112EA}, {0x112F0, 0x112F9},
{0x11300, 0x11303}, {0x1133B, 0x1133C}, {0x1133E, 0x11344}, {0x11300, 0x11303}, {0x1133B, 0x1133C}, {0x1133E, 0x11344},
{0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357},
{0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374},
@ -344,19 +348,22 @@ static const llvm::sys::UnicodeCharRange XIDContinueRanges[] = {
{0x11D31, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D31, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D},
{0x11D3F, 0x11D45}, {0x11D47, 0x11D47}, {0x11D50, 0x11D59}, {0x11D3F, 0x11D45}, {0x11D47, 0x11D47}, {0x11D50, 0x11D59},
{0x11D8A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D93, 0x11D97}, {0x11D8A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D93, 0x11D97},
{0x11DA0, 0x11DA9}, {0x11EF3, 0x11EF6}, {0x16A60, 0x16A69}, {0x11DA0, 0x11DA9}, {0x11EF3, 0x11EF6}, {0x11F00, 0x11F01},
{0x16AC0, 0x16AC9}, {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x11F03, 0x11F03}, {0x11F34, 0x11F3A}, {0x11F3E, 0x11F42},
{0x16B50, 0x16B59}, {0x16F4F, 0x16F4F}, {0x16F51, 0x16F87}, {0x11F50, 0x11F59}, {0x13440, 0x13440}, {0x13447, 0x13455},
{0x16F8F, 0x16F92}, {0x16FE4, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x16A60, 0x16A69}, {0x16AC0, 0x16AC9}, {0x16AF0, 0x16AF4},
{0x1BC9D, 0x1BC9E}, {0x1CF00, 0x1CF2D}, {0x1CF30, 0x1CF46}, {0x16B30, 0x16B36}, {0x16B50, 0x16B59}, {0x16F4F, 0x16F4F},
{0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x16F51, 0x16F87}, {0x16F8F, 0x16F92}, {0x16FE4, 0x16FE4},
{0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x16FF0, 0x16FF1}, {0x1BC9D, 0x1BC9E}, {0x1CF00, 0x1CF2D},
{0x1D7CE, 0x1D7FF}, {0x1DA00, 0x1DA36}, {0x1DA3B, 0x1DA6C}, {0x1CF30, 0x1CF46}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172},
{0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD},
{0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1D242, 0x1D244}, {0x1D7CE, 0x1D7FF}, {0x1DA00, 0x1DA36},
{0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84},
{0x1E130, 0x1E136}, {0x1E140, 0x1E149}, {0x1E2AE, 0x1E2AE}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006},
{0x1E2EC, 0x1E2F9}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024},
{0x1E026, 0x1E02A}, {0x1E08F, 0x1E08F}, {0x1E130, 0x1E136},
{0x1E140, 0x1E149}, {0x1E2AE, 0x1E2AE}, {0x1E2EC, 0x1E2F9},
{0x1E4EC, 0x1E4F9}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A},
{0x1E950, 0x1E959}, {0x1FBF0, 0x1FBF9}, {0xE0100, 0xE01EF}}; {0x1E950, 0x1E959}, {0x1FBF0, 0x1FBF9}, {0xE0100, 0xE01EF}};
// C11 D.1, C++11 [charname.allowed] // C11 D.1, C++11 [charname.allowed]

5
clang/test/Lexer/unicode.c
View File

@ -40,6 +40,7 @@ extern int 𐠈;
extern int ꙮ; extern int ꙮ;
extern int \u1B4C; // BALINESE LETTER ARCHAIC JNYA - Added in Unicode 14 extern int \u1B4C; // BALINESE LETTER ARCHAIC JNYA - Added in Unicode 14
extern int \U00016AA2; // TANGSA LETTER GA - Added in Unicode 14 extern int \U00016AA2; // TANGSA LETTER GA - Added in Unicode 14
extern int \U0001E4D0; // 𞓐 NAG MUNDARI LETTER O - Added in Unicode 15
extern int _\N{TANGSA LETTER GA}; extern int _\N{TANGSA LETTER GA};
extern int _\N{TANGSALETTERGA}; // expected-error {{'TANGSALETTERGA' is not a valid Unicode character name}} \ extern int _\N{TANGSALETTERGA}; // expected-error {{'TANGSALETTERGA' is not a valid Unicode character name}} \
// expected-note {{characters names in Unicode escape sequences are sensitive to case and whitespace}} // expected-note {{characters names in Unicode escape sequences are sensitive to case and whitespace}}
@ -53,6 +54,10 @@ extern int \U00016AC0; // TANGSA DIGIT ZERO // cxx-error {{expected unqualifie
extern int 🌹; // expected-error {{unexpected character <U+1F339>}} \ extern int 🌹; // expected-error {{unexpected character <U+1F339>}} \
expected-warning {{declaration does not declare anything}} expected-warning {{declaration does not declare anything}}
extern int 🫎; // MOOSE (Unicode 15) \
// expected-error {{unexpected character <U+1FACE>}} \
expected-warning {{declaration does not declare anything}}
extern int 👷; // expected-error {{unexpected character <U+1F477>}} \ extern int 👷; // expected-error {{unexpected character <U+1F477>}} \
expected-warning {{declaration does not declare anything}} expected-warning {{declaration does not declare anything}}

297
llvm/lib/Support/Unicode.cpp
View File

@ -25,7 +25,7 @@ namespace unicode {
/// it's actually displayed on most terminals. \return true if the character is /// it's actually displayed on most terminals. \return true if the character is
/// considered printable. /// considered printable.
bool isPrintable(int UCS) { bool isPrintable(int UCS) {
// https://unicode.org/Public/14.0.0/ucdxml/ // https://unicode.org/Public/15.0.0/ucdxml/
static const UnicodeCharRange PrintableRanges[] = { static const UnicodeCharRange PrintableRanges[] = {
{0x0020, 0x007E}, {0x00A0, 0x00AC}, {0x00AE, 0x0377}, {0x0020, 0x007E}, {0x00A0, 0x00AC}, {0x00AE, 0x0377},
{0x037A, 0x037F}, {0x0384, 0x038A}, {0x038C, 0x038C}, {0x037A, 0x037F}, {0x0384, 0x038A}, {0x038C, 0x038C},
@ -68,7 +68,7 @@ bool isPrintable(int UCS) {
{0x0C8E, 0x0C90}, {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0C8E, 0x0C90}, {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3},
{0x0CB5, 0x0CB9}, {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CB5, 0x0CB9}, {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8},
{0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, {0x0CDD, 0x0CDE}, {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, {0x0CDD, 0x0CDE},
{0x0CE0, 0x0CE3}, {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, {0x0CE0, 0x0CE3}, {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF3},
{0x0D00, 0x0D0C}, {0x0D0E, 0x0D10}, {0x0D12, 0x0D44}, {0x0D00, 0x0D0C}, {0x0D0E, 0x0D10}, {0x0D12, 0x0D44},
{0x0D46, 0x0D48}, {0x0D4A, 0x0D4F}, {0x0D54, 0x0D63}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4F}, {0x0D54, 0x0D63},
{0x0D66, 0x0D7F}, {0x0D81, 0x0D83}, {0x0D85, 0x0D96}, {0x0D66, 0x0D7F}, {0x0D81, 0x0D83}, {0x0D85, 0x0D96},
@ -78,7 +78,7 @@ bool isPrintable(int UCS) {
{0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B}, {0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B},
{0x0E81, 0x0E82}, {0x0E84, 0x0E84}, {0x0E86, 0x0E8A}, {0x0E81, 0x0E82}, {0x0E84, 0x0E84}, {0x0E86, 0x0E8A},
{0x0E8C, 0x0EA3}, {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD}, {0x0E8C, 0x0EA3}, {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD},
{0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECD}, {0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECE},
{0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47},
{0x0F49, 0x0F6C}, {0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0F49, 0x0F6C}, {0x0F71, 0x0F97}, {0x0F99, 0x0FBC},
{0x0FBE, 0x0FCC}, {0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x0FBE, 0x0FCC}, {0x0FCE, 0x0FDA}, {0x1000, 0x10C5},
@ -169,13 +169,13 @@ bool isPrintable(int UCS) {
{0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48},
{0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D27}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D27},
{0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9},
{0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, {0x10F00, 0x10F27}, {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, {0x10EFD, 0x10F27},
{0x10F30, 0x10F59}, {0x10F70, 0x10F89}, {0x10FB0, 0x10FCB}, {0x10F30, 0x10F59}, {0x10F70, 0x10F89}, {0x10FB0, 0x10FCB},
{0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x11075}, {0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x11075},
{0x1107F, 0x110BC}, {0x110BE, 0x110C2}, {0x110D0, 0x110E8}, {0x1107F, 0x110BC}, {0x110BE, 0x110C2}, {0x110D0, 0x110E8},
{0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147}, {0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147},
{0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4}, {0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4},
{0x11200, 0x11211}, {0x11213, 0x1123E}, {0x11280, 0x11286}, {0x11200, 0x11211}, {0x11213, 0x11241}, {0x11280, 0x11286},
{0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D},
{0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9}, {0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9},
{0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310},
@ -193,43 +193,47 @@ bool isPrintable(int UCS) {
{0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959}, {0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959},
{0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4}, {0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4},
{0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8}, {0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8},
{0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, {0x11C38, 0x11C45}, {0x11B00, 0x11B09}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C36},
{0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, {0x11C92, 0x11CA7}, {0x11C38, 0x11C45}, {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F},
{0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06},
{0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A},
{0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D47}, {0x11D50, 0x11D59},
{0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E},
{0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF8}, {0x11D90, 0x11D91}, {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9},
{0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, {0x11FFF, 0x12399}, {0x11EE0, 0x11EF8}, {0x11F00, 0x11F10}, {0x11F12, 0x11F3A},
{0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543}, {0x11F3E, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1},
{0x12F90, 0x12FF2}, {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x11FFF, 0x12399}, {0x12400, 0x1246E}, {0x12470, 0x12474},
{0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x12480, 0x12543}, {0x12F90, 0x12FF2}, {0x13000, 0x1342F},
{0x16A6E, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x13440, 0x13455}, {0x14400, 0x14646}, {0x16800, 0x16A38},
{0x16AF0, 0x16AF5}, {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16ABE},
{0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5},
{0x16E40, 0x16E9A}, {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61},
{0x16F8F, 0x16F9F}, {0x16FE0, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E9A},
{0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F},
{0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x16FE0, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x17000, 0x187F7},
{0x1B000, 0x1B122}, {0x1B150, 0x1B152}, {0x1B164, 0x1B167}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3},
{0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122},
{0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BC9F}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155},
{0x1CF00, 0x1CF2D}, {0x1CF30, 0x1CF46}, {0x1CF50, 0x1CFC3}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A},
{0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D172}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99},
{0x1D17B, 0x1D1EA}, {0x1D200, 0x1D245}, {0x1D2E0, 0x1D2F3}, {0x1BC9C, 0x1BC9F}, {0x1CF00, 0x1CF2D}, {0x1CF30, 0x1CF46},
{0x1D300, 0x1D356}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, {0x1CF50, 0x1CFC3}, {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126},
{0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D129, 0x1D172}, {0x1D17B, 0x1D1EA}, {0x1D200, 0x1D245},
{0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D300, 0x1D356},
{0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C},
{0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6},
{0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB},
{0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A},
{0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539},
{0x1DAA1, 0x1DAAF}, {0x1DF00, 0x1DF1E}, {0x1E000, 0x1E006}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546},
{0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB},
{0x1D7CE, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF},
{0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E000, 0x1E006},
{0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024},
{0x1E026, 0x1E02A}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, {0x1E026, 0x1E02A}, {0x1E030, 0x1E06D}, {0x1E08F, 0x1E08F},
{0x1E140, 0x1E149}, {0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, {0x1E140, 0x1E149},
{0x1E2C0, 0x1E2F9}, {0x1E2FF, 0x1E2FF}, {0x1E7E0, 0x1E7E6}, {0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE}, {0x1E2C0, 0x1E2F9},
{0x1E2FF, 0x1E2FF}, {0x1E4D0, 0x1E4F9}, {0x1E7E0, 0x1E7E6},
{0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE},
{0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, {0x1E900, 0x1E94B}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, {0x1E900, 0x1E94B},
{0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, {0x1EC71, 0x1ECB4}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, {0x1EC71, 0x1ECB4},
@ -249,18 +253,17 @@ bool isPrintable(int UCS) {
{0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F100, 0x1F1AD}, {0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F100, 0x1F1AD},
{0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, {0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248},
{0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D7}, {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D7},
{0x1F6DD, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F773}, {0x1F6DC, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F776},
{0x1F780, 0x1F7D8}, {0x1F7E0, 0x1F7EB}, {0x1F7F0, 0x1F7F0}, {0x1F77B, 0x1F7D9}, {0x1F7E0, 0x1F7EB}, {0x1F7F0, 0x1F7F0},
{0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859},
{0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8B1}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8B1},
{0x1F900, 0x1FA53}, {0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA74}, {0x1F900, 0x1FA53}, {0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA7C},
{0x1FA78, 0x1FA7C}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAAC}, {0x1FA80, 0x1FA88}, {0x1FA90, 0x1FABD}, {0x1FABF, 0x1FAC5},
{0x1FAB0, 0x1FABA}, {0x1FAC0, 0x1FAC5}, {0x1FAD0, 0x1FAD9}, {0x1FACE, 0x1FADB}, {0x1FAE0, 0x1FAE8}, {0x1FAF0, 0x1FAF8},
{0x1FAE0, 0x1FAE7}, {0x1FAF0, 0x1FAF6}, {0x1FB00, 0x1FB92}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, {0x1FBF0, 0x1FBF9},
{0x1FB94, 0x1FBCA}, {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D},
{0x2A700, 0x2B738}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D},
{0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, {0xE0100, 0xE01EF}};
{0xE0100, 0xE01EF}};
static const UnicodeCharSet Printables(PrintableRanges); static const UnicodeCharSet Printables(PrintableRanges);
// Clang special cases 0x00AD (SOFT HYPHEN) which is rendered as an actual // Clang special cases 0x00AD (SOFT HYPHEN) which is rendered as an actual
@ -272,7 +275,7 @@ bool isPrintable(int UCS) {
/// formatting characters. /// formatting characters.
bool isFormatting(int UCS) { bool isFormatting(int UCS) {
// https://unicode.org/Public/14.0.0/ucdxml/ // https://unicode.org/Public/15.0.0/ucdxml/
static const UnicodeCharRange Cf[] = { static const UnicodeCharRange Cf[] = {
{0x00AD, 0x00AD}, {0x0600, 0x0605}, {0x061C, 0x061C}, {0x00AD, 0x00AD}, {0x0600, 0x0605}, {0x061C, 0x061C},
{0x06DD, 0x06DD}, {0x070F, 0x070F}, {0x0890, 0x0891}, {0x06DD, 0x06DD}, {0x070F, 0x070F}, {0x0890, 0x0891},
@ -304,81 +307,123 @@ static inline int charWidth(int UCS)
// Sorted list of non-spacing and enclosing combining mark intervals as // Sorted list of non-spacing and enclosing combining mark intervals as
// defined in "3.6 Combination" of // defined in "3.6 Combination" of
// http://www.unicode.org/versions/Unicode6.2.0/UnicodeStandard-6.2.pdf // https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf
static const UnicodeCharRange CombiningCharacterRanges[] = { static const UnicodeCharRange CombiningCharacterRanges[] = {
{ 0x0300, 0x036F }, { 0x0483, 0x0489 }, { 0x0591, 0x05BD }, {0x0300, 0x036F}, {0x0483, 0x0489}, {0x0591, 0x05BD},
{ 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, { 0x05C4, 0x05C5 }, {0x05BF, 0x05BF}, {0x05C1, 0x05C2}, {0x05C4, 0x05C5},
{ 0x05C7, 0x05C7 }, { 0x0610, 0x061A }, { 0x064B, 0x065F }, {0x05C7, 0x05C7}, {0x0610, 0x061A}, {0x064B, 0x065F},
{ 0x0670, 0x0670 }, { 0x06D6, 0x06DC }, { 0x06DF, 0x06E4 }, {0x0670, 0x0670}, {0x06D6, 0x06DC}, {0x06DF, 0x06E4},
{ 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, { 0x0711, 0x0711 }, {0x06E7, 0x06E8}, {0x06EA, 0x06ED}, {0x0711, 0x0711},
{ 0x0730, 0x074A }, { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, {0x0730, 0x074A}, {0x07A6, 0x07B0}, {0x07EB, 0x07F3},
{ 0x0816, 0x0819 }, { 0x081B, 0x0823 }, { 0x0825, 0x0827 }, {0x07FD, 0x07FD}, {0x0816, 0x0819}, {0x081B, 0x0823},
{ 0x0829, 0x082D }, { 0x0859, 0x085B }, { 0x08E4, 0x08FE }, {0x0825, 0x0827}, {0x0829, 0x082D}, {0x0859, 0x085B},
{ 0x0900, 0x0902 }, { 0x093A, 0x093A }, { 0x093C, 0x093C }, {0x0898, 0x089F}, {0x08CA, 0x08E1}, {0x08E3, 0x0902},
{ 0x0941, 0x0948 }, { 0x094D, 0x094D }, { 0x0951, 0x0957 }, {0x093A, 0x093A}, {0x093C, 0x093C}, {0x0941, 0x0948},
{ 0x0962, 0x0963 }, { 0x0981, 0x0981 }, { 0x09BC, 0x09BC }, {0x094D, 0x094D}, {0x0951, 0x0957}, {0x0962, 0x0963},
{ 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, { 0x09E2, 0x09E3 }, {0x0981, 0x0981}, {0x09BC, 0x09BC}, {0x09C1, 0x09C4},
{ 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C }, { 0x0A41, 0x0A42 }, {0x09CD, 0x09CD}, {0x09E2, 0x09E3}, {0x09FE, 0x09FE},
{ 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, { 0x0A51, 0x0A51 }, {0x0A01, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42},
{ 0x0A70, 0x0A71 }, { 0x0A75, 0x0A75 }, { 0x0A81, 0x0A82 }, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51},
{ 0x0ABC, 0x0ABC }, { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, {0x0A70, 0x0A71}, {0x0A75, 0x0A75}, {0x0A81, 0x0A82},
{ 0x0ACD, 0x0ACD }, { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5}, {0x0AC7, 0x0AC8},
{ 0x0B3C, 0x0B3C }, { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B44 }, {0x0ACD, 0x0ACD}, {0x0AE2, 0x0AE3}, {0x0AFA, 0x0AFF},
{ 0x0B4D, 0x0B4D }, { 0x0B56, 0x0B56 }, { 0x0B62, 0x0B63 }, {0x0B01, 0x0B01}, {0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F},
{ 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 }, { 0x0BCD, 0x0BCD }, {0x0B41, 0x0B44}, {0x0B4D, 0x0B4D}, {0x0B55, 0x0B56},
{ 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 }, { 0x0C4A, 0x0C4D }, {0x0B62, 0x0B63}, {0x0B82, 0x0B82}, {0x0BC0, 0x0BC0},
{ 0x0C55, 0x0C56 }, { 0x0C62, 0x0C63 }, { 0x0CBC, 0x0CBC }, {0x0BCD, 0x0BCD}, {0x0C00, 0x0C00}, {0x0C04, 0x0C04},
{ 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, {0x0C3C, 0x0C3C}, {0x0C3E, 0x0C40}, {0x0C46, 0x0C48},
{ 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D44 }, { 0x0D4D, 0x0D4D }, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, {0x0C62, 0x0C63},
{ 0x0D62, 0x0D63 }, { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, {0x0C81, 0x0C81}, {0x0CBC, 0x0CBC}, {0x0CBF, 0x0CBF},
{ 0x0DD6, 0x0DD6 }, { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD}, {0x0CE2, 0x0CE3},
{ 0x0E47, 0x0E4E }, { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, {0x0D00, 0x0D01}, {0x0D3B, 0x0D3C}, {0x0D41, 0x0D44},
{ 0x0EBB, 0x0EBC }, { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, {0x0D4D, 0x0D4D}, {0x0D62, 0x0D63}, {0x0D81, 0x0D81},
{ 0x0F35, 0x0F35 }, { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, {0x0DCA, 0x0DCA}, {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6},
{ 0x0F71, 0x0F7E }, { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, {0x0E31, 0x0E31}, {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E},
{ 0x0F8D, 0x0F97 }, { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, {0x0EB1, 0x0EB1}, {0x0EB4, 0x0EBC}, {0x0EC8, 0x0ECE},
{ 0x102D, 0x1030 }, { 0x1032, 0x1037 }, { 0x1039, 0x103A }, {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37},
{ 0x103D, 0x103E }, { 0x1058, 0x1059 }, { 0x105E, 0x1060 }, {0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84},
{ 0x1071, 0x1074 }, { 0x1082, 0x1082 }, { 0x1085, 0x1086 }, {0x0F86, 0x0F87}, {0x0F8D, 0x0F97}, {0x0F99, 0x0FBC},
{ 0x108D, 0x108D }, { 0x109D, 0x109D }, { 0x135D, 0x135F }, {0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1037},
{ 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, {0x1039, 0x103A}, {0x103D, 0x103E}, {0x1058, 0x1059},
{ 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, {0x105E, 0x1060}, {0x1071, 0x1074}, {0x1082, 0x1082},
{ 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, {0x1085, 0x1086}, {0x108D, 0x108D}, {0x109D, 0x109D},
{ 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, {0x135D, 0x135F}, {0x1712, 0x1714}, {0x1732, 0x1733},
{ 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B }, {0x1752, 0x1753}, {0x1772, 0x1773}, {0x17B4, 0x17B5},
{ 0x1A17, 0x1A18 }, { 0x1A56, 0x1A56 }, { 0x1A58, 0x1A5E }, {0x17B7, 0x17BD}, {0x17C6, 0x17C6}, {0x17C9, 0x17D3},
{ 0x1A60, 0x1A60 }, { 0x1A62, 0x1A62 }, { 0x1A65, 0x1A6C }, {0x17DD, 0x17DD}, {0x180B, 0x180D}, {0x180F, 0x180F},
{ 0x1A73, 0x1A7C }, { 0x1A7F, 0x1A7F }, { 0x1B00, 0x1B03 }, {0x1885, 0x1886}, {0x18A9, 0x18A9}, {0x1920, 0x1922},
{ 0x1B34, 0x1B34 }, { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, {0x1927, 0x1928}, {0x1932, 0x1932}, {0x1939, 0x193B},
{ 0x1B42, 0x1B42 }, { 0x1B6B, 0x1B73 }, { 0x1B80, 0x1B81 }, {0x1A17, 0x1A18}, {0x1A1B, 0x1A1B}, {0x1A56, 0x1A56},
{ 0x1BA2, 0x1BA5 }, { 0x1BA8, 0x1BA9 }, { 0x1BAB, 0x1BAB }, {0x1A58, 0x1A5E}, {0x1A60, 0x1A60}, {0x1A62, 0x1A62},
{ 0x1BE6, 0x1BE6 }, { 0x1BE8, 0x1BE9 }, { 0x1BED, 0x1BED }, {0x1A65, 0x1A6C}, {0x1A73, 0x1A7C}, {0x1A7F, 0x1A7F},
{ 0x1BEF, 0x1BF1 }, { 0x1C2C, 0x1C33 }, { 0x1C36, 0x1C37 }, {0x1AB0, 0x1ACE}, {0x1B00, 0x1B03}, {0x1B34, 0x1B34},
{ 0x1CD0, 0x1CD2 }, { 0x1CD4, 0x1CE0 }, { 0x1CE2, 0x1CE8 }, {0x1B36, 0x1B3A}, {0x1B3C, 0x1B3C}, {0x1B42, 0x1B42},
{ 0x1CED, 0x1CED }, { 0x1CF4, 0x1CF4 }, { 0x1DC0, 0x1DE6 }, {0x1B6B, 0x1B73}, {0x1B80, 0x1B81}, {0x1BA2, 0x1BA5},
{ 0x1DFC, 0x1DFF }, { 0x20D0, 0x20F0 }, { 0x2CEF, 0x2CF1 }, {0x1BA8, 0x1BA9}, {0x1BAB, 0x1BAD}, {0x1BE6, 0x1BE6},
{ 0x2D7F, 0x2D7F }, { 0x2DE0, 0x2DFF }, { 0x302A, 0x302D }, {0x1BE8, 0x1BE9}, {0x1BED, 0x1BED}, {0x1BEF, 0x1BF1},
{ 0x3099, 0x309A }, { 0xA66F, 0xA672 }, { 0xA674, 0xA67D }, {0x1C2C, 0x1C33}, {0x1C36, 0x1C37}, {0x1CD0, 0x1CD2},
{ 0xA69F, 0xA69F }, { 0xA6F0, 0xA6F1 }, { 0xA802, 0xA802 }, {0x1CD4, 0x1CE0}, {0x1CE2, 0x1CE8}, {0x1CED, 0x1CED},
{ 0xA806, 0xA806 }, { 0xA80B, 0xA80B }, { 0xA825, 0xA826 }, {0x1CF4, 0x1CF4}, {0x1CF8, 0x1CF9}, {0x1DC0, 0x1DFF},
{ 0xA8C4, 0xA8C4 }, { 0xA8E0, 0xA8F1 }, { 0xA926, 0xA92D }, {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F},
{ 0xA947, 0xA951 }, { 0xA980, 0xA982 }, { 0xA9B3, 0xA9B3 }, {0x2DE0, 0x2DFF}, {0x302A, 0x302D}, {0x3099, 0x309A},
{ 0xA9B6, 0xA9B9 }, { 0xA9BC, 0xA9BC }, { 0xAA29, 0xAA2E }, {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F},
{ 0xAA31, 0xAA32 }, { 0xAA35, 0xAA36 }, { 0xAA43, 0xAA43 }, {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806},
{ 0xAA4C, 0xAA4C }, { 0xAAB0, 0xAAB0 }, { 0xAAB2, 0xAAB4 }, {0xA80B, 0xA80B}, {0xA825, 0xA826}, {0xA82C, 0xA82C},
{ 0xAAB7, 0xAAB8 }, { 0xAABE, 0xAABF }, { 0xAAC1, 0xAAC1 }, {0xA8C4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA8FF, 0xA8FF},
{ 0xAAEC, 0xAAED }, { 0xAAF6, 0xAAF6 }, { 0xABE5, 0xABE5 }, {0xA926, 0xA92D}, {0xA947, 0xA951}, {0xA980, 0xA982},
{ 0xABE8, 0xABE8 }, { 0xABED, 0xABED }, { 0xFB1E, 0xFB1E }, {0xA9B3, 0xA9B3}, {0xA9B6, 0xA9B9}, {0xA9BC, 0xA9BD},
{ 0xFE00, 0xFE0F }, { 0xFE20, 0xFE26 }, { 0x101FD, 0x101FD }, {0xA9E5, 0xA9E5}, {0xAA29, 0xAA2E}, {0xAA31, 0xAA32},
{ 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F }, {0xAA35, 0xAA36}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4C},
{ 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x11001, 0x11001 }, {0xAA7C, 0xAA7C}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4},
{ 0x11038, 0x11046 }, { 0x11080, 0x11081 }, { 0x110B3, 0x110B6 }, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1},
{ 0x110B9, 0x110BA }, { 0x11100, 0x11102 }, { 0x11127, 0x1112B }, {0xAAEC, 0xAAED}, {0xAAF6, 0xAAF6}, {0xABE5, 0xABE5},
{ 0x1112D, 0x11134 }, { 0x11180, 0x11181 }, { 0x111B6, 0x111BE }, {0xABE8, 0xABE8}, {0xABED, 0xABED}, {0xFB1E, 0xFB1E},
{ 0x116AB, 0x116AB }, { 0x116AD, 0x116AD }, { 0x116B0, 0x116B5 }, {0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD},
{ 0x116B7, 0x116B7 }, { 0x16F8F, 0x16F92 }, { 0x1D167, 0x1D169 }, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03},
{ 0x1D17B, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD }, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A},
{ 0x1D242, 0x1D244 }, { 0xE0100, 0xE01EF }, {0x10A3F, 0x10A3F}, {0x10AE5, 0x10AE6}, {0x10D24, 0x10D27},
{0x10EAB, 0x10EAC}, {0x10EFD, 0x10EFF}, {0x10F46, 0x10F50},
{0x10F82, 0x10F85}, {0x11001, 0x11001}, {0x11038, 0x11046},
{0x11070, 0x11070}, {0x11073, 0x11074}, {0x1107F, 0x11081},
{0x110B3, 0x110B6}, {0x110B9, 0x110BA}, {0x110C2, 0x110C2},
{0x11100, 0x11102}, {0x11127, 0x1112B}, {0x1112D, 0x11134},
{0x11173, 0x11173}, {0x11180, 0x11181}, {0x111B6, 0x111BE},
{0x111C9, 0x111CC}, {0x111CF, 0x111CF}, {0x1122F, 0x11231},
{0x11234, 0x11234}, {0x11236, 0x11237}, {0x1123E, 0x1123E},
{0x11241, 0x11241}, {0x112DF, 0x112DF}, {0x112E3, 0x112EA},
{0x11300, 0x11301}, {0x1133B, 0x1133C}, {0x11340, 0x11340},
{0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11438, 0x1143F},
{0x11442, 0x11444}, {0x11446, 0x11446}, {0x1145E, 0x1145E},
{0x114B3, 0x114B8}, {0x114BA, 0x114BA}, {0x114BF, 0x114C0},
{0x114C2, 0x114C3}, {0x115B2, 0x115B5}, {0x115BC, 0x115BD},
{0x115BF, 0x115C0}, {0x115DC, 0x115DD}, {0x11633, 0x1163A},
{0x1163D, 0x1163D}, {0x1163F, 0x11640}, {0x116AB, 0x116AB},
{0x116AD, 0x116AD}, {0x116B0, 0x116B5}, {0x116B7, 0x116B7},
{0x1171D, 0x1171F}, {0x11722, 0x11725}, {0x11727, 0x1172B},
{0x1182F, 0x11837}, {0x11839, 0x1183A}, {0x1193B, 0x1193C},
{0x1193E, 0x1193E}, {0x11943, 0x11943}, {0x119D4, 0x119D7},
{0x119DA, 0x119DB}, {0x119E0, 0x119E0}, {0x11A01, 0x11A0A},
{0x11A33, 0x11A38}, {0x11A3B, 0x11A3E}, {0x11A47, 0x11A47},
{0x11A51, 0x11A56}, {0x11A59, 0x11A5B}, {0x11A8A, 0x11A96},
{0x11A98, 0x11A99}, {0x11C30, 0x11C36}, {0x11C38, 0x11C3D},
{0x11C3F, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CAA, 0x11CB0},
{0x11CB2, 0x11CB3}, {0x11CB5, 0x11CB6}, {0x11D31, 0x11D36},
{0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D45},
{0x11D47, 0x11D47}, {0x11D90, 0x11D91}, {0x11D95, 0x11D95},
{0x11D97, 0x11D97}, {0x11EF3, 0x11EF4}, {0x11F00, 0x11F01},
{0x11F36, 0x11F3A}, {0x11F40, 0x11F40}, {0x11F42, 0x11F42},
{0x13440, 0x13440}, {0x13447, 0x13455}, {0x16AF0, 0x16AF4},
{0x16B30, 0x16B36}, {0x16F4F, 0x16F4F}, {0x16F8F, 0x16F92},
{0x16FE4, 0x16FE4}, {0x1BC9D, 0x1BC9E}, {0x1CF00, 0x1CF2D},
{0x1CF30, 0x1CF46}, {0x1D167, 0x1D169}, {0x1D17B, 0x1D182},
{0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244},
{0x1DA00, 0x1DA36}, {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75},
{0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF},
{0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021},
{0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E08F, 0x1E08F},
{0x1E130, 0x1E136}, {0x1E2AE, 0x1E2AE}, {0x1E2EC, 0x1E2EF},
{0x1E4EC, 0x1E4EF}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A},
{0xE0100, 0xE01EF},
}; };
static const UnicodeCharSet CombiningCharacters(CombiningCharacterRanges); static const UnicodeCharSet CombiningCharacters(CombiningCharacterRanges);

71
llvm/lib/Support/UnicodeCaseFold.cpp
View File

@ -2,11 +2,11 @@
// //
// This file was generated by utils/unicode-case-fold.py from the Unicode // This file was generated by utils/unicode-case-fold.py from the Unicode
// case folding database at // case folding database at
// http://www.unicode.org/Public/9.0.0/ucd/CaseFolding.txt // http://www.unicode.org/Public/15.0.0/ucd/CaseFolding.txt
// //
// To regenerate this file, run: // To regenerate this file, run:
// utils/unicode-case-fold.py \ // utils/unicode-case-fold.py \
// "http://www.unicode.org/Public/9.0.0/ucd/CaseFolding.txt" \ // "http://www.unicode.org/Public/15.0.0/ucd/CaseFolding.txt" \
// > lib/Support/UnicodeCaseFold.cpp // > lib/Support/UnicodeCaseFold.cpp
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
@ -411,6 +411,16 @@ int llvm::sys::unicode::foldCharSimple(int C) {
// CYRILLIC SMALL LETTER UNBLENDED UK // CYRILLIC SMALL LETTER UNBLENDED UK
if (C == 0x1c88) if (C == 0x1c88)
return 0xa64b; return 0xa64b;
if (C < 0x1c90)
return C;
// 43 characters
if (C <= 0x1cba)
return C + -3008;
if (C < 0x1cbd)
return C;
// 3 characters
if (C <= 0x1cbf)
return C + -3008;
if (C < 0x1e00) if (C < 0x1e00)
return C; return C;
// 75 characters // 75 characters
@ -564,8 +574,8 @@ int llvm::sys::unicode::foldCharSimple(int C) {
return C + 26; return C + 26;
if (C < 0x2c00) if (C < 0x2c00)
return C; return C;
// 47 characters // 48 characters
if (C <= 0x2c2e) if (C <= 0x2c2f)
return C + 48; return C + 48;
// LATIN CAPITAL LETTER L WITH DOUBLE BAR // LATIN CAPITAL LETTER L WITH DOUBLE BAR
if (C == 0x2c60) if (C == 0x2c60)
@ -699,9 +709,33 @@ int llvm::sys::unicode::foldCharSimple(int C) {
return 0xab53; return 0xab53;
if (C < 0xa7b4) if (C < 0xa7b4)
return C; return C;
// 2 characters // 8 characters
if (C <= 0xa7b6) if (C <= 0xa7c2)
return C | 1; return C | 1;
// LATIN CAPITAL LETTER C WITH PALATAL HOOK
if (C == 0xa7c4)
return 0xa794;
// LATIN CAPITAL LETTER S WITH HOOK
if (C == 0xa7c5)
return 0x0282;
// LATIN CAPITAL LETTER Z WITH PALATAL HOOK
if (C == 0xa7c6)
return 0x1d8e;
if (C < 0xa7c7)
return C;
// 2 characters
if (C <= 0xa7c9 && C % 2 == 1)
return C + 1;
if (C < 0xa7d0)
return C;
// 2 characters
if (C <= 0xa7d6 && C % 6 == 0)
return C + 1;
if (C < 0xa7d8)
return C;
// 2 characters
if (C <= 0xa7f5 && C % 29 == 19)
return C + 1;
if (C < 0xab70) if (C < 0xab70)
return C; return C;
// 80 characters // 80 characters
@ -722,6 +756,26 @@ int llvm::sys::unicode::foldCharSimple(int C) {
// 36 characters // 36 characters
if (C <= 0x104d3) if (C <= 0x104d3)
return C + 40; return C + 40;
if (C < 0x10570)
return C;
// 11 characters
if (C <= 0x1057a)
return C + 39;
if (C < 0x1057c)
return C;
// 15 characters
if (C <= 0x1058a)
return C + 39;
if (C < 0x1058c)
return C;
// 7 characters
if (C <= 0x10592)
return C + 39;
if (C < 0x10594)
return C;
// 2 characters
if (C <= 0x10595)
return C + 39;
if (C < 0x10c80) if (C < 0x10c80)
return C; return C;
// 51 characters // 51 characters
@ -732,6 +786,11 @@ int llvm::sys::unicode::foldCharSimple(int C) {
// 32 characters // 32 characters
if (C <= 0x118bf) if (C <= 0x118bf)
return C + 32; return C + 32;
if (C < 0x16e40)
return C;
// 32 characters
if (C <= 0x16e5f)
return C + 32;
if (C < 0x1e900) if (C < 0x1e900)
return C; return C;
// 34 characters // 34 characters

15
llvm/lib/Support/UnicodeNameToCodepoint.cpp
View File

@ -46,7 +46,7 @@ struct Node {
std::string S; std::string S;
// Reserve enough space for most unicode code points. // Reserve enough space for most unicode code points.
// The chosen value represent the 99th percentile of name size as of // The chosen value represent the 99th percentile of name size as of
// Unicode 14. // Unicode 15.0.
S.reserve(46); S.reserve(46);
const Node *N = this; const Node *N = this;
while (N) { while (N) {
@ -251,7 +251,7 @@ constexpr const char *const HangulSyllables[][3] = {
}; };
// clang-format on // clang-format on
// Unicode 14.0 // Unicode 15.0
// 3.12 Conjoining Jamo Behavior Common constants // 3.12 Conjoining Jamo Behavior Common constants
constexpr const char32_t SBase = 0xAC00; constexpr const char32_t SBase = 0xAC00;
constexpr const uint32_t LCount = 19; constexpr const uint32_t LCount = 19;
@ -323,18 +323,17 @@ struct GeneratedNamesData {
uint32_t End; uint32_t End;
}; };
// Unicode 14.0 Table 4-8. Name Derivation Rule Prefix Strings // Unicode 15.0 Table 4-8. Name Derivation Rule Prefix Strings
// This needs to be kept in sync with
// llvm/utils/UnicodeData/UnicodeNameMappingGenerator.cpp
static const GeneratedNamesData GeneratedNamesDataTable[] = { static const GeneratedNamesData GeneratedNamesDataTable[] = {
{"CJK UNIFIED IDEOGRAPH-", 0x3400, 0x4DBF}, {"CJK UNIFIED IDEOGRAPH-", 0x3400, 0x4DBF},
{"CJK UNIFIED IDEOGRAPH-", 0x4E00, 0x9FFC}, {"CJK UNIFIED IDEOGRAPH-", 0x4E00, 0x9FFF},
{"CJK UNIFIED IDEOGRAPH-", 0x20000, 0x2A6DD}, {"CJK UNIFIED IDEOGRAPH-", 0x20000, 0x2A6DF},
{"CJK UNIFIED IDEOGRAPH-", 0x2A700, 0x2B734}, {"CJK UNIFIED IDEOGRAPH-", 0x2A700, 0x2B739},
{"CJK UNIFIED IDEOGRAPH-", 0x2B740, 0x2B81D}, {"CJK UNIFIED IDEOGRAPH-", 0x2B740, 0x2B81D},
{"CJK UNIFIED IDEOGRAPH-", 0x2B820, 0x2CEA1}, {"CJK UNIFIED IDEOGRAPH-", 0x2B820, 0x2CEA1},
{"CJK UNIFIED IDEOGRAPH-", 0x2CEB0, 0x2EBE0}, {"CJK UNIFIED IDEOGRAPH-", 0x2CEB0, 0x2EBE0},
{"CJK UNIFIED IDEOGRAPH-", 0x30000, 0x3134A}, {"CJK UNIFIED IDEOGRAPH-", 0x30000, 0x3134A},
{"CJK UNIFIED IDEOGRAPH-", 0x31350, 0x323AF},
{"TANGUT IDEOGRAPH-", 0x17000, 0x187F7}, {"TANGUT IDEOGRAPH-", 0x17000, 0x187F7},
{"TANGUT IDEOGRAPH-", 0x18D00, 0x18D08}, {"TANGUT IDEOGRAPH-", 0x18D00, 0x18D08},
{"KHITAN SMALL SCRIPT CHARACTER-", 0x18B00, 0x18CD5}, {"KHITAN SMALL SCRIPT CHARACTER-", 0x18B00, 0x18CD5},

41342
llvm/lib/Support/UnicodeNameToCodepointGenerated.cpp
View File

File diff suppressed because it is too large Load Diff

4
llvm/unittests/Support/UnicodeTest.cpp
View File

@ -139,6 +139,7 @@ TEST(Unicode, nameToCodepointStrict) {
EXPECT_EQ(0x0FAD9u, map("CJK COMPATIBILITY IDEOGRAPH-FAD9")); EXPECT_EQ(0x0FAD9u, map("CJK COMPATIBILITY IDEOGRAPH-FAD9"));
EXPECT_EQ(0x2F800u, map("CJK COMPATIBILITY IDEOGRAPH-2F800")); EXPECT_EQ(0x2F800u, map("CJK COMPATIBILITY IDEOGRAPH-2F800"));
EXPECT_EQ(0x2FA1Du, map("CJK COMPATIBILITY IDEOGRAPH-2FA1D")); EXPECT_EQ(0x2FA1Du, map("CJK COMPATIBILITY IDEOGRAPH-2FA1D"));
EXPECT_EQ(0x31350u, map("CJK UNIFIED IDEOGRAPH-31350")); // Unicode 15.0
EXPECT_EQ(0xAC00u, map("HANGUL SYLLABLE GA")); EXPECT_EQ(0xAC00u, map("HANGUL SYLLABLE GA"));
EXPECT_EQ(0xAC14u, map("HANGUL SYLLABLE GASS")); EXPECT_EQ(0xAC14u, map("HANGUL SYLLABLE GASS"));
@ -156,9 +157,12 @@ TEST(Unicode, nameToCodepointStrict) {
EXPECT_EQ(0x02235u, map("BECAUSE")); EXPECT_EQ(0x02235u, map("BECAUSE"));
EXPECT_EQ(0x1F514u, map("BELL")); EXPECT_EQ(0x1F514u, map("BELL"));
EXPECT_EQ(0x1F9A9u, map("FLAMINGO")); EXPECT_EQ(0x1F9A9u, map("FLAMINGO"));
EXPECT_EQ(0x1F9A9u, map("FLAMINGO"));
EXPECT_EQ(0x1F402u, map("OX")); // 2 characters EXPECT_EQ(0x1F402u, map("OX")); // 2 characters
EXPECT_EQ(0x0FBF9u, map("ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA " EXPECT_EQ(0x0FBF9u, map("ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA "
"ABOVE WITH ALEF MAKSURA ISOLATED FORM")); "ABOVE WITH ALEF MAKSURA ISOLATED FORM"));
EXPECT_EQ(0x11F04u, map("KAWI LETTER A")); // Unicode 15.0
EXPECT_EQ(0x1FA77u, map("PINK HEART")); // Unicode 15.0
// Aliases // Aliases
EXPECT_EQ(0x0000u, map("NULL")); EXPECT_EQ(0x0000u, map("NULL"));

6
llvm/utils/UnicodeData/UnicodeNameMappingGenerator.cpp
View File

@ -8,7 +8,7 @@
// //
// This file is used to generate lib/Support/UnicodeNameToCodepointGenerated.cpp // This file is used to generate lib/Support/UnicodeNameToCodepointGenerated.cpp
// using UnicodeData.txt and NameAliases.txt available at // using UnicodeData.txt and NameAliases.txt available at
// https://unicode.org/Public/14.0.0/ucd/ // https://unicode.org/Public/15.0.0/ucd/
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "llvm/ADT/Optional.h" #include "llvm/ADT/Optional.h"
@ -340,9 +340,9 @@ int main(int argc, char **argv) {
"Usage: %s UnicodeData.txt NameAliases.txt output\n\n", "Usage: %s UnicodeData.txt NameAliases.txt output\n\n",
argv[0]); argv[0]);
printf("NameAliases.txt can be found at " printf("NameAliases.txt can be found at "
"https://unicode.org/Public/14.0.0/ucd/NameAliases.txt\n" "https://unicode.org/Public/15.0.0/ucd/NameAliases.txt\n"
"UnicodeData.txt can be found at " "UnicodeData.txt can be found at "
"https://unicode.org/Public/14.0.0/ucd/UnicodeData.txt\n\n"); "https://unicode.org/Public/15.0.0/ucd/UnicodeData.txt\n\n");
if (argc != 4) if (argc != 4)
return EXIT_FAILURE; return EXIT_FAILURE;