Attachment 8809621 Details for Bug 1197230

[patch] Handle non-BMP identifier.

01-Bug_1197230___Handle_non_BMP_identifier_.patch (text/plain), 52.39 KB, created by Tooru Fujisawa [:arai]

(hide)

Description:

Filename:

MIME Type:

Creator: Tooru Fujisawa [:arai]

Size: 52.39 KB

patch

obsolete

># HG changeset patch
># User Tooru Fujisawa <arai_a@mac.com>
># Date 1478816812 -32400
>#      Fri Nov 11 07:26:52 2016 +0900
># Node ID e16ee2547a7ec7afcd3adaa1c9c5e9ffac15a9a3
># Parent  4e1adcac2cfd09116a522c79a4baafc02d26fee3
>Bug 1197230 - Handle non-BMP identifier.
>
>diff --git a/js/src/frontend/TokenStream.cpp b/js/src/frontend/TokenStream.cpp
>--- a/js/src/frontend/TokenStream.cpp
>+++ b/js/src/frontend/TokenStream.cpp
>@@ -113,23 +113,67 @@ IsIdentifier(const CharT* chars, size_t 
>     while (++chars != end) {
>         if (!unicode::IsIdentifierPart(char16_t(*chars)))
>             return false;
>     }
> 
>     return true;
> }
> 
>+static uint32_t
>+GetSingleCodePoint(const char16_t*& p, const char16_t*& end)
>+{
>+    uint32_t codePoint;
>+    if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(*p)) && p + 1 < end) {
>+        char16_t lead = *p;
>+        char16_t maybeTrail = *(p + 1);
>+        if (unicode::IsTrailSurrogate(maybeTrail)) {
>+            p += 2;
>+            return unicode::UTF16Decode(lead, maybeTrail);
>+        }
>+    }
>+
>+    codePoint = *p;
>+    p++;
>+    return codePoint;
>+}
>+
>+static bool
>+IsIdentifierMaybeNonBMP(const char16_t* chars, size_t length)
>+{
>+    if (IsIdentifier(chars, length))
>+        return true;
>+
>+    if (length == 0)
>+        return false;
>+
>+    const char16_t* p = chars;
>+    const char16_t* end = chars + length;
>+    uint32_t codePoint;
>+
>+    codePoint = GetSingleCodePoint(p, end);
>+    if (!unicode::IsIdentifierStart(codePoint))
>+        return false;
>+
>+    while (p < end) {
>+        codePoint = GetSingleCodePoint(p, end);
>+        if (!unicode::IsIdentifierPart(codePoint))
>+            return false;
>+    }
>+
>+    return true;
>+}
>+
> bool
> frontend::IsIdentifier(JSLinearString* str)
> {
>     JS::AutoCheckCannotGC nogc;
>     return str->hasLatin1Chars()
>            ? ::IsIdentifier(str->latin1Chars(nogc), str->length())
>-           : ::IsIdentifier(str->twoByteChars(nogc), str->length());
>+           : ::IsIdentifierMaybeNonBMP(str->twoByteChars(nogc), str->length());
> }
> 
> bool
> frontend::IsIdentifier(const char16_t* chars, size_t length)
> {
>     return ::IsIdentifier(chars, length);
> }
> 
>@@ -998,21 +1042,50 @@ TokenStream::putIdentInTokenbuf(const ch
>     int32_t c;
>     uint32_t qc;
>     const char16_t* tmp = userbuf.addressOfNextRawChar();
>     userbuf.setAddressOfNextRawChar(identStart);
> 
>     tokenbuf.clear();
>     for (;;) {
>         c = getCharIgnoreEOL();
>+
>+        if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
>+            int32_t maybeTrail = getCharIgnoreEOL();
>+            if (unicode::IsTrailSurrogate(maybeTrail)) {
>+                if (!unicode::IsIdentifierPart(unicode::UTF16Decode(c, maybeTrail)))
>+                    break;
>+
>+                if (!tokenbuf.append(c) || !tokenbuf.append(maybeTrail)) {
>+                    userbuf.setAddressOfNextRawChar(tmp);
>+                    return false;
>+                }
>+                continue;
>+            }
>+
>+            ungetCharIgnoreEOL(maybeTrail);
>+        }
>+
>         if (!unicode::IsIdentifierPart(char16_t(c))) {
>             if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
>                 break;
>+
>+            if (MOZ_UNLIKELY(unicode::IsSupplementary(qc))) {
>+                char16_t lead, trail;
>+                unicode::UTF16Encode(qc, &lead, &trail);
>+                if (!tokenbuf.append(lead) || !tokenbuf.append(trail)) {
>+                    userbuf.setAddressOfNextRawChar(tmp);
>+                    return false;
>+                }
>+                continue;
>+            }
>+
>             c = qc;
>         }
>+
>         if (!tokenbuf.append(c)) {
>             userbuf.setAddressOfNextRawChar(tmp);
>             return false;
>         }
>     }
>     userbuf.setAddressOfNextRawChar(tmp);
>     return true;
> }
>@@ -1163,22 +1236,35 @@ TokenStream::getTokenInternal(TokenKind*
>         tp = newToken(-1);
> 
>         static_assert('$' < 128,
>                       "IdentifierStart contains '$', but as !IsUnicodeIDStart('$'), "
>                       "ensure that '$' is never handled here");
>         static_assert('_' < 128,
>                       "IdentifierStart contains '_', but as !IsUnicodeIDStart('_'), "
>                       "ensure that '_' is never handled here");
>-        if (unicode::IsUnicodeIDStart(c)) {
>+        if (unicode::IsUnicodeIDStart(char16_t(c))) {
>             identStart = userbuf.addressOfNextRawChar() - 1;
>             hadUnicodeEscape = false;
>             goto identifier;
>         }
> 
>+        if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
>+            int32_t maybeTrail = getCharIgnoreEOL();
>+            if (unicode::IsTrailSurrogate(maybeTrail)) {
>+                if (unicode::IsUnicodeIDStart(unicode::UTF16Decode(c, maybeTrail))) {
>+                    identStart = userbuf.addressOfNextRawChar() - 2;
>+                    hadUnicodeEscape = false;
>+                    goto identifier;
>+                }
>+            }
>+
>+            ungetCharIgnoreEOL(maybeTrail);
>+        }
>+
>         goto badchar;
>     }
> 
>     // Get the token kind, based on the first char.  The ordering of c1kind
>     // comparison is based on the frequency of tokens in real code -- Parsemark
>     // (which represents typical JS code on the web) and the Unreal demo (which
>     // represents asm.js code).
>     //
>@@ -1219,16 +1305,28 @@ TokenStream::getTokenInternal(TokenKind*
>         identStart = userbuf.addressOfNextRawChar() - 1;
>         hadUnicodeEscape = false;
> 
>       identifier:
>         for (;;) {
>             c = getCharIgnoreEOL();
>             if (c == EOF)
>                 break;
>+
>+            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
>+                int32_t maybeTrail = getCharIgnoreEOL();
>+                if (unicode::IsTrailSurrogate(maybeTrail)) {
>+                    if (!unicode::IsIdentifierPart(unicode::UTF16Decode(c, maybeTrail)))
>+                        break;
>+
>+                    continue;
>+                }
>+                ungetCharIgnoreEOL(maybeTrail);
>+            }
>+
>             if (!unicode::IsIdentifierPart(char16_t(c))) {
>                 if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
>                     break;
>                 hadUnicodeEscape = true;
>             }
>         }
>         ungetCharIgnoreEOL(c);
> 
>@@ -1313,19 +1411,32 @@ TokenStream::getTokenInternal(TokenKind*
>                 goto error;
>             }
>             do {
>                 c = getCharIgnoreEOL();
>             } while (JS7_ISDEC(c));
>         }
>         ungetCharIgnoreEOL(c);
> 
>-        if (c != EOF && unicode::IsIdentifierStart(char16_t(c))) {
>-            reportError(JSMSG_IDSTART_AFTER_NUMBER);
>-            goto error;
>+        if (c != EOF) {
>+            if (unicode::IsIdentifierStart(char16_t(c))) {
>+                reportError(JSMSG_IDSTART_AFTER_NUMBER);
>+                goto error;
>+            }
>+
>+            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
>+                int32_t maybeTrail = getCharIgnoreEOL();
>+                if (unicode::IsTrailSurrogate(maybeTrail)) {
>+                    if (unicode::IsIdentifierStart(unicode::UTF16Decode(c, maybeTrail))) {
>+                        reportError(JSMSG_IDSTART_AFTER_NUMBER);
>+                        goto error;
>+                    }
>+                }
>+                ungetCharIgnoreEOL(maybeTrail);
>+            }
>         }
> 
>         // Unlike identifiers and strings, numbers cannot contain escaped
>         // chars, so we don't need to use tokenbuf.  Instead we can just
>         // convert the char16_t characters in userbuf to the numeric value.
>         double dval;
>         if (!((decimalPoint == HasDecimal) || hasExp)) {
>             if (!GetDecimalInteger(cx, numStart, userbuf.addressOfNextRawChar(), &dval))
>@@ -1420,19 +1531,32 @@ TokenStream::getTokenInternal(TokenKind*
>             }
>         } else {
>             // '0' not followed by 'x', 'X' or a digit;  scan as a decimal number.
>             numStart = userbuf.addressOfNextRawChar() - 1;
>             goto decimal;
>         }
>         ungetCharIgnoreEOL(c);
> 
>-        if (c != EOF && unicode::IsIdentifierStart(char16_t(c))) {
>-            reportError(JSMSG_IDSTART_AFTER_NUMBER);
>-            goto error;
>+        if (c != EOF) {
>+            if (unicode::IsIdentifierStart(char16_t(c))) {
>+                reportError(JSMSG_IDSTART_AFTER_NUMBER);
>+                goto error;
>+            }
>+
>+            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
>+                int32_t maybeTrail = getCharIgnoreEOL();
>+                if (unicode::IsTrailSurrogate(maybeTrail)) {
>+                    if (unicode::IsIdentifierStart(unicode::UTF16Decode(c, maybeTrail))) {
>+                        reportError(JSMSG_IDSTART_AFTER_NUMBER);
>+                        goto error;
>+                    }
>+                }
>+                ungetCharIgnoreEOL(maybeTrail);
>+            }
>         }
> 
>         double dval;
>         const char16_t* dummy;
>         if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, &dummy, &dval))
>             goto error;
>         tp->type = TOK_NUMBER;
>         tp->setNumber(dval, NoDecimal);
>diff --git a/js/src/tests/ecma_6/Syntax/identifiers-with-extended-unicode-escape.js b/js/src/tests/ecma_6/Syntax/identifiers-with-extended-unicode-escape.js
>--- a/js/src/tests/ecma_6/Syntax/identifiers-with-extended-unicode-escape.js
>+++ b/js/src/tests/ecma_6/Syntax/identifiers-with-extended-unicode-escape.js
>@@ -93,55 +93,41 @@ const otherIdContinue = [
>     0x136D,     // ETHIOPIC DIGIT FIVE, Gc=No
>     0x136E,     // ETHIOPIC DIGIT SIX, Gc=No
>     0x136F,     // ETHIOPIC DIGIT SEVEN, Gc=No
>     0x1370,     // ETHIOPIC DIGIT EIGHT, Gc=No
>     0x1371,     // ETHIOPIC DIGIT NINE, Gc=No
>     0x19DA,     // NEW TAI LUE THAM DIGIT ONE, Gc=No
> ];
> 
>-for (let ident of [...idStart, ...otherIdStart]) {
>+for (let ident of [...idStart, ...otherIdStart, ...idStartSupplemental]) {
>     for (let count of leadingZeros) {
>         let zeros = "0".repeat(count);
>         eval(`
>             let \\u{${zeros}${ident.toString(16)}} = 123;
>             assertEq(${String.fromCodePoint(ident)}, 123);
>         `);
>     }
> }
> 
>-// Move this to the loop above when Bug 1197230 is fixed.
>-for (let ident of [...idStartSupplemental]) {
>-    for (let zeros of leadingZeros) {
>-        assertThrowsInstanceOf(() => eval(`\\u{${zeros}${ident.toString(16)}}`), SyntaxError);
>-    }
>-}
>-
> for (let ident of [...idContinue, ...idContinueSupplemental, ...otherIdContinue]) {
>     for (let zeros of leadingZeros) {
>         assertThrowsInstanceOf(() => eval(`\\u{${zeros}${ident.toString(16)}}`), SyntaxError);
>     }
> }
> 
>-for (let ident of [...idStart, ...otherIdStart, ...idContinue, ...otherIdContinue]) {
>+for (let ident of [...idStart, ...otherIdStart, ...idContinue, ...otherIdContinue, ...idStartSupplemental, ...idContinueSupplemental]) {
>     for (let zeros of leadingZeros) {
>         eval(`
>             let A\\u{${zeros}${ident.toString(16)}} = 123;
>             assertEq(${String.fromCodePoint(0x41, ident)}, 123);
>         `);
>     }
> }
> 
>-// Move this to the loop above when Bug 1197230 is fixed.
>-for (let ident of [...idStartSupplemental, ...idContinueSupplemental]) {
>-    for (let zeros of leadingZeros) {
>-        assertThrowsInstanceOf(() => eval(`\\u{${zeros}${ident.toString(16)}}`), SyntaxError);
>-    }
>-}
>-
> 
> const notIdentifiers = [
>     0x0000,     // NULL, Gc=Cc
>     0x000A,     // LINE FEED (LF), Gc=Cc
>     0x005E,     // CIRCUMFLEX ACCENT, Gc=Sk
>     0x00B1,     // PLUS-MINUS SIGN, Gc=Sm
>     0xFF61,     // HALFWIDTH IDEOGRAPHIC FULL STOP, Gc=Po
>     0x10061,    // Not assigned.
>diff --git a/js/src/vm/Unicode.h b/js/src/vm/Unicode.h
>--- a/js/src/vm/Unicode.h
>+++ b/js/src/vm/Unicode.h
>@@ -141,18 +141,25 @@ IsIdentifierStart(char16_t ch)
>         return js_isidstart[ch];
> 
>     return CharInfo(ch).isUnicodeIDStart();
> }
> 
> inline bool
> IsIdentifierStart(uint32_t codePoint)
> {
>-    // TODO: Supplemental code points not yet supported (bug 1197230).
>-    return codePoint <= UTF16Max && IsIdentifierStart(char16_t(codePoint));
>+    if (MOZ_LIKELY(codePoint <= UTF16Max))
>+        return IsIdentifierStart(char16_t(codePoint));
>+
>+#define CHECK_RANGE(FROM, TO, LEAD, TRAIL_FROM, TRAIL_TO) \
>+    if (codePoint >= FROM && codePoint <= TO) \
>+        return true;
>+FOR_EACH_NON_BMP_ID_START(CHECK_RANGE)
>+#undef CHECK_RANGE
>+    return false;
> }
> 
> inline bool
> IsIdentifierPart(char16_t ch)
> {
>     /*
>      * ES2016 11.6 IdentifierPart
>      *  $ (dollar sign)
>@@ -168,27 +175,48 @@ IsIdentifierPart(char16_t ch)
>         return js_isident[ch];
> 
>     return CharInfo(ch).isUnicodeIDContinue();
> }
> 
> inline bool
> IsIdentifierPart(uint32_t codePoint)
> {
>-    // TODO: Supplemental code points not yet supported (bug 1197230).
>-    return codePoint <= UTF16Max && IsIdentifierPart(char16_t(codePoint));
>+    if (MOZ_LIKELY(codePoint <= UTF16Max))
>+        return IsIdentifierPart(char16_t(codePoint));
>+
>+#define CHECK_RANGE(FROM, TO, LEAD, TRAIL_FROM, TRAIL_TO) \
>+    if (codePoint >= FROM && codePoint <= TO) \
>+        return true;
>+FOR_EACH_NON_BMP_ID_CONT(CHECK_RANGE)
>+#undef CHECK_RANGE
>+    return false;
> }
> 
> inline bool
> IsUnicodeIDStart(char16_t ch)
> {
>     return CharInfo(ch).isUnicodeIDStart();
> }
> 
> inline bool
>+IsUnicodeIDStart(uint32_t codePoint)
>+{
>+    if (MOZ_LIKELY(codePoint <= UTF16Max))
>+        return IsUnicodeIDStart(char16_t(codePoint));
>+
>+#define CHECK_RANGE(FROM, TO, LEAD, TRAIL_FROM, TRAIL_TO) \
>+    if (codePoint >= FROM && codePoint <= TO) \
>+        return true;
>+FOR_EACH_NON_BMP_ID_START(CHECK_RANGE)
>+#undef CHECK_RANGE
>+    return false;
>+}
>+
>+inline bool
> IsSpace(char16_t ch)
> {
>     /*
>      * IsSpace checks if some character is included in the merged set
>      * of WhiteSpace and LineTerminator, specified by ES2016 11.2 and 11.3.
>      * We combined them, because in practice nearly every
>      * calling function wants this, except some code in the tokenizer.
>      *
>diff --git a/js/src/vm/UnicodeNonBMP.h b/js/src/vm/UnicodeNonBMP.h
>--- a/js/src/vm/UnicodeNonBMP.h
>+++ b/js/src/vm/UnicodeNonBMP.h
>@@ -33,9 +33,561 @@
> 
> #define FOR_EACH_NON_BMP_REV_CASE_FOLDING(macro) \
>     macro(0x10428, 0x1044f, 0xd801, 0xdc28, 0xdc4f, -40) \
>     macro(0x104d8, 0x104fb, 0xd801, 0xdcd8, 0xdcfb, -40) \
>     macro(0x10cc0, 0x10cf2, 0xd803, 0xdcc0, 0xdcf2, -64) \
>     macro(0x118c0, 0x118df, 0xd806, 0xdcc0, 0xdcdf, -32) \
>     macro(0x1e922, 0x1e943, 0xd83a, 0xdd22, 0xdd43, -34)
> 
>+#define FOR_EACH_NON_BMP_ID_START(macro) \
>+    macro(0x10000, 0x1000b, 0xd800, 0xdc00, 0xdc0b) \
>+    macro(0x1000d, 0x10026, 0xd800, 0xdc0d, 0xdc26) \
>+    macro(0x10028, 0x1003a, 0xd800, 0xdc28, 0xdc3a) \
>+    macro(0x1003c, 0x1003d, 0xd800, 0xdc3c, 0xdc3d) \
>+    macro(0x1003f, 0x1004d, 0xd800, 0xdc3f, 0xdc4d) \
>+    macro(0x10050, 0x1005d, 0xd800, 0xdc50, 0xdc5d) \
>+    macro(0x10080, 0x100fa, 0xd800, 0xdc80, 0xdcfa) \
>+    macro(0x10140, 0x10174, 0xd800, 0xdd40, 0xdd74) \
>+    macro(0x10280, 0x1029c, 0xd800, 0xde80, 0xde9c) \
>+    macro(0x102a0, 0x102d0, 0xd800, 0xdea0, 0xded0) \
>+    macro(0x10300, 0x1031f, 0xd800, 0xdf00, 0xdf1f) \
>+    macro(0x10330, 0x1034a, 0xd800, 0xdf30, 0xdf4a) \
>+    macro(0x10350, 0x10375, 0xd800, 0xdf50, 0xdf75) \
>+    macro(0x10380, 0x1039d, 0xd800, 0xdf80, 0xdf9d) \
>+    macro(0x103a0, 0x103c3, 0xd800, 0xdfa0, 0xdfc3) \
>+    macro(0x103c8, 0x103cf, 0xd800, 0xdfc8, 0xdfcf) \
>+    macro(0x103d1, 0x103d5, 0xd800, 0xdfd1, 0xdfd5) \
>+    macro(0x10400, 0x1049d, 0xd801, 0xdc00, 0xdc9d) \
>+    macro(0x104b0, 0x104d3, 0xd801, 0xdcb0, 0xdcd3) \
>+    macro(0x104d8, 0x104fb, 0xd801, 0xdcd8, 0xdcfb) \
>+    macro(0x10500, 0x10527, 0xd801, 0xdd00, 0xdd27) \
>+    macro(0x10530, 0x10563, 0xd801, 0xdd30, 0xdd63) \
>+    macro(0x10600, 0x10736, 0xd801, 0xde00, 0xdf36) \
>+    macro(0x10740, 0x10755, 0xd801, 0xdf40, 0xdf55) \
>+    macro(0x10760, 0x10767, 0xd801, 0xdf60, 0xdf67) \
>+    macro(0x10800, 0x10805, 0xd802, 0xdc00, 0xdc05) \
>+    macro(0x10808, 0x10808, 0xd802, 0xdc08, 0xdc08) \
>+    macro(0x1080a, 0x10835, 0xd802, 0xdc0a, 0xdc35) \
>+    macro(0x10837, 0x10838, 0xd802, 0xdc37, 0xdc38) \
>+    macro(0x1083c, 0x1083c, 0xd802, 0xdc3c, 0xdc3c) \
>+    macro(0x1083f, 0x10855, 0xd802, 0xdc3f, 0xdc55) \
>+    macro(0x10860, 0x10876, 0xd802, 0xdc60, 0xdc76) \
>+    macro(0x10880, 0x1089e, 0xd802, 0xdc80, 0xdc9e) \
>+    macro(0x108e0, 0x108f2, 0xd802, 0xdce0, 0xdcf2) \
>+    macro(0x108f4, 0x108f5, 0xd802, 0xdcf4, 0xdcf5) \
>+    macro(0x10900, 0x10915, 0xd802, 0xdd00, 0xdd15) \
>+    macro(0x10920, 0x10939, 0xd802, 0xdd20, 0xdd39) \
>+    macro(0x10980, 0x109b7, 0xd802, 0xdd80, 0xddb7) \
>+    macro(0x109be, 0x109bf, 0xd802, 0xddbe, 0xddbf) \
>+    macro(0x10a00, 0x10a00, 0xd802, 0xde00, 0xde00) \
>+    macro(0x10a10, 0x10a13, 0xd802, 0xde10, 0xde13) \
>+    macro(0x10a15, 0x10a17, 0xd802, 0xde15, 0xde17) \
>+    macro(0x10a19, 0x10a33, 0xd802, 0xde19, 0xde33) \
>+    macro(0x10a60, 0x10a7c, 0xd802, 0xde60, 0xde7c) \
>+    macro(0x10a80, 0x10a9c, 0xd802, 0xde80, 0xde9c) \
>+    macro(0x10ac0, 0x10ac7, 0xd802, 0xdec0, 0xdec7) \
>+    macro(0x10ac9, 0x10ae4, 0xd802, 0xdec9, 0xdee4) \
>+    macro(0x10b00, 0x10b35, 0xd802, 0xdf00, 0xdf35) \
>+    macro(0x10b40, 0x10b55, 0xd802, 0xdf40, 0xdf55) \
>+    macro(0x10b60, 0x10b72, 0xd802, 0xdf60, 0xdf72) \
>+    macro(0x10b80, 0x10b91, 0xd802, 0xdf80, 0xdf91) \
>+    macro(0x10c00, 0x10c48, 0xd803, 0xdc00, 0xdc48) \
>+    macro(0x10c80, 0x10cb2, 0xd803, 0xdc80, 0xdcb2) \
>+    macro(0x10cc0, 0x10cf2, 0xd803, 0xdcc0, 0xdcf2) \
>+    macro(0x11003, 0x11037, 0xd804, 0xdc03, 0xdc37) \
>+    macro(0x11083, 0x110af, 0xd804, 0xdc83, 0xdcaf) \
>+    macro(0x110d0, 0x110e8, 0xd804, 0xdcd0, 0xdce8) \
>+    macro(0x11103, 0x11126, 0xd804, 0xdd03, 0xdd26) \
>+    macro(0x11150, 0x11172, 0xd804, 0xdd50, 0xdd72) \
>+    macro(0x11176, 0x11176, 0xd804, 0xdd76, 0xdd76) \
>+    macro(0x11183, 0x111b2, 0xd804, 0xdd83, 0xddb2) \
>+    macro(0x111c1, 0x111c4, 0xd804, 0xddc1, 0xddc4) \
>+    macro(0x111da, 0x111da, 0xd804, 0xddda, 0xddda) \
>+    macro(0x111dc, 0x111dc, 0xd804, 0xdddc, 0xdddc) \
>+    macro(0x11200, 0x11211, 0xd804, 0xde00, 0xde11) \
>+    macro(0x11213, 0x1122b, 0xd804, 0xde13, 0xde2b) \
>+    macro(0x11280, 0x11286, 0xd804, 0xde80, 0xde86) \
>+    macro(0x11288, 0x11288, 0xd804, 0xde88, 0xde88) \
>+    macro(0x1128a, 0x1128d, 0xd804, 0xde8a, 0xde8d) \
>+    macro(0x1128f, 0x1129d, 0xd804, 0xde8f, 0xde9d) \
>+    macro(0x1129f, 0x112a8, 0xd804, 0xde9f, 0xdea8) \
>+    macro(0x112b0, 0x112de, 0xd804, 0xdeb0, 0xdede) \
>+    macro(0x11305, 0x1130c, 0xd804, 0xdf05, 0xdf0c) \
>+    macro(0x1130f, 0x11310, 0xd804, 0xdf0f, 0xdf10) \
>+    macro(0x11313, 0x11328, 0xd804, 0xdf13, 0xdf28) \
>+    macro(0x1132a, 0x11330, 0xd804, 0xdf2a, 0xdf30) \
>+    macro(0x11332, 0x11333, 0xd804, 0xdf32, 0xdf33) \
>+    macro(0x11335, 0x11339, 0xd804, 0xdf35, 0xdf39) \
>+    macro(0x1133d, 0x1133d, 0xd804, 0xdf3d, 0xdf3d) \
>+    macro(0x11350, 0x11350, 0xd804, 0xdf50, 0xdf50) \
>+    macro(0x1135d, 0x11361, 0xd804, 0xdf5d, 0xdf61) \
>+    macro(0x11400, 0x11434, 0xd805, 0xdc00, 0xdc34) \
>+    macro(0x11447, 0x1144a, 0xd805, 0xdc47, 0xdc4a) \
>+    macro(0x11480, 0x114af, 0xd805, 0xdc80, 0xdcaf) \
>+    macro(0x114c4, 0x114c5, 0xd805, 0xdcc4, 0xdcc5) \
>+    macro(0x114c7, 0x114c7, 0xd805, 0xdcc7, 0xdcc7) \
>+    macro(0x11580, 0x115ae, 0xd805, 0xdd80, 0xddae) \
>+    macro(0x115d8, 0x115db, 0xd805, 0xddd8, 0xdddb) \
>+    macro(0x11600, 0x1162f, 0xd805, 0xde00, 0xde2f) \
>+    macro(0x11644, 0x11644, 0xd805, 0xde44, 0xde44) \
>+    macro(0x11680, 0x116aa, 0xd805, 0xde80, 0xdeaa) \
>+    macro(0x11700, 0x11719, 0xd805, 0xdf00, 0xdf19) \
>+    macro(0x118a0, 0x118df, 0xd806, 0xdca0, 0xdcdf) \
>+    macro(0x118ff, 0x118ff, 0xd806, 0xdcff, 0xdcff) \
>+    macro(0x11ac0, 0x11af8, 0xd806, 0xdec0, 0xdef8) \
>+    macro(0x11c00, 0x11c08, 0xd807, 0xdc00, 0xdc08) \
>+    macro(0x11c0a, 0x11c2e, 0xd807, 0xdc0a, 0xdc2e) \
>+    macro(0x11c40, 0x11c40, 0xd807, 0xdc40, 0xdc40) \
>+    macro(0x11c72, 0x11c8f, 0xd807, 0xdc72, 0xdc8f) \
>+    macro(0x12000, 0x12399, 0xd808, 0xdc00, 0xdf99) \
>+    macro(0x12400, 0x1246e, 0xd809, 0xdc00, 0xdc6e) \
>+    macro(0x12480, 0x12543, 0xd809, 0xdc80, 0xdd43) \
>+    macro(0x13000, 0x133ff, 0xd80c, 0xdc00, 0xdfff) \
>+    macro(0x13400, 0x1342e, 0xd80d, 0xdc00, 0xdc2e) \
>+    macro(0x14400, 0x14646, 0xd811, 0xdc00, 0xde46) \
>+    macro(0x16800, 0x16a38, 0xd81a, 0xdc00, 0xde38) \
>+    macro(0x16a40, 0x16a5e, 0xd81a, 0xde40, 0xde5e) \
>+    macro(0x16ad0, 0x16aed, 0xd81a, 0xded0, 0xdeed) \
>+    macro(0x16b00, 0x16b2f, 0xd81a, 0xdf00, 0xdf2f) \
>+    macro(0x16b40, 0x16b43, 0xd81a, 0xdf40, 0xdf43) \
>+    macro(0x16b63, 0x16b77, 0xd81a, 0xdf63, 0xdf77) \
>+    macro(0x16b7d, 0x16b8f, 0xd81a, 0xdf7d, 0xdf8f) \
>+    macro(0x16f00, 0x16f44, 0xd81b, 0xdf00, 0xdf44) \
>+    macro(0x16f50, 0x16f50, 0xd81b, 0xdf50, 0xdf50) \
>+    macro(0x16f93, 0x16f9f, 0xd81b, 0xdf93, 0xdf9f) \
>+    macro(0x16fe0, 0x16fe0, 0xd81b, 0xdfe0, 0xdfe0) \
>+    macro(0x17000, 0x173ff, 0xd81c, 0xdc00, 0xdfff) \
>+    macro(0x17400, 0x177ff, 0xd81d, 0xdc00, 0xdfff) \
>+    macro(0x17800, 0x17bff, 0xd81e, 0xdc00, 0xdfff) \
>+    macro(0x17c00, 0x17fff, 0xd81f, 0xdc00, 0xdfff) \
>+    macro(0x18000, 0x183ff, 0xd820, 0xdc00, 0xdfff) \
>+    macro(0x18400, 0x187ec, 0xd821, 0xdc00, 0xdfec) \
>+    macro(0x18800, 0x18af2, 0xd822, 0xdc00, 0xdef2) \
>+    macro(0x1b000, 0x1b001, 0xd82c, 0xdc00, 0xdc01) \
>+    macro(0x1bc00, 0x1bc6a, 0xd82f, 0xdc00, 0xdc6a) \
>+    macro(0x1bc70, 0x1bc7c, 0xd82f, 0xdc70, 0xdc7c) \
>+    macro(0x1bc80, 0x1bc88, 0xd82f, 0xdc80, 0xdc88) \
>+    macro(0x1bc90, 0x1bc99, 0xd82f, 0xdc90, 0xdc99) \
>+    macro(0x1d400, 0x1d454, 0xd835, 0xdc00, 0xdc54) \
>+    macro(0x1d456, 0x1d49c, 0xd835, 0xdc56, 0xdc9c) \
>+    macro(0x1d49e, 0x1d49f, 0xd835, 0xdc9e, 0xdc9f) \
>+    macro(0x1d4a2, 0x1d4a2, 0xd835, 0xdca2, 0xdca2) \
>+    macro(0x1d4a5, 0x1d4a6, 0xd835, 0xdca5, 0xdca6) \
>+    macro(0x1d4a9, 0x1d4ac, 0xd835, 0xdca9, 0xdcac) \
>+    macro(0x1d4ae, 0x1d4b9, 0xd835, 0xdcae, 0xdcb9) \
>+    macro(0x1d4bb, 0x1d4bb, 0xd835, 0xdcbb, 0xdcbb) \
>+    macro(0x1d4bd, 0x1d4c3, 0xd835, 0xdcbd, 0xdcc3) \
>+    macro(0x1d4c5, 0x1d505, 0xd835, 0xdcc5, 0xdd05) \
>+    macro(0x1d507, 0x1d50a, 0xd835, 0xdd07, 0xdd0a) \
>+    macro(0x1d50d, 0x1d514, 0xd835, 0xdd0d, 0xdd14) \
>+    macro(0x1d516, 0x1d51c, 0xd835, 0xdd16, 0xdd1c) \
>+    macro(0x1d51e, 0x1d539, 0xd835, 0xdd1e, 0xdd39) \
>+    macro(0x1d53b, 0x1d53e, 0xd835, 0xdd3b, 0xdd3e) \
>+    macro(0x1d540, 0x1d544, 0xd835, 0xdd40, 0xdd44) \
>+    macro(0x1d546, 0x1d546, 0xd835, 0xdd46, 0xdd46) \
>+    macro(0x1d54a, 0x1d550, 0xd835, 0xdd4a, 0xdd50) \
>+    macro(0x1d552, 0x1d6a5, 0xd835, 0xdd52, 0xdea5) \
>+    macro(0x1d6a8, 0x1d6c0, 0xd835, 0xdea8, 0xdec0) \
>+    macro(0x1d6c2, 0x1d6da, 0xd835, 0xdec2, 0xdeda) \
>+    macro(0x1d6dc, 0x1d6fa, 0xd835, 0xdedc, 0xdefa) \
>+    macro(0x1d6fc, 0x1d714, 0xd835, 0xdefc, 0xdf14) \
>+    macro(0x1d716, 0x1d734, 0xd835, 0xdf16, 0xdf34) \
>+    macro(0x1d736, 0x1d74e, 0xd835, 0xdf36, 0xdf4e) \
>+    macro(0x1d750, 0x1d76e, 0xd835, 0xdf50, 0xdf6e) \
>+    macro(0x1d770, 0x1d788, 0xd835, 0xdf70, 0xdf88) \
>+    macro(0x1d78a, 0x1d7a8, 0xd835, 0xdf8a, 0xdfa8) \
>+    macro(0x1d7aa, 0x1d7c2, 0xd835, 0xdfaa, 0xdfc2) \
>+    macro(0x1d7c4, 0x1d7cb, 0xd835, 0xdfc4, 0xdfcb) \
>+    macro(0x1e800, 0x1e8c4, 0xd83a, 0xdc00, 0xdcc4) \
>+    macro(0x1e900, 0x1e943, 0xd83a, 0xdd00, 0xdd43) \
>+    macro(0x1ee00, 0x1ee03, 0xd83b, 0xde00, 0xde03) \
>+    macro(0x1ee05, 0x1ee1f, 0xd83b, 0xde05, 0xde1f) \
>+    macro(0x1ee21, 0x1ee22, 0xd83b, 0xde21, 0xde22) \
>+    macro(0x1ee24, 0x1ee24, 0xd83b, 0xde24, 0xde24) \
>+    macro(0x1ee27, 0x1ee27, 0xd83b, 0xde27, 0xde27) \
>+    macro(0x1ee29, 0x1ee32, 0xd83b, 0xde29, 0xde32) \
>+    macro(0x1ee34, 0x1ee37, 0xd83b, 0xde34, 0xde37) \
>+    macro(0x1ee39, 0x1ee39, 0xd83b, 0xde39, 0xde39) \
>+    macro(0x1ee3b, 0x1ee3b, 0xd83b, 0xde3b, 0xde3b) \
>+    macro(0x1ee42, 0x1ee42, 0xd83b, 0xde42, 0xde42) \
>+    macro(0x1ee47, 0x1ee47, 0xd83b, 0xde47, 0xde47) \
>+    macro(0x1ee49, 0x1ee49, 0xd83b, 0xde49, 0xde49) \
>+    macro(0x1ee4b, 0x1ee4b, 0xd83b, 0xde4b, 0xde4b) \
>+    macro(0x1ee4d, 0x1ee4f, 0xd83b, 0xde4d, 0xde4f) \
>+    macro(0x1ee51, 0x1ee52, 0xd83b, 0xde51, 0xde52) \
>+    macro(0x1ee54, 0x1ee54, 0xd83b, 0xde54, 0xde54) \
>+    macro(0x1ee57, 0x1ee57, 0xd83b, 0xde57, 0xde57) \
>+    macro(0x1ee59, 0x1ee59, 0xd83b, 0xde59, 0xde59) \
>+    macro(0x1ee5b, 0x1ee5b, 0xd83b, 0xde5b, 0xde5b) \
>+    macro(0x1ee5d, 0x1ee5d, 0xd83b, 0xde5d, 0xde5d) \
>+    macro(0x1ee5f, 0x1ee5f, 0xd83b, 0xde5f, 0xde5f) \
>+    macro(0x1ee61, 0x1ee62, 0xd83b, 0xde61, 0xde62) \
>+    macro(0x1ee64, 0x1ee64, 0xd83b, 0xde64, 0xde64) \
>+    macro(0x1ee67, 0x1ee6a, 0xd83b, 0xde67, 0xde6a) \
>+    macro(0x1ee6c, 0x1ee72, 0xd83b, 0xde6c, 0xde72) \
>+    macro(0x1ee74, 0x1ee77, 0xd83b, 0xde74, 0xde77) \
>+    macro(0x1ee79, 0x1ee7c, 0xd83b, 0xde79, 0xde7c) \
>+    macro(0x1ee7e, 0x1ee7e, 0xd83b, 0xde7e, 0xde7e) \
>+    macro(0x1ee80, 0x1ee89, 0xd83b, 0xde80, 0xde89) \
>+    macro(0x1ee8b, 0x1ee9b, 0xd83b, 0xde8b, 0xde9b) \
>+    macro(0x1eea1, 0x1eea3, 0xd83b, 0xdea1, 0xdea3) \
>+    macro(0x1eea5, 0x1eea9, 0xd83b, 0xdea5, 0xdea9) \
>+    macro(0x1eeab, 0x1eebb, 0xd83b, 0xdeab, 0xdebb) \
>+    macro(0x20000, 0x203ff, 0xd840, 0xdc00, 0xdfff) \
>+    macro(0x20400, 0x207ff, 0xd841, 0xdc00, 0xdfff) \
>+    macro(0x20800, 0x20bff, 0xd842, 0xdc00, 0xdfff) \
>+    macro(0x20c00, 0x20fff, 0xd843, 0xdc00, 0xdfff) \
>+    macro(0x21000, 0x213ff, 0xd844, 0xdc00, 0xdfff) \
>+    macro(0x21400, 0x217ff, 0xd845, 0xdc00, 0xdfff) \
>+    macro(0x21800, 0x21bff, 0xd846, 0xdc00, 0xdfff) \
>+    macro(0x21c00, 0x21fff, 0xd847, 0xdc00, 0xdfff) \
>+    macro(0x22000, 0x223ff, 0xd848, 0xdc00, 0xdfff) \
>+    macro(0x22400, 0x227ff, 0xd849, 0xdc00, 0xdfff) \
>+    macro(0x22800, 0x22bff, 0xd84a, 0xdc00, 0xdfff) \
>+    macro(0x22c00, 0x22fff, 0xd84b, 0xdc00, 0xdfff) \
>+    macro(0x23000, 0x233ff, 0xd84c, 0xdc00, 0xdfff) \
>+    macro(0x23400, 0x237ff, 0xd84d, 0xdc00, 0xdfff) \
>+    macro(0x23800, 0x23bff, 0xd84e, 0xdc00, 0xdfff) \
>+    macro(0x23c00, 0x23fff, 0xd84f, 0xdc00, 0xdfff) \
>+    macro(0x24000, 0x243ff, 0xd850, 0xdc00, 0xdfff) \
>+    macro(0x24400, 0x247ff, 0xd851, 0xdc00, 0xdfff) \
>+    macro(0x24800, 0x24bff, 0xd852, 0xdc00, 0xdfff) \
>+    macro(0x24c00, 0x24fff, 0xd853, 0xdc00, 0xdfff) \
>+    macro(0x25000, 0x253ff, 0xd854, 0xdc00, 0xdfff) \
>+    macro(0x25400, 0x257ff, 0xd855, 0xdc00, 0xdfff) \
>+    macro(0x25800, 0x25bff, 0xd856, 0xdc00, 0xdfff) \
>+    macro(0x25c00, 0x25fff, 0xd857, 0xdc00, 0xdfff) \
>+    macro(0x26000, 0x263ff, 0xd858, 0xdc00, 0xdfff) \
>+    macro(0x26400, 0x267ff, 0xd859, 0xdc00, 0xdfff) \
>+    macro(0x26800, 0x26bff, 0xd85a, 0xdc00, 0xdfff) \
>+    macro(0x26c00, 0x26fff, 0xd85b, 0xdc00, 0xdfff) \
>+    macro(0x27000, 0x273ff, 0xd85c, 0xdc00, 0xdfff) \
>+    macro(0x27400, 0x277ff, 0xd85d, 0xdc00, 0xdfff) \
>+    macro(0x27800, 0x27bff, 0xd85e, 0xdc00, 0xdfff) \
>+    macro(0x27c00, 0x27fff, 0xd85f, 0xdc00, 0xdfff) \
>+    macro(0x28000, 0x283ff, 0xd860, 0xdc00, 0xdfff) \
>+    macro(0x28400, 0x287ff, 0xd861, 0xdc00, 0xdfff) \
>+    macro(0x28800, 0x28bff, 0xd862, 0xdc00, 0xdfff) \
>+    macro(0x28c00, 0x28fff, 0xd863, 0xdc00, 0xdfff) \
>+    macro(0x29000, 0x293ff, 0xd864, 0xdc00, 0xdfff) \
>+    macro(0x29400, 0x297ff, 0xd865, 0xdc00, 0xdfff) \
>+    macro(0x29800, 0x29bff, 0xd866, 0xdc00, 0xdfff) \
>+    macro(0x29c00, 0x29fff, 0xd867, 0xdc00, 0xdfff) \
>+    macro(0x2a000, 0x2a3ff, 0xd868, 0xdc00, 0xdfff) \
>+    macro(0x2a400, 0x2a6d6, 0xd869, 0xdc00, 0xded6) \
>+    macro(0x2a700, 0x2a7ff, 0xd869, 0xdf00, 0xdfff) \
>+    macro(0x2a800, 0x2abff, 0xd86a, 0xdc00, 0xdfff) \
>+    macro(0x2ac00, 0x2afff, 0xd86b, 0xdc00, 0xdfff) \
>+    macro(0x2b000, 0x2b3ff, 0xd86c, 0xdc00, 0xdfff) \
>+    macro(0x2b400, 0x2b734, 0xd86d, 0xdc00, 0xdf34) \
>+    macro(0x2b740, 0x2b7ff, 0xd86d, 0xdf40, 0xdfff) \
>+    macro(0x2b800, 0x2b81d, 0xd86e, 0xdc00, 0xdc1d) \
>+    macro(0x2b820, 0x2bbff, 0xd86e, 0xdc20, 0xdfff) \
>+    macro(0x2bc00, 0x2bfff, 0xd86f, 0xdc00, 0xdfff) \
>+    macro(0x2c000, 0x2c3ff, 0xd870, 0xdc00, 0xdfff) \
>+    macro(0x2c400, 0x2c7ff, 0xd871, 0xdc00, 0xdfff) \
>+    macro(0x2c800, 0x2cbff, 0xd872, 0xdc00, 0xdfff) \
>+    macro(0x2cc00, 0x2cea1, 0xd873, 0xdc00, 0xdea1) \
>+    macro(0x2f800, 0x2fa1d, 0xd87e, 0xdc00, 0xde1d)
>+
>+#define FOR_EACH_NON_BMP_ID_CONT(macro) \
>+    macro(0x10000, 0x1000b, 0xd800, 0xdc00, 0xdc0b) \
>+    macro(0x1000d, 0x10026, 0xd800, 0xdc0d, 0xdc26) \
>+    macro(0x10028, 0x1003a, 0xd800, 0xdc28, 0xdc3a) \
>+    macro(0x1003c, 0x1003d, 0xd800, 0xdc3c, 0xdc3d) \
>+    macro(0x1003f, 0x1004d, 0xd800, 0xdc3f, 0xdc4d) \
>+    macro(0x10050, 0x1005d, 0xd800, 0xdc50, 0xdc5d) \
>+    macro(0x10080, 0x100fa, 0xd800, 0xdc80, 0xdcfa) \
>+    macro(0x10140, 0x10174, 0xd800, 0xdd40, 0xdd74) \
>+    macro(0x101fd, 0x101fd, 0xd800, 0xddfd, 0xddfd) \
>+    macro(0x10280, 0x1029c, 0xd800, 0xde80, 0xde9c) \
>+    macro(0x102a0, 0x102d0, 0xd800, 0xdea0, 0xded0) \
>+    macro(0x102e0, 0x102e0, 0xd800, 0xdee0, 0xdee0) \
>+    macro(0x10300, 0x1031f, 0xd800, 0xdf00, 0xdf1f) \
>+    macro(0x10330, 0x1034a, 0xd800, 0xdf30, 0xdf4a) \
>+    macro(0x10350, 0x1037a, 0xd800, 0xdf50, 0xdf7a) \
>+    macro(0x10380, 0x1039d, 0xd800, 0xdf80, 0xdf9d) \
>+    macro(0x103a0, 0x103c3, 0xd800, 0xdfa0, 0xdfc3) \
>+    macro(0x103c8, 0x103cf, 0xd800, 0xdfc8, 0xdfcf) \
>+    macro(0x103d1, 0x103d5, 0xd800, 0xdfd1, 0xdfd5) \
>+    macro(0x10400, 0x1049d, 0xd801, 0xdc00, 0xdc9d) \
>+    macro(0x104a0, 0x104a9, 0xd801, 0xdca0, 0xdca9) \
>+    macro(0x104b0, 0x104d3, 0xd801, 0xdcb0, 0xdcd3) \
>+    macro(0x104d8, 0x104fb, 0xd801, 0xdcd8, 0xdcfb) \
>+    macro(0x10500, 0x10527, 0xd801, 0xdd00, 0xdd27) \
>+    macro(0x10530, 0x10563, 0xd801, 0xdd30, 0xdd63) \
>+    macro(0x10600, 0x10736, 0xd801, 0xde00, 0xdf36) \
>+    macro(0x10740, 0x10755, 0xd801, 0xdf40, 0xdf55) \
>+    macro(0x10760, 0x10767, 0xd801, 0xdf60, 0xdf67) \
>+    macro(0x10800, 0x10805, 0xd802, 0xdc00, 0xdc05) \
>+    macro(0x10808, 0x10808, 0xd802, 0xdc08, 0xdc08) \
>+    macro(0x1080a, 0x10835, 0xd802, 0xdc0a, 0xdc35) \
>+    macro(0x10837, 0x10838, 0xd802, 0xdc37, 0xdc38) \
>+    macro(0x1083c, 0x1083c, 0xd802, 0xdc3c, 0xdc3c) \
>+    macro(0x1083f, 0x10855, 0xd802, 0xdc3f, 0xdc55) \
>+    macro(0x10860, 0x10876, 0xd802, 0xdc60, 0xdc76) \
>+    macro(0x10880, 0x1089e, 0xd802, 0xdc80, 0xdc9e) \
>+    macro(0x108e0, 0x108f2, 0xd802, 0xdce0, 0xdcf2) \
>+    macro(0x108f4, 0x108f5, 0xd802, 0xdcf4, 0xdcf5) \
>+    macro(0x10900, 0x10915, 0xd802, 0xdd00, 0xdd15) \
>+    macro(0x10920, 0x10939, 0xd802, 0xdd20, 0xdd39) \
>+    macro(0x10980, 0x109b7, 0xd802, 0xdd80, 0xddb7) \
>+    macro(0x109be, 0x109bf, 0xd802, 0xddbe, 0xddbf) \
>+    macro(0x10a00, 0x10a03, 0xd802, 0xde00, 0xde03) \
>+    macro(0x10a05, 0x10a06, 0xd802, 0xde05, 0xde06) \
>+    macro(0x10a0c, 0x10a13, 0xd802, 0xde0c, 0xde13) \
>+    macro(0x10a15, 0x10a17, 0xd802, 0xde15, 0xde17) \
>+    macro(0x10a19, 0x10a33, 0xd802, 0xde19, 0xde33) \
>+    macro(0x10a38, 0x10a3a, 0xd802, 0xde38, 0xde3a) \
>+    macro(0x10a3f, 0x10a3f, 0xd802, 0xde3f, 0xde3f) \
>+    macro(0x10a60, 0x10a7c, 0xd802, 0xde60, 0xde7c) \
>+    macro(0x10a80, 0x10a9c, 0xd802, 0xde80, 0xde9c) \
>+    macro(0x10ac0, 0x10ac7, 0xd802, 0xdec0, 0xdec7) \
>+    macro(0x10ac9, 0x10ae6, 0xd802, 0xdec9, 0xdee6) \
>+    macro(0x10b00, 0x10b35, 0xd802, 0xdf00, 0xdf35) \
>+    macro(0x10b40, 0x10b55, 0xd802, 0xdf40, 0xdf55) \
>+    macro(0x10b60, 0x10b72, 0xd802, 0xdf60, 0xdf72) \
>+    macro(0x10b80, 0x10b91, 0xd802, 0xdf80, 0xdf91) \
>+    macro(0x10c00, 0x10c48, 0xd803, 0xdc00, 0xdc48) \
>+    macro(0x10c80, 0x10cb2, 0xd803, 0xdc80, 0xdcb2) \
>+    macro(0x10cc0, 0x10cf2, 0xd803, 0xdcc0, 0xdcf2) \
>+    macro(0x11000, 0x11046, 0xd804, 0xdc00, 0xdc46) \
>+    macro(0x11066, 0x1106f, 0xd804, 0xdc66, 0xdc6f) \
>+    macro(0x1107f, 0x110ba, 0xd804, 0xdc7f, 0xdcba) \
>+    macro(0x110d0, 0x110e8, 0xd804, 0xdcd0, 0xdce8) \
>+    macro(0x110f0, 0x110f9, 0xd804, 0xdcf0, 0xdcf9) \
>+    macro(0x11100, 0x11134, 0xd804, 0xdd00, 0xdd34) \
>+    macro(0x11136, 0x1113f, 0xd804, 0xdd36, 0xdd3f) \
>+    macro(0x11150, 0x11173, 0xd804, 0xdd50, 0xdd73) \
>+    macro(0x11176, 0x11176, 0xd804, 0xdd76, 0xdd76) \
>+    macro(0x11180, 0x111c4, 0xd804, 0xdd80, 0xddc4) \
>+    macro(0x111ca, 0x111cc, 0xd804, 0xddca, 0xddcc) \
>+    macro(0x111d0, 0x111da, 0xd804, 0xddd0, 0xddda) \
>+    macro(0x111dc, 0x111dc, 0xd804, 0xdddc, 0xdddc) \
>+    macro(0x11200, 0x11211, 0xd804, 0xde00, 0xde11) \
>+    macro(0x11213, 0x11237, 0xd804, 0xde13, 0xde37) \
>+    macro(0x1123e, 0x1123e, 0xd804, 0xde3e, 0xde3e) \
>+    macro(0x11280, 0x11286, 0xd804, 0xde80, 0xde86) \
>+    macro(0x11288, 0x11288, 0xd804, 0xde88, 0xde88) \
>+    macro(0x1128a, 0x1128d, 0xd804, 0xde8a, 0xde8d) \
>+    macro(0x1128f, 0x1129d, 0xd804, 0xde8f, 0xde9d) \
>+    macro(0x1129f, 0x112a8, 0xd804, 0xde9f, 0xdea8) \
>+    macro(0x112b0, 0x112ea, 0xd804, 0xdeb0, 0xdeea) \
>+    macro(0x112f0, 0x112f9, 0xd804, 0xdef0, 0xdef9) \
>+    macro(0x11300, 0x11303, 0xd804, 0xdf00, 0xdf03) \
>+    macro(0x11305, 0x1130c, 0xd804, 0xdf05, 0xdf0c) \
>+    macro(0x1130f, 0x11310, 0xd804, 0xdf0f, 0xdf10) \
>+    macro(0x11313, 0x11328, 0xd804, 0xdf13, 0xdf28) \
>+    macro(0x1132a, 0x11330, 0xd804, 0xdf2a, 0xdf30) \
>+    macro(0x11332, 0x11333, 0xd804, 0xdf32, 0xdf33) \
>+    macro(0x11335, 0x11339, 0xd804, 0xdf35, 0xdf39) \
>+    macro(0x1133c, 0x11344, 0xd804, 0xdf3c, 0xdf44) \
>+    macro(0x11347, 0x11348, 0xd804, 0xdf47, 0xdf48) \
>+    macro(0x1134b, 0x1134d, 0xd804, 0xdf4b, 0xdf4d) \
>+    macro(0x11350, 0x11350, 0xd804, 0xdf50, 0xdf50) \
>+    macro(0x11357, 0x11357, 0xd804, 0xdf57, 0xdf57) \
>+    macro(0x1135d, 0x11363, 0xd804, 0xdf5d, 0xdf63) \
>+    macro(0x11366, 0x1136c, 0xd804, 0xdf66, 0xdf6c) \
>+    macro(0x11370, 0x11374, 0xd804, 0xdf70, 0xdf74) \
>+    macro(0x11400, 0x1144a, 0xd805, 0xdc00, 0xdc4a) \
>+    macro(0x11450, 0x11459, 0xd805, 0xdc50, 0xdc59) \
>+    macro(0x11480, 0x114c5, 0xd805, 0xdc80, 0xdcc5) \
>+    macro(0x114c7, 0x114c7, 0xd805, 0xdcc7, 0xdcc7) \
>+    macro(0x114d0, 0x114d9, 0xd805, 0xdcd0, 0xdcd9) \
>+    macro(0x11580, 0x115b5, 0xd805, 0xdd80, 0xddb5) \
>+    macro(0x115b8, 0x115c0, 0xd805, 0xddb8, 0xddc0) \
>+    macro(0x115d8, 0x115dd, 0xd805, 0xddd8, 0xdddd) \
>+    macro(0x11600, 0x11640, 0xd805, 0xde00, 0xde40) \
>+    macro(0x11644, 0x11644, 0xd805, 0xde44, 0xde44) \
>+    macro(0x11650, 0x11659, 0xd805, 0xde50, 0xde59) \
>+    macro(0x11680, 0x116b7, 0xd805, 0xde80, 0xdeb7) \
>+    macro(0x116c0, 0x116c9, 0xd805, 0xdec0, 0xdec9) \
>+    macro(0x11700, 0x11719, 0xd805, 0xdf00, 0xdf19) \
>+    macro(0x1171d, 0x1172b, 0xd805, 0xdf1d, 0xdf2b) \
>+    macro(0x11730, 0x11739, 0xd805, 0xdf30, 0xdf39) \
>+    macro(0x118a0, 0x118e9, 0xd806, 0xdca0, 0xdce9) \
>+    macro(0x118ff, 0x118ff, 0xd806, 0xdcff, 0xdcff) \
>+    macro(0x11ac0, 0x11af8, 0xd806, 0xdec0, 0xdef8) \
>+    macro(0x11c00, 0x11c08, 0xd807, 0xdc00, 0xdc08) \
>+    macro(0x11c0a, 0x11c36, 0xd807, 0xdc0a, 0xdc36) \
>+    macro(0x11c38, 0x11c40, 0xd807, 0xdc38, 0xdc40) \
>+    macro(0x11c50, 0x11c59, 0xd807, 0xdc50, 0xdc59) \
>+    macro(0x11c72, 0x11c8f, 0xd807, 0xdc72, 0xdc8f) \
>+    macro(0x11c92, 0x11ca7, 0xd807, 0xdc92, 0xdca7) \
>+    macro(0x11ca9, 0x11cb6, 0xd807, 0xdca9, 0xdcb6) \
>+    macro(0x12000, 0x12399, 0xd808, 0xdc00, 0xdf99) \
>+    macro(0x12400, 0x1246e, 0xd809, 0xdc00, 0xdc6e) \
>+    macro(0x12480, 0x12543, 0xd809, 0xdc80, 0xdd43) \
>+    macro(0x13000, 0x133ff, 0xd80c, 0xdc00, 0xdfff) \
>+    macro(0x13400, 0x1342e, 0xd80d, 0xdc00, 0xdc2e) \
>+    macro(0x14400, 0x14646, 0xd811, 0xdc00, 0xde46) \
>+    macro(0x16800, 0x16a38, 0xd81a, 0xdc00, 0xde38) \
>+    macro(0x16a40, 0x16a5e, 0xd81a, 0xde40, 0xde5e) \
>+    macro(0x16a60, 0x16a69, 0xd81a, 0xde60, 0xde69) \
>+    macro(0x16ad0, 0x16aed, 0xd81a, 0xded0, 0xdeed) \
>+    macro(0x16af0, 0x16af4, 0xd81a, 0xdef0, 0xdef4) \
>+    macro(0x16b00, 0x16b36, 0xd81a, 0xdf00, 0xdf36) \
>+    macro(0x16b40, 0x16b43, 0xd81a, 0xdf40, 0xdf43) \
>+    macro(0x16b50, 0x16b59, 0xd81a, 0xdf50, 0xdf59) \
>+    macro(0x16b63, 0x16b77, 0xd81a, 0xdf63, 0xdf77) \
>+    macro(0x16b7d, 0x16b8f, 0xd81a, 0xdf7d, 0xdf8f) \
>+    macro(0x16f00, 0x16f44, 0xd81b, 0xdf00, 0xdf44) \
>+    macro(0x16f50, 0x16f7e, 0xd81b, 0xdf50, 0xdf7e) \
>+    macro(0x16f8f, 0x16f9f, 0xd81b, 0xdf8f, 0xdf9f) \
>+    macro(0x16fe0, 0x16fe0, 0xd81b, 0xdfe0, 0xdfe0) \
>+    macro(0x17000, 0x173ff, 0xd81c, 0xdc00, 0xdfff) \
>+    macro(0x17400, 0x177ff, 0xd81d, 0xdc00, 0xdfff) \
>+    macro(0x17800, 0x17bff, 0xd81e, 0xdc00, 0xdfff) \
>+    macro(0x17c00, 0x17fff, 0xd81f, 0xdc00, 0xdfff) \
>+    macro(0x18000, 0x183ff, 0xd820, 0xdc00, 0xdfff) \
>+    macro(0x18400, 0x187ec, 0xd821, 0xdc00, 0xdfec) \
>+    macro(0x18800, 0x18af2, 0xd822, 0xdc00, 0xdef2) \
>+    macro(0x1b000, 0x1b001, 0xd82c, 0xdc00, 0xdc01) \
>+    macro(0x1bc00, 0x1bc6a, 0xd82f, 0xdc00, 0xdc6a) \
>+    macro(0x1bc70, 0x1bc7c, 0xd82f, 0xdc70, 0xdc7c) \
>+    macro(0x1bc80, 0x1bc88, 0xd82f, 0xdc80, 0xdc88) \
>+    macro(0x1bc90, 0x1bc99, 0xd82f, 0xdc90, 0xdc99) \
>+    macro(0x1bc9d, 0x1bc9e, 0xd82f, 0xdc9d, 0xdc9e) \
>+    macro(0x1d165, 0x1d169, 0xd834, 0xdd65, 0xdd69) \
>+    macro(0x1d16d, 0x1d172, 0xd834, 0xdd6d, 0xdd72) \
>+    macro(0x1d17b, 0x1d182, 0xd834, 0xdd7b, 0xdd82) \
>+    macro(0x1d185, 0x1d18b, 0xd834, 0xdd85, 0xdd8b) \
>+    macro(0x1d1aa, 0x1d1ad, 0xd834, 0xddaa, 0xddad) \
>+    macro(0x1d242, 0x1d244, 0xd834, 0xde42, 0xde44) \
>+    macro(0x1d400, 0x1d454, 0xd835, 0xdc00, 0xdc54) \
>+    macro(0x1d456, 0x1d49c, 0xd835, 0xdc56, 0xdc9c) \
>+    macro(0x1d49e, 0x1d49f, 0xd835, 0xdc9e, 0xdc9f) \
>+    macro(0x1d4a2, 0x1d4a2, 0xd835, 0xdca2, 0xdca2) \
>+    macro(0x1d4a5, 0x1d4a6, 0xd835, 0xdca5, 0xdca6) \
>+    macro(0x1d4a9, 0x1d4ac, 0xd835, 0xdca9, 0xdcac) \
>+    macro(0x1d4ae, 0x1d4b9, 0xd835, 0xdcae, 0xdcb9) \
>+    macro(0x1d4bb, 0x1d4bb, 0xd835, 0xdcbb, 0xdcbb) \
>+    macro(0x1d4bd, 0x1d4c3, 0xd835, 0xdcbd, 0xdcc3) \
>+    macro(0x1d4c5, 0x1d505, 0xd835, 0xdcc5, 0xdd05) \
>+    macro(0x1d507, 0x1d50a, 0xd835, 0xdd07, 0xdd0a) \
>+    macro(0x1d50d, 0x1d514, 0xd835, 0xdd0d, 0xdd14) \
>+    macro(0x1d516, 0x1d51c, 0xd835, 0xdd16, 0xdd1c) \
>+    macro(0x1d51e, 0x1d539, 0xd835, 0xdd1e, 0xdd39) \
>+    macro(0x1d53b, 0x1d53e, 0xd835, 0xdd3b, 0xdd3e) \
>+    macro(0x1d540, 0x1d544, 0xd835, 0xdd40, 0xdd44) \
>+    macro(0x1d546, 0x1d546, 0xd835, 0xdd46, 0xdd46) \
>+    macro(0x1d54a, 0x1d550, 0xd835, 0xdd4a, 0xdd50) \
>+    macro(0x1d552, 0x1d6a5, 0xd835, 0xdd52, 0xdea5) \
>+    macro(0x1d6a8, 0x1d6c0, 0xd835, 0xdea8, 0xdec0) \
>+    macro(0x1d6c2, 0x1d6da, 0xd835, 0xdec2, 0xdeda) \
>+    macro(0x1d6dc, 0x1d6fa, 0xd835, 0xdedc, 0xdefa) \
>+    macro(0x1d6fc, 0x1d714, 0xd835, 0xdefc, 0xdf14) \
>+    macro(0x1d716, 0x1d734, 0xd835, 0xdf16, 0xdf34) \
>+    macro(0x1d736, 0x1d74e, 0xd835, 0xdf36, 0xdf4e) \
>+    macro(0x1d750, 0x1d76e, 0xd835, 0xdf50, 0xdf6e) \
>+    macro(0x1d770, 0x1d788, 0xd835, 0xdf70, 0xdf88) \
>+    macro(0x1d78a, 0x1d7a8, 0xd835, 0xdf8a, 0xdfa8) \
>+    macro(0x1d7aa, 0x1d7c2, 0xd835, 0xdfaa, 0xdfc2) \
>+    macro(0x1d7c4, 0x1d7cb, 0xd835, 0xdfc4, 0xdfcb) \
>+    macro(0x1d7ce, 0x1d7ff, 0xd835, 0xdfce, 0xdfff) \
>+    macro(0x1da00, 0x1da36, 0xd836, 0xde00, 0xde36) \
>+    macro(0x1da3b, 0x1da6c, 0xd836, 0xde3b, 0xde6c) \
>+    macro(0x1da75, 0x1da75, 0xd836, 0xde75, 0xde75) \
>+    macro(0x1da84, 0x1da84, 0xd836, 0xde84, 0xde84) \
>+    macro(0x1da9b, 0x1da9f, 0xd836, 0xde9b, 0xde9f) \
>+    macro(0x1daa1, 0x1daaf, 0xd836, 0xdea1, 0xdeaf) \
>+    macro(0x1e000, 0x1e006, 0xd838, 0xdc00, 0xdc06) \
>+    macro(0x1e008, 0x1e018, 0xd838, 0xdc08, 0xdc18) \
>+    macro(0x1e01b, 0x1e021, 0xd838, 0xdc1b, 0xdc21) \
>+    macro(0x1e023, 0x1e024, 0xd838, 0xdc23, 0xdc24) \
>+    macro(0x1e026, 0x1e02a, 0xd838, 0xdc26, 0xdc2a) \
>+    macro(0x1e800, 0x1e8c4, 0xd83a, 0xdc00, 0xdcc4) \
>+    macro(0x1e8d0, 0x1e8d6, 0xd83a, 0xdcd0, 0xdcd6) \
>+    macro(0x1e900, 0x1e94a, 0xd83a, 0xdd00, 0xdd4a) \
>+    macro(0x1e950, 0x1e959, 0xd83a, 0xdd50, 0xdd59) \
>+    macro(0x1ee00, 0x1ee03, 0xd83b, 0xde00, 0xde03) \
>+    macro(0x1ee05, 0x1ee1f, 0xd83b, 0xde05, 0xde1f) \
>+    macro(0x1ee21, 0x1ee22, 0xd83b, 0xde21, 0xde22) \
>+    macro(0x1ee24, 0x1ee24, 0xd83b, 0xde24, 0xde24) \
>+    macro(0x1ee27, 0x1ee27, 0xd83b, 0xde27, 0xde27) \
>+    macro(0x1ee29, 0x1ee32, 0xd83b, 0xde29, 0xde32) \
>+    macro(0x1ee34, 0x1ee37, 0xd83b, 0xde34, 0xde37) \
>+    macro(0x1ee39, 0x1ee39, 0xd83b, 0xde39, 0xde39) \
>+    macro(0x1ee3b, 0x1ee3b, 0xd83b, 0xde3b, 0xde3b) \
>+    macro(0x1ee42, 0x1ee42, 0xd83b, 0xde42, 0xde42) \
>+    macro(0x1ee47, 0x1ee47, 0xd83b, 0xde47, 0xde47) \
>+    macro(0x1ee49, 0x1ee49, 0xd83b, 0xde49, 0xde49) \
>+    macro(0x1ee4b, 0x1ee4b, 0xd83b, 0xde4b, 0xde4b) \
>+    macro(0x1ee4d, 0x1ee4f, 0xd83b, 0xde4d, 0xde4f) \
>+    macro(0x1ee51, 0x1ee52, 0xd83b, 0xde51, 0xde52) \
>+    macro(0x1ee54, 0x1ee54, 0xd83b, 0xde54, 0xde54) \
>+    macro(0x1ee57, 0x1ee57, 0xd83b, 0xde57, 0xde57) \
>+    macro(0x1ee59, 0x1ee59, 0xd83b, 0xde59, 0xde59) \
>+    macro(0x1ee5b, 0x1ee5b, 0xd83b, 0xde5b, 0xde5b) \
>+    macro(0x1ee5d, 0x1ee5d, 0xd83b, 0xde5d, 0xde5d) \
>+    macro(0x1ee5f, 0x1ee5f, 0xd83b, 0xde5f, 0xde5f) \
>+    macro(0x1ee61, 0x1ee62, 0xd83b, 0xde61, 0xde62) \
>+    macro(0x1ee64, 0x1ee64, 0xd83b, 0xde64, 0xde64) \
>+    macro(0x1ee67, 0x1ee6a, 0xd83b, 0xde67, 0xde6a) \
>+    macro(0x1ee6c, 0x1ee72, 0xd83b, 0xde6c, 0xde72) \
>+    macro(0x1ee74, 0x1ee77, 0xd83b, 0xde74, 0xde77) \
>+    macro(0x1ee79, 0x1ee7c, 0xd83b, 0xde79, 0xde7c) \
>+    macro(0x1ee7e, 0x1ee7e, 0xd83b, 0xde7e, 0xde7e) \
>+    macro(0x1ee80, 0x1ee89, 0xd83b, 0xde80, 0xde89) \
>+    macro(0x1ee8b, 0x1ee9b, 0xd83b, 0xde8b, 0xde9b) \
>+    macro(0x1eea1, 0x1eea3, 0xd83b, 0xdea1, 0xdea3) \
>+    macro(0x1eea5, 0x1eea9, 0xd83b, 0xdea5, 0xdea9) \
>+    macro(0x1eeab, 0x1eebb, 0xd83b, 0xdeab, 0xdebb) \
>+    macro(0x20000, 0x203ff, 0xd840, 0xdc00, 0xdfff) \
>+    macro(0x20400, 0x207ff, 0xd841, 0xdc00, 0xdfff) \
>+    macro(0x20800, 0x20bff, 0xd842, 0xdc00, 0xdfff) \
>+    macro(0x20c00, 0x20fff, 0xd843, 0xdc00, 0xdfff) \
>+    macro(0x21000, 0x213ff, 0xd844, 0xdc00, 0xdfff) \
>+    macro(0x21400, 0x217ff, 0xd845, 0xdc00, 0xdfff) \
>+    macro(0x21800, 0x21bff, 0xd846, 0xdc00, 0xdfff) \
>+    macro(0x21c00, 0x21fff, 0xd847, 0xdc00, 0xdfff) \
>+    macro(0x22000, 0x223ff, 0xd848, 0xdc00, 0xdfff) \
>+    macro(0x22400, 0x227ff, 0xd849, 0xdc00, 0xdfff) \
>+    macro(0x22800, 0x22bff, 0xd84a, 0xdc00, 0xdfff) \
>+    macro(0x22c00, 0x22fff, 0xd84b, 0xdc00, 0xdfff) \
>+    macro(0x23000, 0x233ff, 0xd84c, 0xdc00, 0xdfff) \
>+    macro(0x23400, 0x237ff, 0xd84d, 0xdc00, 0xdfff) \
>+    macro(0x23800, 0x23bff, 0xd84e, 0xdc00, 0xdfff) \
>+    macro(0x23c00, 0x23fff, 0xd84f, 0xdc00, 0xdfff) \
>+    macro(0x24000, 0x243ff, 0xd850, 0xdc00, 0xdfff) \
>+    macro(0x24400, 0x247ff, 0xd851, 0xdc00, 0xdfff) \
>+    macro(0x24800, 0x24bff, 0xd852, 0xdc00, 0xdfff) \
>+    macro(0x24c00, 0x24fff, 0xd853, 0xdc00, 0xdfff) \
>+    macro(0x25000, 0x253ff, 0xd854, 0xdc00, 0xdfff) \
>+    macro(0x25400, 0x257ff, 0xd855, 0xdc00, 0xdfff) \
>+    macro(0x25800, 0x25bff, 0xd856, 0xdc00, 0xdfff) \
>+    macro(0x25c00, 0x25fff, 0xd857, 0xdc00, 0xdfff) \
>+    macro(0x26000, 0x263ff, 0xd858, 0xdc00, 0xdfff) \
>+    macro(0x26400, 0x267ff, 0xd859, 0xdc00, 0xdfff) \
>+    macro(0x26800, 0x26bff, 0xd85a, 0xdc00, 0xdfff) \
>+    macro(0x26c00, 0x26fff, 0xd85b, 0xdc00, 0xdfff) \
>+    macro(0x27000, 0x273ff, 0xd85c, 0xdc00, 0xdfff) \
>+    macro(0x27400, 0x277ff, 0xd85d, 0xdc00, 0xdfff) \
>+    macro(0x27800, 0x27bff, 0xd85e, 0xdc00, 0xdfff) \
>+    macro(0x27c00, 0x27fff, 0xd85f, 0xdc00, 0xdfff) \
>+    macro(0x28000, 0x283ff, 0xd860, 0xdc00, 0xdfff) \
>+    macro(0x28400, 0x287ff, 0xd861, 0xdc00, 0xdfff) \
>+    macro(0x28800, 0x28bff, 0xd862, 0xdc00, 0xdfff) \
>+    macro(0x28c00, 0x28fff, 0xd863, 0xdc00, 0xdfff) \
>+    macro(0x29000, 0x293ff, 0xd864, 0xdc00, 0xdfff) \
>+    macro(0x29400, 0x297ff, 0xd865, 0xdc00, 0xdfff) \
>+    macro(0x29800, 0x29bff, 0xd866, 0xdc00, 0xdfff) \
>+    macro(0x29c00, 0x29fff, 0xd867, 0xdc00, 0xdfff) \
>+    macro(0x2a000, 0x2a3ff, 0xd868, 0xdc00, 0xdfff) \
>+    macro(0x2a400, 0x2a6d6, 0xd869, 0xdc00, 0xded6) \
>+    macro(0x2a700, 0x2a7ff, 0xd869, 0xdf00, 0xdfff) \
>+    macro(0x2a800, 0x2abff, 0xd86a, 0xdc00, 0xdfff) \
>+    macro(0x2ac00, 0x2afff, 0xd86b, 0xdc00, 0xdfff) \
>+    macro(0x2b000, 0x2b3ff, 0xd86c, 0xdc00, 0xdfff) \
>+    macro(0x2b400, 0x2b734, 0xd86d, 0xdc00, 0xdf34) \
>+    macro(0x2b740, 0x2b7ff, 0xd86d, 0xdf40, 0xdfff) \
>+    macro(0x2b800, 0x2b81d, 0xd86e, 0xdc00, 0xdc1d) \
>+    macro(0x2b820, 0x2bbff, 0xd86e, 0xdc20, 0xdfff) \
>+    macro(0x2bc00, 0x2bfff, 0xd86f, 0xdc00, 0xdfff) \
>+    macro(0x2c000, 0x2c3ff, 0xd870, 0xdc00, 0xdfff) \
>+    macro(0x2c400, 0x2c7ff, 0xd871, 0xdc00, 0xdfff) \
>+    macro(0x2c800, 0x2cbff, 0xd872, 0xdc00, 0xdfff) \
>+    macro(0x2cc00, 0x2cea1, 0xd873, 0xdc00, 0xdea1) \
>+    macro(0x2f800, 0x2fa1d, 0xd87e, 0xdc00, 0xde1d) \
>+    macro(0xe0100, 0xe01ef, 0xdb40, 0xdd00, 0xddef)
>+
> #endif /* vm_UnicodeNonBMP_h */
>diff --git a/js/src/vm/make_unicode.py b/js/src/vm/make_unicode.py
>--- a/js/src/vm/make_unicode.py
>+++ b/js/src/vm/make_unicode.py
>@@ -142,39 +142,81 @@ def utf16_encode(code):
>     trail = ((code - NonBMPMin) % 1024) + TrailSurrogateMin
> 
>     return lead, trail
> 
> def make_non_bmp_convert_macro(out_file, name, convert_map):
>     convert_list = []
>     entry = None
>     for code in sorted(convert_map.keys()):
>+        lead, trail = utf16_encode(code)
>         converted = convert_map[code]
>         diff = converted - code
> 
>-        if entry and code == entry['code'] + entry['length'] and diff == entry['diff']:
>+        if entry and code == entry['code'] + entry['length'] and \
>+           diff == entry['diff'] and lead == entry['lead']:
>             entry['length'] += 1
>             continue
> 
>-        entry = { 'code': code, 'diff': diff, 'length': 1 }
>+        entry = {
>+            'code': code,
>+            'diff': diff,
>+            'length': 1,
>+            'lead': lead,
>+            'trail': trail,
>+        }
>         convert_list.append(entry)
> 
>     lines = []
>     for entry in convert_list:
>         from_code = entry['code']
>         to_code = entry['code'] + entry['length'] - 1
>         diff = entry['diff']
> 
>-        from_lead, from_trail = utf16_encode(from_code)
>-        to_lead, to_trail = utf16_encode(to_code)
>-
>-        assert from_lead == to_lead
>+        lead = entry['lead']
>+        from_trail = entry['trail']
>+        to_trail = entry['trail'] + entry['length'] - 1
> 
>         lines.append('    macro(0x{:x}, 0x{:x}, 0x{:x}, 0x{:x}, 0x{:x}, {:d})'.format(
>-            from_code, to_code, from_lead, from_trail, to_trail, diff))
>+            from_code, to_code, lead, from_trail, to_trail, diff))
>+
>+    out_file.write('#define FOR_EACH_NON_BMP_{}(macro) \\\n'.format(name))
>+    out_file.write(' \\\n'.join(lines))
>+    out_file.write('\n')
>+
>+def make_non_bmp_group_macro(out_file, name, group_set):
>+    group_list = []
>+    entry = None
>+    for code in sorted(group_set.keys()):
>+        lead, trail = utf16_encode(code)
>+
>+        if entry and code == entry['code'] + entry['length'] and \
>+           lead == entry['lead']:
>+            entry['length'] += 1
>+            continue
>+
>+        entry = {
>+            'code': code,
>+            'length': 1,
>+            'lead': lead,
>+            'trail': trail
>+        }
>+        group_list.append(entry)
>+
>+    lines = []
>+    for entry in group_list:
>+        from_code = entry['code']
>+        to_code = entry['code'] + entry['length'] - 1
>+
>+        lead = entry['lead']
>+        from_trail = entry['trail']
>+        to_trail = entry['trail'] + entry['length'] - 1
>+
>+        lines.append('    macro(0x{:x}, 0x{:x}, 0x{:x}, 0x{:x}, 0x{:x})'.format(
>+            from_code, to_code, lead, from_trail, to_trail))
> 
>     out_file.write('#define FOR_EACH_NON_BMP_{}(macro) \\\n'.format(name))
>     out_file.write(' \\\n'.join(lines))
>     out_file.write('\n')
> 
> def process_derived_core_properties(derived_core_properties):
>     id_start = set()
>     id_continue = set()
>@@ -198,16 +240,19 @@ def process_unicode_data(unicode_data, d
>     same_upper_cache = {same_upper_dummy: 0}
>     same_upper_index = [0] * (MAX_BMP + 1)
> 
>     test_table = {}
>     test_space_table = []
> 
>     non_bmp_lower_map = {}
>     non_bmp_upper_map = {}
>+    non_bmp_id_start_set = {}
>+    non_bmp_id_cont_set = {}
>+    non_bmp_space_set = {}
> 
>     (id_start, id_continue) = process_derived_core_properties(derived_core_properties)
> 
>     for row in read_unicode_data(unicode_data):
>         code = row[0]
>         name = row[1]
>         category = row[2]
>         alias = row[-5]
>@@ -230,16 +275,23 @@ def process_unicode_data(unicode_data, d
>         else:
>             lower = code
> 
>         if code > MAX_BMP:
>             if code != lower:
>                 non_bmp_lower_map[code] = lower
>             if code != upper:
>                 non_bmp_upper_map[code] = upper
>+            if category == 'Zs':
>+                non_bmp_space_set[code] = 1
>+                test_space_table.append(code)
>+            if code in id_start:
>+                non_bmp_id_start_set[code] = 1
>+            if code in id_continue:
>+                non_bmp_id_cont_set[code] = 1
>             continue
> 
>         # we combine whitespace and lineterminators because in pratice we don't need them separated
>         if category == 'Zs' or code in whitespace or code in line_terminator:
>             flags |= FLAG_SPACE
>             test_space_table.append(code)
> 
>         # Â§11.6 (IdentifierStart)
>@@ -299,16 +351,18 @@ def process_unicode_data(unicode_data, d
>             same_upper_cache[item] = i = len(same_upper_table)
>             same_upper_table.append(item)
>         same_upper_index[code] = i
> 
>     return (
>         table, index,
>         same_upper_table, same_upper_index,
>         non_bmp_lower_map, non_bmp_upper_map,
>+        non_bmp_space_set,
>+        non_bmp_id_start_set, non_bmp_id_cont_set,
>         test_table, test_space_table,
>     )
> 
> def process_case_folding(case_folding):
>     folding_map = {}
>     rev_folding_map = {}
>     folding_dummy = (0, 0, 0, 0)
>     folding_table = [folding_dummy]
>@@ -385,16 +439,18 @@ def process_case_folding(case_folding):
>     return (
>         folding_table, folding_index,
>         non_bmp_folding_map, non_bmp_rev_folding_map,
>         folding_tests
>     )
> 
> def make_non_bmp_file(version,
>                       non_bmp_lower_map, non_bmp_upper_map,
>+                      non_bmp_space_set,
>+                      non_bmp_id_start_set, non_bmp_id_cont_set,
>                       non_bmp_folding_map, non_bmp_rev_folding_map):
>     file_name = 'UnicodeNonBMP.h';
>     with io.open(file_name, mode='wb') as non_bmp_file:
>         non_bmp_file.write(mpl_license)
>         non_bmp_file.write('\n')
>         non_bmp_file.write(warning_message)
>         non_bmp_file.write(unicode_version_message.format(version))
>         non_bmp_file.write("""
>@@ -405,16 +461,27 @@ def make_non_bmp_file(version,
> 
>         make_non_bmp_convert_macro(non_bmp_file, 'LOWERCASE', non_bmp_lower_map)
>         non_bmp_file.write('\n')
>         make_non_bmp_convert_macro(non_bmp_file, 'UPPERCASE', non_bmp_upper_map)
>         non_bmp_file.write('\n')
>         make_non_bmp_convert_macro(non_bmp_file, 'CASE_FOLDING', non_bmp_folding_map)
>         non_bmp_file.write('\n')
>         make_non_bmp_convert_macro(non_bmp_file, 'REV_CASE_FOLDING', non_bmp_rev_folding_map)
>+        non_bmp_file.write('\n')
>+
>+        # If the following assert fails, it means space character is added to
>+        # non-BMP area.  In that case the following code should be uncommented
>+        # and the corresponding code should be added to frontend.
>+        assert len(non_bmp_space_set.keys()) == 0
>+        # make_non_bmp_group_macro(non_bmp_file, 'SPACE', non_bmp_space_set)
>+
>+        make_non_bmp_group_macro(non_bmp_file, 'ID_START', non_bmp_id_start_set)
>+        non_bmp_file.write('\n')
>+        make_non_bmp_group_macro(non_bmp_file, 'ID_CONT', non_bmp_id_cont_set)
> 
>         non_bmp_file.write("""
> #endif /* vm_UnicodeNonBMP_h */
> """)
> 
> def make_bmp_mapping_test(version, test_table):
>     file_name = '../tests/ecma_5/String/string-upper-lower-mapping.js'
>     with io.open(file_name, mode='wb') as test_mapping:
>@@ -786,31 +853,35 @@ def update_unicode(args):
>          download_or_open('DerivedCoreProperties.txt') as derived_core_properties:
>         unicode_version = version_from_file(derived_core_properties, 'DerivedCoreProperties')
> 
>         print('Processing...')
>         (
>             table, index,
>             same_upper_table, same_upper_index,
>             non_bmp_lower_map, non_bmp_upper_map,
>+            non_bmp_space_set,
>+            non_bmp_id_start_set, non_bmp_id_cont_set,
>             test_table, test_space_table
>         ) = process_unicode_data(unicode_data, derived_core_properties)
>         (
>             folding_table, folding_index,
>             non_bmp_folding_map, non_bmp_rev_folding_map,
>             folding_tests
>         ) = process_case_folding(case_folding)
> 
>     print('Generating...')
>     make_unicode_file(unicode_version,
>                       table, index,
>                       same_upper_table, same_upper_index,
>                       folding_table, folding_index)
>     make_non_bmp_file(unicode_version,
>                       non_bmp_lower_map, non_bmp_upper_map,
>+                      non_bmp_space_set,
>+                      non_bmp_id_start_set, non_bmp_id_cont_set,
>                       non_bmp_folding_map, non_bmp_rev_folding_map)
> 
>     make_bmp_mapping_test(unicode_version, test_table)
>     make_non_bmp_mapping_test(unicode_version, non_bmp_upper_map, non_bmp_lower_map)
>     make_space_test(unicode_version, test_space_table)
>     make_icase_test(unicode_version, folding_tests)
> 
> if __name__ == '__main__':

Flags:

shu: review+

Actions: View | Diff | Review

Attachments on bug 1197230: 8809621 | 8812411