Decode JavaScript Unicode code point escapes during extraction

babel/messages/jslexer.py

@@ -31,6 +31,7 @@
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
+codepoint_escape_re = re.compile(r'\{([a-fA-F0-9]+)\}')
 hex_escape_re = re.compile(r'[a-fA-F0-9]{1,2}')
 
 
@@ -125,6 +126,14 @@ def unquote_string(string: str) -> str:
         # character point.  If there is no such character point, put
         # all the consumed characters into the string.
         elif next_char in 'uU':
+            if next_char == 'u':
+                codepoint = codepoint_escape_re.match(string, escape_pos + 2)
+                if codepoint is not None:
+                    value = int(codepoint.group(1), 16)
+                    if value <= 0x10ffff:
+                        add(chr(value))
+                        pos = codepoint.end()
+                        continue
             escaped = uni_escape_re.match(string, escape_pos + 2)
             if escaped is not None:
                 escaped_value = escaped.group()

tests/messages/test_js_unicode_codepoints.py

@@ -0,0 +1,40 @@
+from io import BytesIO
+
+import pytest
+
+from babel.messages.extract import extract
+from babel.messages.jslexer import unquote_string
+
+
+@pytest.mark.parametrize('quote', ["'", '"', '`'])
+@pytest.mark.parametrize(('body', 'expected'), [
+    (r'a\u{0}b', 'a\0b'),
+    (r'\u{41}', 'A'),
+    (r'\u{e9}', 'é'),
+    (r'\u{1F600}', '😀'),
+    (r'\u{10FFFF}', '\U0010ffff'),
+    (r'\u{000000000041}', 'A'),
+])
+def test_unicode_codepoint_escapes(quote, body, expected):
+    assert unquote_string(quote + body + quote) == expected
+
+
+@pytest.mark.parametrize('quote', ["'", '"', '`'])
+def test_unicode_codepoint_extraction_uses_runtime_message(quote):
+    source = f'gettext({quote}Hello \\u{{1F600}}{quote});\ngettext("next");'
+    messages = list(extract('javascript', BytesIO(source.encode()), options={}))
+    assert messages == [(1, 'Hello 😀', [], None), (2, 'next', [], None)]
+
+
+@pytest.mark.parametrize(('body', 'expected'), [
+    (r'\u{1f600}\u{1f680}', '😀🚀'),
+    (r'\u00e9\u{1f600}\x21', 'é😀!'),
+    (r'\\u{1F600}', r'\u{1F600}'),
+    (r'\U{41}', 'U{41}'),
+    (r'\u{}', 'u{}'),
+    (r'\u{xyz}', 'u{xyz}'),
+    (r'\u{110000}', 'u{110000}'),
+    (r'\u{41', 'u{41'),
+])
+def test_codepoint_escapes_preserve_existing_escape_behavior(body, expected):
+    assert unquote_string('"' + body + '"') == expected