parser: allow non-UTF-8 bytes in literals and regexes.

* parser.l (grammar): Just like we do in SREGEX, allow an
arbitrary byte in REGEX, mapping it to the DCxx range.
Do the same inside string literals of all types.

* lex.yy.c.shipped: Updated.

* tests/012/parse.tl: New tests.

1 files changed, 3 insertions, 11 deletions

diff --git a/parser.l b/parser.l
index d7e53c49..98cdf344 100644
--- a/parser.l
+++ b/parser.l
@@ -945,18 +945,11 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return REGCHAR;
 }
 
-<SREGEX>. {
-  /* Allow non-UTF-8 byte for regexes scanned from string */
+<SREGEX,REGEX>. {
   yylval->chr = convert(unsigned char, yytext[0]) + 0xDC00;
   return REGCHAR;
 }
 
-<REGEX>. {
-  yyerrprepf(yyg, lit("non-UTF-8 byte in regex: '\\x~02x'"),
-             num(convert(unsigned char, yytext[0])), nao);
-  return ERRTOK;
-}
-
 <INITIAL>[ ]+ {
   yylval->lexeme = utf8_dup_from(yytext);
   return SPACE;
@@ -1128,9 +1121,8 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <STRLIT,CHRLIT,QSILIT,WLIT,QWLIT>. {
-  yyerrprepf(yyg, lit("non-UTF-8 byte in literal: '\\x~02x'"),
-             num(convert(unsigned char, yytext[0])), nao);
-  return ERRTOK;
+  yylval->chr = convert(unsigned char, yytext[0]) + 0xDC00;
+  return LITCHAR;
 }
 
 %%