Support trailing semicolon after hex/octal characters.

* parser.l (%option): Remove nounput option since we need
yyunput.
(grammar): Rule for matching hex and octal escape in SPECIAL
state recognizes optional semicolon. In 109 compatibility,
this is pushed back into the stream, otherwise consumed.

* txr.1: Updated documentation, including compat notes.

* genvim.txr (txr_char): Include optional semicolon in
match. Corrected some errors where 8 and 9 were being
included as matches for octal digits.
(txr_error): Default match for \x or \o  not followed
by digits.

1 files changed, 9 insertions, 2 deletions

diff --git a/parser.l b/parser.l
index 37bbdc70..30cb3034 100644
--- a/parser.l
+++ b/parser.l
@@ -166,7 +166,7 @@ static wchar_t num_esc(scanner_t *scn, char *num)
 
 %}
 
-%option stack nounput noinput reentrant bison-bridge extra-type="parser_t *"
+%option stack noinput reentrant bison-bridge extra-type="parser_t *"
 
 SYM     [a-zA-Z0-9_]+
 SGN     [+\-]
@@ -685,11 +685,18 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return TEXT;
 }
 
-<SPECIAL>[\\](x{HEX}+|{OCT}+) {
+<SPECIAL>[\\](x{HEX}+|{OCT}+);? {
   wchar_t lexeme[2];
   lexeme[0] = num_esc(yyg, yytext + 1);
   lexeme[1] = 0;
   yylval->lexeme = chk_strdup(lexeme);
+
+  {
+    char lastchar = yytext[yyleng-1];
+    if (lastchar == ';' && opt_compat && opt_compat <= 109)
+      unput(lastchar);
+  }
+
   yy_pop_state(yyscanner);
   return TEXT;
 }