Fixing gaping bug in the handling of @-delimited expressions

within quasiliterals. This has been a problem for years.
Quasiliteral strings existed very early before TXR Lisp was introduced.
So it made sense that when @ is seen in a quasiliteral, the
lexical analyzer pushed into the SPECIAL state in which directives
are recognized, like in the pattern language. I noticed
this because there is an @(if) directive now, which prevents
`@(if ...)` from being valid.

* parser.l (QSPECIAL): New scanner state. This is a state
similar to SPECIAL that we enter into when @ is seen in a QSLIT state.
In this state we recognize constructs like braced variables, but not
certain other features like directives.

2 files changed, 40 insertions, 24 deletions

diff --git a/ChangeLog b/ChangeLog
index f3eaa15a..b3ca4bb5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,21 @@
 2014-03-19  Kaz Kylheku  <kaz@kylheku.com>
 
+	Fixing gaping bug in the handling of @-delimited expressions
+	within quasiliterals. This has been a problem for years.
+	Quasiliteral strings existed very early before TXR Lisp was introduced.
+	So it made sense that when @ is seen in a quasiliteral, the
+	lexical analyzer pushed into the SPECIAL state in which directives
+	are recognized, like in the pattern language. I noticed
+	this because there is an @(if) directive now, which prevents
+	`@(if ...)` from being valid.
+
+	* parser.l (QSPECIAL): New scanner state. This is a state
+	similar to SPECIAL that we enter into when @ is seen in a QSLIT state.
+	In this state we recognize constructs like braced variables, but not
+	certain other features like directives.
+
+2014-03-19  Kaz Kylheku  <kaz@kylheku.com>
+
 	* arith.c (int_flo): Fix non-handling of negative values.
 
 2014-03-19  Kaz Kylheku  <kaz@kylheku.com>
diff --git a/parser.l b/parser.l
index 335650b5..d678100a 100644
--- a/parser.l
+++ b/parser.l
@@ -190,11 +190,11 @@ UANY    {ASC}|{U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 UANYN   {ASCN}|{U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U} 
 UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 
-%x      SPECIAL BRACED NESTED REGEX STRLIT CHRLIT QSILIT
+%x      SPECIAL BRACED NESTED REGEX STRLIT CHRLIT QSILIT QSPECIAL
 
 %%
 
-<SPECIAL,NESTED,BRACED>{NUM} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{NUM} {
   val str = string_own(utf8_dup_from(yytext));
 
   if (yy_top_state() == INITIAL
@@ -205,7 +205,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL,NESTED,BRACED>{XNUM} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{XNUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
   if (yy_top_state() == INITIAL
@@ -216,7 +216,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL,NESTED,BRACED>{ONUM} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{ONUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
   if (yy_top_state() == INITIAL
@@ -227,7 +227,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL,NESTED,BRACED>{BNUM} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{BNUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
   if (yy_top_state() == INITIAL
@@ -238,7 +238,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL,NESTED,BRACED>{FLO} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{FLO} {
   val str = string_own(utf8_dup_from(yytext));
 
   if (yy_top_state() == INITIAL
@@ -249,7 +249,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL,NESTED,BRACED>{FLODOT}/[^.] {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{FLODOT}/[^.] {
   val str = string_own(utf8_dup_from(yytext));
 
   if (yy_top_state() == INITIAL
@@ -260,7 +260,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return NUMBER;
 }
 
-<SPECIAL>({FLO}|{FLODOT}){TOK} |
+<SPECIAL,QSPECIAL>({FLO}|{FLODOT}){TOK} |
 <BRACED>({FLO}|{FLODOT}){BTOK} |
 <NESTED>({FLO}|{FLODOT}){NTOK} {
   val str = string_utf8(yytext);
@@ -315,7 +315,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return METANUM;
 }
 
-<SPECIAL>{TOK} |
+<SPECIAL,QSPECIAL>{TOK} |
 <BRACED>{BTOK} |
 <NESTED>{NTOK} {
   if (yy_top_state() == INITIAL
@@ -507,13 +507,13 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return ELSE;
 }
 
-<SPECIAL>[{] {
+<SPECIAL,QSPECIAL>[{] {
   yy_push_state(BRACED);
   yylval.lineno = lineno;
   return yytext[0];
 }
 
-<SPECIAL,NESTED,BRACED>[(\[] {
+<SPECIAL,QSPECIAL,NESTED,BRACED>[(\[] {
  yy_push_state(NESTED);
  yylval.lineno = lineno;
  return yytext[0];
@@ -542,7 +542,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return yytext[0];
 }
 
-<SPECIAL,NESTED>[)\]] {
+<SPECIAL,QSPECIAL,NESTED>[)\]] {
   yy_pop_state();
   if (yy_top_state() == INITIAL
       || yy_top_state() == QSILIT)
@@ -550,26 +550,26 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return yytext[0];
 }
 
-<SPECIAL,NESTED,BRACED>{WS} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{WS} {
   /* Eat whitespace in directive */
 }
 
-<SPECIAL,NESTED,BRACED>\" {
+<SPECIAL,QSPECIAL,NESTED,BRACED>\" {
   yy_push_state(STRLIT);
   return '"';
 }
 
-<SPECIAL,NESTED,BRACED>#\\ {
+<SPECIAL,QSPECIAL,NESTED,BRACED>#\\ {
   yy_push_state(CHRLIT);
   return HASH_BACKSLASH;
 }
 
-<SPECIAL,NESTED,BRACED>#[/] {
+<SPECIAL,QSPECIAL,NESTED,BRACED>#[/] {
   yy_push_state(REGEX);
   return HASH_SLASH;
 }
 
-<SPECIAL,NESTED,BRACED>` {
+<SPECIAL,QSPECIAL,NESTED,BRACED>` {
   yy_push_state(QSILIT);
   return '`';
 }
@@ -594,7 +594,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return TEXT;
 }
 
-<SPECIAL,NESTED,BRACED>\n {
+<SPECIAL,QSPECIAL,NESTED,BRACED>\n {
   lineno++;
 }
 
@@ -603,12 +603,12 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   return '/';
 }
 
-<SPECIAL,NESTED>\. {
+<SPECIAL,QSPECIAL,NESTED>\. {
   yylval.chr = '.';
   return '.';
 }
 
-<SPECIAL,NESTED,BRACED>[\\]\n{WS} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>[\\]\n{WS} {
   if (YYSTATE == SPECIAL)
     yy_pop_state();  /* @\ continuation */
   lineno++;
@@ -636,17 +636,17 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
   yyerrorf(lit("unrecognized escape: \\~a"), chr(yytext[1]), nao);
 }
 
-<SPECIAL,NESTED,BRACED>[;].* {
+<SPECIAL,QSPECIAL,NESTED,BRACED>[;].* {
   /* comment */
 }
 
-<SPECIAL,NESTED,BRACED>{UANYN} {
+<SPECIAL,QSPECIAL,NESTED,BRACED>{UANYN} {
   yyerrprepf(lit("bad character in directive: '~a'"),
              string_utf8(yytext), nao);
   return ERRTOK;
 }
 
-<SPECIAL,NESTED,BRACED>. {
+<SPECIAL,QSPECIAL,NESTED,BRACED>. {
   yyerrprepf(lit("non-UTF-8 byte in directive: "
                  "'\\x~02x'"),
              num((unsigned char) yytext[0]), nao);
@@ -826,7 +826,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <QSILIT>@ {
-  yy_push_state(SPECIAL);
+  yy_push_state(QSPECIAL);
 }
 
 <STRLIT,CHRLIT,QSILIT>{UANYN} {