parser: fix problems at EOF involving #; syntax.

This patch addresses a problem whereby if a TXR Lisp file ends
with an erased object notation such as #;(a b c), there is a
syntax error.

The strategy is to simplify the grammar so that a single
yyparse primed with SECRET_ESCAPE_E or SECRET_ESCAPE_I will
read either an object, or just one instance of the #;
notation. If #;OBJ is read, then the parse tree is returned as
the nao value. The caller knows that #;OBJ must have occurred
because there are no errors and the parser isn't at EOF, yet
there is no parse tree. Then in lisp_parse we can loop on this
situation, and make adjustments elsewhere also.

So that iread continues to work, we must separate the
parser_eof condition from the lookahead token. Under iread,
we were clearing the token in prime_parser_post, but that
was having the side effect of making the parser look like
it is in EOF. We now preserve the EOF indication in a flag,
so we can manipulate the token.

* parser.h (struct parser): new member, eof.

* parser.c (parser_common_init): Initialize new eof flag
in parser structure to zero.
(prime_parser_post): Set the eof flag if the parser's most
recent token is zero.
(lisp_parse_impl): Call the parser repeatedly while there
are no errors, and no EOF, yet no object has been produced.
This indicates that a #; erasure has been processed.
(read_eval_stream): Restructure the logic here for
clarity. Do not break the loop if error_val was returned
from the parser, but there are no errors, and the parser isn't
at EOF. This is behavior is probably redundant with respect
to the loop in lisp_parse_impl.
(read_eval_ret_last): Bugfixes here. Pass an error indicating
value down to lisp_parse, like in read_eval_stream and
make the logic similar.
(parser_eof): Just return an indication based no the
eof flag.

* parser.y (hash_semis_n_expr, hash_semis_i_expr,
ignored_i_exprs, ignored_n_exprs): Grammar rules removed.
(hash_semi_or_n_expr, hash_semi_or_i_expr): New grammar
rules.
(spec): Retarget SECRET_ESCAPE_E and SECRET_ESCAPE_I
cases to new rules.
(parse): Clear eof flag to zero.

1 files changed, 14 insertions, 24 deletions

diff --git a/parser.y b/parser.y
index 24114b88..5703bef0 100644
--- a/parser.y
+++ b/parser.y
@@ -111,8 +111,7 @@ int yyparse(scanner_t *, parser_t *);
 
 %token <chr> REGCHAR REGTOKEN LITCHAR SPLICE CONSDOT LAMBDOT
 
-%type <val> spec hash_semis_n_expr hash_semis_i_expr
-%type <val> ignored_i_exprs ignored_n_exprs
+%type <val> spec hash_semi_or_n_expr hash_semi_or_i_expr
 %type <val> clauses_rev clauses_opt clause
 %type <val> all_clause some_clause none_clause maybe_clause block_clause
 %type <val> cases_clause choose_clause gather_clause collect_clause until_last
@@ -151,10 +150,10 @@ int yyparse(scanner_t *, parser_t *);
 
 spec : clauses_opt              { parser->syntax_tree = $1; }
      | SECRET_ESCAPE_R regexpr  { parser->syntax_tree = $2; end_of_regex(scnr); }
-     | SECRET_ESCAPE_E hash_semis_n_expr
+     | SECRET_ESCAPE_E hash_semi_or_n_expr
                                 { parser->syntax_tree = $2; YYACCEPT; }
        byacc_fool               { internal_error("notreached"); }
-     | SECRET_ESCAPE_I hash_semis_i_expr
+     | SECRET_ESCAPE_I hash_semi_or_i_expr
                                 { parser->syntax_tree = $2; YYACCEPT; }
        byacc_fool               { internal_error("notreached"); }
      | SECRET_ESCAPE_E          { if (yychar == YYEOF) {
@@ -179,29 +178,19 @@ spec : clauses_opt              { parser->syntax_tree = $1; }
 
      ;
 
-hash_semis_n_expr : ignored_n_exprs n_expr      { $$ = $2; }
-                  | n_expr                      { $$ = $1; }
-                  ;
 
-ignored_n_exprs : ignored_n_exprs HASH_SEMI     { parser->circ_suppress = 1; }
-                  n_expr                        { parser->circ_suppress = 0;
-                                                  $$ = nil; }
-                | HASH_SEMI                     { parser->circ_suppress = 1; }
-                  n_expr                        { parser->circ_suppress = 0;
-                                                  $$ = nil; }
-                ;
+hash_semi_or_n_expr : HASH_SEMI                 { parser->circ_suppress = 1; }
+                      n_expr                    { parser->circ_suppress = 0;
+                                                  $$ = nao; }
+                    | n_expr                    { $$ = $1; }
+                    ;
 
-hash_semis_i_expr : ignored_i_exprs i_expr      { $$ = $2; }
-                  | i_expr                      { $$ = $1; }
-                  ;
+hash_semi_or_i_expr : HASH_SEMI                 { parser->circ_suppress = 1; }
+                      i_expr                    { parser->circ_suppress = 0;
+                                                  $$ = nao; }
+                    | i_expr                    { $$ = $1; }
+                    ;
 
-ignored_i_exprs : ignored_i_exprs HASH_SEMI     { parser->circ_suppress = 1; }
-                  i_expr                        { parser->circ_suppress = 0;
-                                                  $$ = nil; }
-                | HASH_SEMI                     { parser->circ_suppress = 1; }
-                  i_expr                        { parser->circ_suppress = 0;
-                                                  $$ = nil; }
-                ;
 
 /* Hack needed for Berkeley Yacc */
 byacc_fool : n_expr { internal_error("notreached"); }
@@ -1787,6 +1776,7 @@ int parse(parser_t *parser, val name, enum prime_parser prim)
   int res = 0;
 
   parser->errors = 0;
+  parser->eof = 0;
   parser->prepared_msg = nil;
   parser->circ_ref_hash = nil;
   parser->circ_count = 0;