Bugfix: lexer loses unmatched "hold char" between top-level forms.

Test case: file containing 4(prinl 3). Scanner consumes 4 and (.
The ( is lost when the scanner is reset for the next call to yyparse,
resulting in jut prinl being read and interpreted as a variable.

* parser.c (prime_parser): If present, append hold byte to priming
string. Takes parser_t * instead of parser, and returns void now.

* parser.l (reset_scanner): Now returns int value, the value
of the scanner's yy_hold_char variable which is nonzero when
the scanner is hanging on to an unmatched byte of input.

* parser.h (reset_scanner, prime_parser): Declarations updated.

* parser.y (parse): Pass hold byte returned by reset_scanner to
prime_parser.

5 files changed, 39 insertions, 11 deletions

diff --git a/ChangeLog b/ChangeLog
index bab4ca13..ade86529 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,25 @@
 2015-07-10  Kaz Kylheku  <kaz@kylheku.com>
 
+	Bugfix: lexer loses unmatched "hold char" between top-level forms.
+
+	Test case: file containing 4(prinl 3). Scanner consumes 4 and (.
+	The ( is lost when the scanner is reset for the next call to yyparse,
+	resulting in jut prinl being read and interpreted as a variable.
+
+	* parser.c (prime_parser): If present, append hold byte to priming
+	string. Takes parser_t * instead of parser, and returns void now.
+
+	* parser.l (reset_scanner): Now returns int value, the value
+	of the scanner's yy_hold_char variable which is nonzero when
+	the scanner is hanging on to an unmatched byte of input.
+
+	* parser.h (reset_scanner, prime_parser): Declarations updated.
+
+	* parser.y (parse): Pass hold byte returned by reset_scanner to
+	prime_parser.
+
+2015-07-10  Kaz Kylheku  <kaz@kylheku.com>
+
 	* stream.c (byte_in_unget_byte): Wrong function name in error message.
 
 2015-07-10  Kaz Kylheku  <kaz@kylheku.com>
diff --git a/parser.c b/parser.c
index aa85a7f4..97c8f44a 100644
--- a/parser.c
+++ b/parser.c
@@ -127,19 +127,24 @@ static val ensure_parser(val stream, val primer)
   return set(cdr_l(cell), parser(stream, one, primer));
 }
 
-val prime_parser(val parser)
+void prime_parser(parser_t *p, int hold_byte)
 {
-  parser_t *p = get_parser_impl(parser);
-  val secret_token_stream = make_string_byte_input_stream(lit("@\x01" "E"));
+  val secret_token_stream;
+
+  if (hold_byte) {
+      val secret_token_string = format(nil, lit("@\x01" "E~a"),
+                                       chr(hold_byte + 0xDC00), nao);
+      secret_token_stream = make_string_byte_input_stream(secret_token_string);
+  } else {
+    secret_token_stream = make_string_byte_input_stream(lit("@\x01" "E"));
+  }
 
   if (catenated_stream_p(p->stream)) {
     catenated_stream_push(secret_token_stream, p->stream);
   } else {
-    set(mkloc(p->stream, parser),
+    set(mkloc(p->stream, p->parser),
         make_catenated_stream(list(secret_token_stream, p->stream, nao)));
   }
-
-  return parser;
 }
 
 void open_txr_file(val spec_file, val *txr_lisp_p, val *name, val *stream)
diff --git a/parser.h b/parser.h
index 0db10d4e..83bd7c38 100644
--- a/parser.h
+++ b/parser.h
@@ -53,14 +53,14 @@ void yyerrorf(scanner_t *scanner, val s, ...);
 void yybadtoken(parser_t *, int tok, val context);
 void end_of_regex(scanner_t *scanner);
 void end_of_char(scanner_t *scanner);
-void reset_scanner(scanner_t *scanner);
+int reset_scanner(scanner_t *scanner);
 int yylex_init(yyscan_t *pscanner);
 int yylex_destroy(yyscan_t scanner);
 parser_t *yyget_extra(yyscan_t scanner);
 void yyset_extra(parser_t *, yyscan_t);
 void parser_l_init(void);
 void open_txr_file(val spec_file, val *txr_lisp_p, val *name, val *stream);
-val prime_parser(val parser);
+void prime_parser(parser_t *, int hold_byte);
 int parse_once(val stream, val name, parser_t *parser);
 int parse(parser_t *parser);
 val source_loc(val form);
diff --git a/parser.l b/parser.l
index 4739dd59..29a45ab1 100644
--- a/parser.l
+++ b/parser.l
@@ -960,12 +960,16 @@ void end_of_char(scanner_t *yyg)
   yy_pop_state(yyg);
 }
 
-void reset_scanner(scanner_t *yyg)
+int reset_scanner(scanner_t *yyg)
 {
+  int hold_byte = yyg->yy_hold_char;
+
   while (YYSTATE != INITIAL)
     yy_pop_state(yyg);
 
   yy_flush_buffer(YY_CURRENT_BUFFER, yyg);
+
+  return hold_byte;
 }
 
 val source_loc(val form)
diff --git a/parser.y b/parser.y
index de3f600d..e71adfda 100644
--- a/parser.y
+++ b/parser.y
@@ -1463,8 +1463,7 @@ int parse(parser_t *parser)
   parser->errors = 0;
   parser->prepared_msg = nil;
   parser->syntax_tree = nil;
-  prime_parser(parser->parser);
-  reset_scanner(parser->scanner);
+  prime_parser(parser, reset_scanner(parser->scanner));
 
   res = yyparse(parser->scanner, parser);