read/get-json: reject trailing junk in string input.

* parser.c (lisp_parse_impl): If parsing from string, check
for trailing junk and diagnose. JSON parsing doesn't use
lookahead because it doesn't have a.b syntax, so the
recent_tok gives the last token that actually went into the
syntax, and not a lookahead token. So in the case of JSON,
we call yylex to see if there is any trailing token.

* tests/010/json.tl: Extend get-json tests to more kinds of
objects, and then replicate with trailing whitespace and
trailing junk to provide coverage for these cases.

* tests/012/parse.t: Slew of new read tests and iread also.

* txr.1: Documented.

2 files changed, 73 insertions, 1 deletions

diff --git a/tests/010/json.tl b/tests/010/json.tl
index 29ee2833..8d414b2e 100644
--- a/tests/010/json.tl
+++ b/tests/010/json.tl
@@ -92,7 +92,27 @@
   (get-json "\"abc\"") "abc"
   (get-json "true") t
   (get-json "false") nil
-  (get-json "null") null)
+  (get-json "null") null
+  (get-json "[1,2,3]") #(1.0 2.0 3.0)
+  (get-json "{\"a\":\"b\"}") #H(() ("a" "b")))
+
+(mtest
+  (get-json "0 \n") 0.0
+  (get-json "\"abc\" \n") "abc"
+  (get-json "true \n") t
+  (get-json "false \n") nil
+  (get-json "null \n") null
+  (get-json "[1,2,3] \n") #(1.0 2.0 3.0)
+  (get-json "{\"a\":\"b\"} \n") #H(() ("a" "b")))
+
+(mtest
+  (get-json "0,") :error
+  (get-json "\"abc\",") :error
+  (get-json "true,") :error
+  (get-json "false,") :error
+  (get-json "null,") :error
+  (get-json "[1,2,3],") :error
+  (get-json "{\"a\":\"b\"},") :error)
 
 (mtest
   (tojson #(1.0 "abc" t)) "[1,\"abc\",true]"
diff --git a/tests/012/parse.tl b/tests/012/parse.tl
index 2bce20dc..6d091b00 100644
--- a/tests/012/parse.tl
+++ b/tests/012/parse.tl
@@ -12,3 +12,55 @@
 
   (test (regex-parse (str-buf #b'F3FF'))
         (compound #\xDCF3 #\xDCFF)))
+
+(mtest
+  (read "0") 0
+  (read "0x") 0x
+  (read "a") a
+  (read "abc") abc
+  (read "abc.def") abc.def
+  (read "(1 2 3)") (1 2 3)
+  (read "#;(1 2 3) 4") 4
+  (read "#;(1 2 3) #; a.b 4") 4
+  (read "0 ") 0
+  (read "0x ") 0x
+  (read "a ") a
+  (read "abc ") abc
+  (read "abc.def ") abc.def
+  (read "(1 2 3) ") (1 2 3)
+  (read "#;(1 2 3) 4 ") 4
+  (read "#;(1 2 3) #; a.b 4 ") 4
+  (read "0,") :error
+  (read "0x,") :error
+  (read "a,") :error
+  (read "abc,") :error
+  (read "abc.def,") :error
+  (read "(1 2 3),") :error
+  (read "#;(1 2 3) 4,") :error
+  (read "#;(1 2 3) #; a.b 4,") :error)
+
+(mtest
+  (iread "0") 0
+  (iread "0x") 0x
+  (iread "a") a
+  (iread "abc") abc
+  (iread "abc.def") abc
+  (iread "(1 2 3)") (1 2 3)
+  (iread "#;(1 2 3) 4") 4
+  (iread "#;(1 2 3) #; a.b 4") .b
+  (iread "0 ") 0
+  (iread "0x ") 0x
+  (iread "a ") a
+  (iread "abc ") abc
+  (iread "abc.def ") abc
+  (iread "(1 2 3) ") (1 2 3)
+  (iread "#;(1 2 3) 4 ") 4
+  (iread "#;(1 2 3) #; a.b 4 ") .b
+  (iread "0,") 0
+  (iread "0x,") 0x
+  (iread "a,") a
+  (iread "abc,") abc
+  (iread "abc.def,") abc
+  (iread "(1 2 3),") (1 2 3)
+  (iread "#;(1 2 3) 4,") 4
+  (iread "#;(1 2 3) #; a.b 4,") .b)