* Makefile (TXR_ARGS): Pass new file to tests/009/json.txr test.

* tests/009/json.expected: Updated.

* tests/009/json.txr: Updated source. Translates to a more native
representation with vectors and hash tables. Numbers go to
floating point instead of remaining as strings.

* tests/009/pass1.json: New file: a test case from json.org.

3 files changed, 117 insertions, 17 deletions

diff --git a/tests/009/json.expected b/tests/009/json.expected
index 130058c2..b2d5f0b7 100644
--- a/tests/009/json.expected
+++ b/tests/009/json.expected
@@ -1,2 +1,8 @@
-(v "O" ((("S" "web-app") ("O" ((("S" "servlet") ("A" (("O" ((("S" "servlet-name") ("S" "cofaxCDS")) (("S" "servlet-class") ("S" "org.cofax.cds.CDSServlet")) (("S" "init-param") ("O" ((("S" "configGlossary:installationAt") ("S" "Philadelphia, PA")) (("S" "configGlossary:adminEmail") ("S" "ksm@pobox.com")) (("S" "configGlossary:poweredBy") ("S" "Cofax")) (("S" "configGlossary:poweredByIcon") ("S" "/images/cofax.gif")) (("S" "configGlossary:staticPath") ("S" "/content/static")) (("S" "templateProcessorClass") ("S" "org.cofax.WysiwygTemplate")) (("S" "templateLoaderClass") ("S" "org.cofax.FilesTemplateLoader")) (("S" "templatePath") ("S" "templates")) (("S" "templateOverridePath") ("S" "")) (("S" "defaultListTemplate") ("S" "listTemplate.htm")) (("S" "defaultFileTemplate") ("S" "articleTemplate.htm")) (("S" "useJSP") ("K" "false")) (("S" "jspListTemplate") ("S" "listTemplate.jsp")) (("S" "jspFileTemplate") ("S" "articleTemplate.jsp")) (("S" "cachePackageTagsTrack") ("N" "200")) (("S" "cachePackageTagsStore") ("N" "200")) (("S" "cachePackageTagsRefresh") ("N" "60")) (("S" "cacheTemplatesTrack") ("N" "100")) (("S" "cacheTemplatesStore") ("N" "50")) (("S" "cacheTemplatesRefresh") ("N" "15")) (("S" "cachePagesTrack") ("N" "200")) (("S" "cachePagesStore") ("N" "100")) (("S" "cachePagesRefresh") ("N" "10")) (("S" "cachePagesDirtyRead") ("N" "10")) (("S" "searchEngineListTemplate") ("S" "forSearchEnginesList.htm")) (("S" "searchEngineFileTemplate") ("S" "forSearchEngines.htm")) (("S" "searchEngineRobotsDb") ("S" "WEB-INF/robots.db")) (("S" "useDataStore") ("K" "true")) (("S" "dataStoreClass") ("S" "org.cofax.SqlDataStore")) (("S" "redirectionClass") ("S" "org.cofax.SqlRedirection")) (("S" "dataStoreName") ("S" "cofax")) (("S" "dataStoreDriver") ("S" "com.microsoft.jdbc.sqlserver.SQLServerDriver")) (("S" "dataStoreUrl") ("S" "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon")) (("S" "dataStoreUser") ("S" "sa")) (("S" "dataStorePassword") ("S" "dataStoreTestQuery")) (("S" "dataStoreTestQuery") ("S" "SET NOCOUNT ON;select test='test';")) (("S" "dataStoreLogFile") ("S" "/usr/local/tomcat/logs/datastore.log")) (("S" "dataStoreInitConns") ("N" "10")) (("S" "dataStoreMaxConns") ("N" "100")) (("S" "dataStoreConnUsageLimit") ("N" "100")) (("S" "dataStoreLogLevel") ("S" "debug")) (("S" "maxUrlLength") ("N" "500"))))))) ("O" ((("S" "servlet-name") ("S" "cofaxEmail")) (("S" "servlet-class") ("S" "org.cofax.cds.EmailServlet")) (("S" "init-param") ("O" ((("S" "mailHost") ("S" "mail1")) (("S" "mailHostOverride") ("S" "mail2"))))))) ("O" ((("S" "servlet-name") ("S" "cofaxAdmin")) (("S" "servlet-class") ("S" "org.cofax.cds.AdminServlet")))) ("O" ((("S" "servlet-name") ("S" "fileServlet")) (("S" "servlet-class") ("S" "org.cofax.cds.FileServlet")))) ("O" ((("S" "servlet-name") ("S" "cofaxTools")) (("S" "servlet-class") ("S" "org.cofax.cms.CofaxToolsServlet")) (("S" "init-param") ("O" ((("S" "templatePath") ("S" "toolstemplates/")) (("S" "log") ("N" "1")) (("S" "logLocation") ("S" "/usr/local/tomcat/logs/CofaxTools.log")) (("S" "logMaxSize") ("S" "")) (("S" "dataLog") ("N" "1")) (("S" "dataLogLocation") ("S" "/usr/local/tomcat/logs/dataLog.log")) (("S" "dataLogMaxSize") ("S" "")) (("S" "removePageCache") ("S" "/content/admin/remove?cache=pages&id=")) (("S" "removeTemplateCache") ("S" "/content/admin/remove?cache=templates&id=")) (("S" "fileTransferFolder") ("S" "/usr/local/tomcat/webapps/content/fileTransferFolder")) (("S" "lookInContext") ("N" "1")) (("S" "adminGroupID") ("N" "4")) (("S" "betaServer") ("K" "true")))))))))) (("S" "servlet-mapping") ("O" ((("S" "cofaxCDS") ("S" "/")) (("S" "cofaxEmail") ("S" "/cofaxutil/aemail/*")) (("S" "cofaxAdmin") ("S" "/admin/*")) (("S" "fileServlet") ("S" "/static/*")) (("S" "cofaxTools") ("S" "/tools/*"))))) (("S" "taglib") ("O" ((("S" "taglib-uri") ("S" "cofax.tld")) (("S" "taglib-location") ("S" "/WEB-INF/tlds/cofax.tld"))))))))))
-(badsyntax . "")
+AST: #H((:equal-based) ("web-app" #H((:equal-based) ("servlet-mapping" #H((:equal-based) ("cofaxTools" "/tools/*") ("cofaxCDS" "/") ("cofaxAdmin" "/admin/*") ("cofaxEmail" "/cofaxutil/aemail/*") ("fileServlet" "/static/*"))) ("servlet" #(#H((:equal-based) ("servlet-class" "org.cofax.cds.CDSServlet") ("init-param" #H((:equal-based) ("defaultFileTemplate" "articleTemplate.htm") ("configGlossary:installationAt" "Philadelphia, PA") ("templateOverridePath" "") ("dataStoreLogLevel" "debug") ("searchEngineListTemplate" "forSearchEnginesList.htm") ("dataStoreClass" "org.cofax.SqlDataStore") ("configGlossary:poweredBy" "Cofax") ("jspFileTemplate" "articleTemplate.jsp") ("cacheTemplatesTrack" 100.0) ("cacheTemplatesStore" 50.0) ("templateProcessorClass" "org.cofax.WysiwygTemplate") ("dataStoreUser" "sa") ("redirectionClass" "org.cofax.SqlRedirection") ("dataStoreConnUsageLimit" 100.0) ("dataStoreMaxConns" 100.0) ("jspListTemplate" "listTemplate.jsp") ("useJSP" :false) ("configGlossary:poweredByIcon" "/images/cofax.gif") ("templateLoaderClass" "org.cofax.FilesTemplateLoader") ("cacheTemplatesRefresh" 15.0) ("cachePagesDirtyRead" 10.0) ("searchEngineRobotsDb" "WEB-INF/robots.db") ("cachePagesStore" 100.0) ("dataStoreTestQuery" "SET NOCOUNT ON;select test='test';") ("configGlossary:adminEmail" "ksm@pobox.com") ("configGlossary:staticPath" "/content/static") ("dataStoreInitConns" 10.0) ("cachePagesTrack" 200.0) ("dataStorePassword" "dataStoreTestQuery") ("defaultListTemplate" "listTemplate.htm") ("dataStoreLogFile" "/usr/local/tomcat/logs/datastore.log") ("maxUrlLength" 500.0) ("dataStoreDriver" "com.microsoft.jdbc.sqlserver.SQLServerDriver") ("dataStoreName" "cofax") ("cachePackageTagsRefresh" 60.0) ("templatePath" "templates") ("dataStoreUrl" "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon") ("useDataStore" :true) ("cachePackageTagsTrack" 200.0) ("searchEngineFileTemplate" "forSearchEngines.htm") ("cachePackageTagsStore" 200.0) ("cachePagesRefresh" 10.0))) ("servlet-name" "cofaxCDS")) #H((:equal-based) ("servlet-class" "org.cofax.cds.EmailServlet") ("init-param" #H((:equal-based) ("mailHost" "mail1") ("mailHostOverride" "mail2"))) ("servlet-name" "cofaxEmail")) #H((:equal-based) ("servlet-class" "org.cofax.cds.AdminServlet") ("servlet-name" "cofaxAdmin")) #H((:equal-based) ("servlet-class" "org.cofax.cds.FileServlet") ("servlet-name" "fileServlet")) #H((:equal-based) ("servlet-class" "org.cofax.cms.CofaxToolsServlet") ("init-param" #H((:equal-based) ("lookInContext" 1.0) ("removePageCache" "/content/admin/remove?cache=pages&id=") ("logMaxSize" "") ("dataLogMaxSize" "") ("removeTemplateCache" "/content/admin/remove?cache=templates&id=") ("dataLog" 1.0) ("logLocation" "/usr/local/tomcat/logs/CofaxTools.log") ("log" 1.0) ("adminGroupID" 4.0) ("templatePath" "toolstemplates/") ("betaServer" :true) ("dataLogLocation" "/usr/local/tomcat/logs/dataLog.log") ("fileTransferFolder" "/usr/local/tomcat/webapps/content/fileTransferFolder"))) ("servlet-name" "cofaxTools")))) ("taglib" #H((:equal-based) ("taglib-uri" "cofax.tld") ("taglib-location" "/WEB-INF/tlds/cofax.tld"))))))
+
+Unmatched junk: ""
+
+AST: #("JSON Test Pattern pass1" #H((:equal-based) ("object with 1 member" #("array with 1 element"))) #H((:equal-based)) #() -42.0 :true :false :null #H((:equal-based) ("" 2.3456789012e+76) ("digit" "0123456789") ("\\/\\\\\"쫾몾ꮘﳞ볚\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" "A key can be any string") ("null" :null) ("one" 1.0) ("E" 1.23456789e+34) ("special" "`1~!@#$%^&*()_+-={':[,]}|;.</>?") ("e" 1.23456789e-13) ("comment" "// /* <!-- --") ("# -- --> */" " ") ("real" -9876.54321) ("backslash" "\\\\") ("array" #()) ("url" "http://www.JSON.org/") ("zero" 0.0) ("false" :false) ("space" " ") ("slash" "/ & \\/") ("address" "50 St. James Street") ("compact" #(1.0 2.0 3.0 4.0 5.0 6.0 7.0)) ("object" #H((:equal-based))) ("quote" "\"") ("jsontext" "{\"object with 1 member\":[\"array with 1 element\"]}") ("true" :true) ("integer" 1234567890.0) ("ALPHA" "ABCDEFGHIJKLMNOPQRSTUVWYZ") ("quotes" "&#34; \" %22 0x22 034 &#x22;") ("hex" "ģ䕧覫췯ꯍ") ("0123456789" "digit") ("controls" "\b\f\n\r\t") ("alpha" "abcdefghijklmnopqrstuvwyz") (" s p a c e d " #(1.0 2.0 3.0 4.0 5.0 6.0 7.0))) 0.5 98.6 99.44 1066.0 10.0 1.0 0.1 1.0 2.0 2.0 "rosebud")
+
+Unmatched junk: ""
+
diff --git a/tests/009/json.txr b/tests/009/json.txr
index bf5530d4..4350207e 100644
--- a/tests/009/json.txr
+++ b/tests/009/json.txr
@@ -1,12 +1,22 @@
+@;
+@; A JSON value is a string, number, associative object, keyword or array.
+@;
 @(define value (v))@\
   @(cases)@\
     @(string v)@(or)@(num v)@(or)@(object v)@(or)@\
     @(keyword v)@(or)@(array v)@\
   @(end)@\
 @(end)
+@;
+@; Pattern function for matching whitespace
+@;
 @(define ws)@/[\n\t ]*/@(end)
-@(define string (g))@\
-  @(local s hex)@\
+@;
+@; Pattern function for matching a JSON string, with all the
+@; escape sequences.
+@;
+@(define string (s))@\
+  @(local hex)@\
   @(ws)@\
   "@(coll :gap 0 :vars (s))@\
      @(cases)@\
@@ -27,18 +37,28 @@
   @(ws)@\
   @(cat s "")@\
   @(filter :from_html s)@\
-  @(bind g ("S" s))@\
 @(end)
-@(define num (v))@\
-  @(local n)@\
-  @(ws)@{n /-?[0-9]+((\.[0-9]+)?([Ee][+\-]?[0-9]+)?)?/}@(ws)@\
-  @(bind v ("N" n))@\
+@;
+@; Pattern function for recognizing a number.
+@;
+@(define num (n))@\
+  @(local tok)@\
+  @(ws)@{tok /-?[0-9]+((\.[0-9]+)?([Ee][+\-]?[0-9]+)?)?/}@(ws)@\
+  @(bind n @(flo-str tok))@\
 @(end)
-@(define keyword (v))@\
-  @(local k)@\
-  @(all)@(ws)@{k /true|false|null/}@(trailer)@/[^A-Za-z0-9_]/@(end)@(ws)@\
-  @(bind v ("K" k))@\
+@;
+@; Recognize the JSON keyword true, false and null, turning
+@; them into TXR Lisp keywords
+@;
+@(define keyword (k))@\
+  @(local tok)@\
+  @(all)@(ws)@{tok /true|false|null/}@(trailer)@/[^A-Za-z0-9_]/@(end)@(ws)@\
+  @(bind k @(intern tok *keyword-package*))@\
 @(end)
+@;
+@; Recognize an object: a collection of string/value pairs,
+@; turning them into an equal-based hash table
+@;
 @(define object (v))@\
   @(local p e pair)@\
   @(ws){@(ws)@(coll :gap 0 :vars (pair))@\
@@ -46,12 +66,28 @@
                 @(bind pair (p e))@\
                 @(until)}@\
              @(end)}@(ws)@\
-  @(bind v ("O" pair))@\
+  @(bind v @(progn '#H((:equal-based) ,*pair)))@\
 @(end)
+@;
+@; Recognize an array.
+@;
 @(define array (v))@\
   @(local e)@\
-  @(ws)[@(ws)@(coll :gap 0 :var (e))@(value e)@/,?/@(until)]@(end)]@(ws)@\
-  @(bind v ("A" e))@\
+  @(ws)[@(ws)@(coll :gap 0 :vars (e))@(value e)@/,?/@(until)]@(end)]@(ws)@\
+  @(bind v @(progn '#(,*e)))@\
 @(end)
+@;
+@; Now parse the input as a JSON object
+@;
+@(next :args)
+@(collect)
+@file
+@(next file)
 @(freeform)
-@(maybe)@(value v)@(end)@badsyntax
+@(maybe)@(value ast)@(end)@badsyntax
+@;
+@; Output resulting abstract syntax tree.
+@;
+@(do (format t "AST: ~s\n\n" ast)
+     (format t "Unmatched junk: ~s\n\n" badsyntax))
+@(end)
diff --git a/tests/009/pass1.json b/tests/009/pass1.json
new file mode 100644
index 00000000..2c10f226
--- /dev/null
+++ b/tests/009/pass1.json
@@ -0,0 +1,58 @@
+[
+    "JSON Test Pattern pass1",
+    {"object with 1 member":["array with 1 element"]},
+    {},
+    [],
+    -42,
+    true,
+    false,
+    null,
+    {
+        "integer": 1234567890,
+        "real": -9876.543210,
+        "e": 0.123456789e-12,
+        "E": 1.234567890E+34,
+        "":  23456789012E66,
+        "zero": 0,
+        "one": 1,
+        "space": " ",
+        "quote": "\"",
+        "backslash": "\\",
+        "controls": "\b\f\n\r\t",
+        "slash": "/ & \/",
+        "alpha": "abcdefghijklmnopqrstuvwyz",
+        "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ",
+        "digit": "0123456789",
+        "0123456789": "digit",
+        "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?",
+        "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A",
+        "true": true,
+        "false": false,
+        "null": null,
+        "array":[  ],
+        "object":{  },
+        "address": "50 St. James Street",
+        "url": "http://www.JSON.org/",
+        "comment": "// /* <!-- --",
+        "# -- --> */": " ",
+        " s p a c e d " :[1,2 , 3
+
+,
+
+4 , 5        ,          6           ,7        ],"compact":[1,2,3,4,5,6,7],
+        "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}",
+        "quotes": "&#34; \u0022 %22 0x22 034 &#x22;",
+        "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?"
+: "A key can be any string"
+    },
+    0.5 ,98.6
+,
+99.44
+,
+
+1066,
+1e1,
+0.1e1,
+1e-1,
+1e00,2e+00,2e-00
+,"rosebud"]