3 files changed, 28 insertions, 0 deletions
diff --git a/tests/008/data b/tests/008/data
new file mode 100644
index 00000000..f171c282
--- /dev/null
+++ b/tests/008/data
@@ -0,0 +1,8 @@
+a!===b=!=c
+a!===!==!=!==b
+
+a
+a=
+=
+==
+===
diff --git a/tests/008/tokenize.expected b/tests/008/tokenize.expected
new file mode 100644
index 00000000..86ebd01a
--- /dev/null
+++ b/tests/008/tokenize.expected
@@ -0,0 +1,8 @@
+"a" {!=} "" {==} "b" {=} "" {!=} "c"
+"a" {!=} "" {==} "" {!=} "" {=} "" {!=} "" {!=} "" {=} "b"
+""
+"a"
+"a" {=} ""
+"" {=} ""
+"" {==} ""
+"" {==} "" {=} ""
diff --git a/tests/008/tokenize.txr b/tests/008/tokenize.txr
new file mode 100644
index 00000000..39211ba9
--- /dev/null
+++ b/tests/008/tokenize.txr
@@ -0,0 +1,12 @@
+@(collect)
+@(coll :gap 0)@(choose :shortest tok)@\
+                @tok@{sep /==/}@\
+              @(or)@\
+                @tok@{sep /!=/}@\
+              @(or)@\
+                @tok@{sep /=/}@\
+              @(end)@(end)@tail
+@(output)
+@(rep)"@tok" {@sep} @(end)"@tail"
+@(end)
+@(end)