lazy-str-get-trailing-list: spurious empty string issue.

* lib.c (lazy_str_get_trailing_list): Remove the spurious
empty string caused by splitting on the terminator.
Whenever the materialized prefix is not-empty, and there is
a non-empty terminator, the prefix necessarily ends in the
termintator. If we split on the terminator, the list of pieces
ends in in an empty string, which is undesirable.
This has to be subject to compat, unfortunately; it's a
very visible behavior that affects the continuation of
line-based matching after the @(freeform) directive.

* tests/006/freeform-5.txr: With this fix, we no longer have
to match the spurious blank line coming from @(freeform).

* tests/015/lazy-str.tl: New file.

* txr.1: Updated documentation with compat notes. There was
some outright incorrect text describing
lazy-str-get-trailing-list. Also, the lazy-str-force-upto
and lazy-str-force were under-documented.  The return value of
the former was not completely described: that it returns t
in the other case when not returning nil. It wasn't mentioned
that the functions observe the limit-count. Moreover, the
exact algorithm for forcing is now documented.

1 files changed, 98 insertions, 0 deletions

diff --git a/tests/015/lazy-str.tl b/tests/015/lazy-str.tl
new file mode 100644
index 00000000..bfcd6328
--- /dev/null
+++ b/tests/015/lazy-str.tl
@@ -0,0 +1,98 @@
+(load "../common")
+
+(defvarl words '#"the quick")
+
+(defvarl lz0 (lazy-str words))
+
+(test (lazy-str-get-trailing-list (copy lz0) 0) #"the quick")
+(test (lazy-str-get-trailing-list (copy lz0) 1) #"he quick")
+(test (lazy-str-get-trailing-list (copy lz0) 2) #"e quick")
+(test (lazy-str-get-trailing-list (copy lz0) 3) ("" . #"quick"))
+(test (lazy-str-get-trailing-list (copy lz0) 4) #"quick")
+(test (lazy-str-get-trailing-list (copy lz0) 5) #"uick")
+(test (lazy-str-get-trailing-list (copy lz0) 6) #"ick")
+(test (lazy-str-get-trailing-list (copy lz0) 7) #"ck")
+(test (lazy-str-get-trailing-list (copy lz0) 8) #"k")
+(test (lazy-str-get-trailing-list (copy lz0) 9) (""))
+(test (lazy-str-get-trailing-list (copy lz0) 10) ())
+(test (lazy-str-get-trailing-list (copy lz0) 11) ())
+
+(defvarl lz1 (lazy-str words ":"))
+
+(test (lazy-str-get-trailing-list (copy lz1) 0) #"the quick")
+(test (lazy-str-get-trailing-list (copy lz1) 1) #"he quick")
+(test (lazy-str-get-trailing-list (copy lz1) 2) #"e quick")
+(test (lazy-str-get-trailing-list (copy lz1) 3) ("" . #"quick"))
+(test (lazy-str-get-trailing-list (copy lz1) 4) #"quick")
+(test (lazy-str-get-trailing-list (copy lz1) 5) #"uick")
+(test (lazy-str-get-trailing-list (copy lz1) 6) #"ick")
+(test (lazy-str-get-trailing-list (copy lz1) 7) #"ck")
+(test (lazy-str-get-trailing-list (copy lz1) 8) #"k")
+(test (lazy-str-get-trailing-list (copy lz1) 9) (""))
+(test (lazy-str-get-trailing-list (copy lz1) 10) ())
+(test (lazy-str-get-trailing-list (copy lz1) 11) ())
+
+(test [(copy lz1) 0..0] "")
+(test [(copy lz1) 0..1] "t")
+(test [(copy lz1) 0..2] "th")
+(test [(copy lz1) 0..3] "the")
+(test [(copy lz1) 0..4] "the:")
+(test [(copy lz1) 0..5] "the:q")
+(test [(copy lz1) 0..6] "the:qu")
+(test [(copy lz1) 0..7] "the:qui")
+(test [(copy lz1) 0..8] "the:quic")
+(test [(copy lz1) 0..9] "the:quick")
+(test [(copy lz1) 0..10] "the:quick:")
+(test [(copy lz1) 0..11] "the:quick:")
+
+(defvarl lz2 (lazy-str '#"the quick brown fox" ":" 2))
+
+(test (lazy-str-get-trailing-list (copy lz2) 0) #"the quick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 1) #"he quick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 2) #"e quick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 3) ("" . #"quick brown fox"))
+(test (lazy-str-get-trailing-list (copy lz2) 4) #"quick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 5) #"uick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 6) #"ick brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 7) #"ck brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 8) #"k brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 9) ("" . #"brown fox"))
+(test (lazy-str-get-trailing-list (copy lz2) 10) #"brown fox")
+(test (lazy-str-get-trailing-list (copy lz2) 11) #"brown fox")
+
+(test [(copy lz2) 0..0] "")
+(test [(copy lz2) 0..1] "t")
+(test [(copy lz2) 0..2] "th")
+(test [(copy lz2) 0..3] "the")
+(test [(copy lz2) 0..4] "the:")
+(test [(copy lz2) 0..5] "the:q")
+(test [(copy lz2) 0..6] "the:qu")
+(test [(copy lz2) 0..7] "the:qui")
+(test [(copy lz2) 0..8] "the:quic")
+(test [(copy lz2) 0..9] "the:quick")
+(test [(copy lz2) 0..10] "the:quick:")
+(test [(copy lz2) 0..11] "the:quick:")
+
+(defvarl lz3 (lazy-str words ""))
+
+(test (lazy-str-get-trailing-list (copy lz3) 0) #"t h e quick")
+(test (lazy-str-get-trailing-list (copy lz3) 1) #"h e quick")
+(test (lazy-str-get-trailing-list (copy lz3) 2) #"e quick")
+(test (lazy-str-get-trailing-list (copy lz3) 3) #"q u i c k")
+(test (lazy-str-get-trailing-list (copy lz3) 4) #"u i c k")
+(test (lazy-str-get-trailing-list (copy lz3) 5) #"i c k")
+(test (lazy-str-get-trailing-list (copy lz3) 6) #"c k")
+(test (lazy-str-get-trailing-list (copy lz3) 7) #"k")
+(test (lazy-str-get-trailing-list (copy lz3) 8) ())
+(test (lazy-str-get-trailing-list (copy lz3) 9) ())
+
+(test [(copy lz3) 0..0] "")
+(test [(copy lz3) 0..1] "t")
+(test [(copy lz3) 0..2] "th")
+(test [(copy lz3) 0..3] "the")
+(test [(copy lz3) 0..4] "theq")
+(test [(copy lz3) 0..5] "thequ")
+(test [(copy lz3) 0..6] "thequi")
+(test [(copy lz3) 0..7] "thequic")
+(test [(copy lz3) 0..8] "thequick")
+(test [(copy lz3) 0..9] "thequick")