Better handling of dot position function calls.

The expander now actually produces apply forms for dot
position function call and dwim forms.  This allows symbol
macros to work naturally.

* eval.c (sys_apply_s): New symbol variable.
(imp_list_to_list, dot_to_apply): New static functions.
(expand_forms, expand_forms_lisp1): We now throw an error if a
non-nil atom terminates a form, Except in compatibility mode
with TXR 137 or less, whereby we emulate the old behavior of
not expanding this atom.
(do_expand): Perform the dot_to_apply transformation
on the arguments of the dwim form.
Perform the dot_to_apply transformation on an
entire function call form.
(eval_init): Initialize sys_apply_s and register
sys:apply function (using the same function object that
is registered under apply).

* txr.1: Documented that both DWIM forms and regular
function call forms work as if by a transformation to apply form,
removing verbiage which separately described the DWIM
handling. Documented that symbol macros work properly in
dot position.

2 files changed, 110 insertions, 31 deletions

diff --git a/eval.c b/eval.c
index 0133caea..41688074 100644
--- a/eval.c
+++ b/eval.c
@@ -87,7 +87,7 @@ val for_s, for_star_s, each_s, each_star_s, collect_each_s, collect_each_star_s;
 val append_each_s, append_each_star_s, while_s, while_star_s, until_star_s;
 val dohash_s;
 val uw_protect_s, return_s, return_from_s, sys_abscond_from_s, block_star_s;
-val list_s, append_s, apply_s, iapply_s;
+val list_s, append_s, apply_s, sys_apply_s, iapply_s;
 val gen_s, gun_s, generate_s, rest_s, plus_s;
 val promise_s, promise_forced_s, promise_inprogress_s, force_s;
 val op_s, ap_s, identity_s, apf_s, ipf_s;
@@ -2456,14 +2456,38 @@ static val me_quasilist(val form, val menv)
   return cons(list_s, cdr(form));
 }
 
+static val imp_list_to_list(val list)
+{
+  list_collect_decl (out, ptail);
+
+  for (; consp(list); list = cdr(list))
+    ptail = list_collect(ptail, car(list));
+
+  list_collect(ptail, list);
+  return out;
+}
+
+static val dot_to_apply(val form, val lisp1_p)
+{
+  if ((opt_compat && opt_compat <= 137) || proper_listp(form)) {
+    return form;
+  } else {
+    val sym = car(form);
+    val args = imp_list_to_list(cdr(form));
+    return cons(sys_apply_s, cons(if3(lisp1_p,
+                                      sym,
+                                      list(fun_s, sym, nao)),
+                                  args));
+  }
+}
+
 val expand_forms(val form, val menv)
 {
   if (atom(form)) {
-    val ex_f = expand(form, menv);
-    if (consp(ex_f))
-      uw_throwf(error_s, lit("symbol macro ~s in dot position must produce "
-                             "atom form, not compound"), form, nao);
-    return ex_f;
+    if (!form || (opt_compat && opt_compat <= 137))
+      return form;
+    uw_throwf(error_s, lit("dotted argument ~!~s "
+                           "was not converted to apply form"), form, nao);
   } else {
     val f = car(form);
     val r = cdr(form);
@@ -2527,7 +2551,10 @@ tail:
 static val expand_forms_lisp1(val form, val menv)
 {
   if (atom(form)) {
-    return form;
+    if (!form || (opt_compat && opt_compat <= 137))
+      return form;
+    uw_throwf(error_s, lit("dotted function call ~!~s "
+                           "was not converted to apply form"), form, nao);
   } else {
     val f = car(form);
     val r = cdr(form);
@@ -3513,7 +3540,7 @@ tail:
       return expand_symacrolet(form, menv);
     } else if (sym == dwim_s) {
       val args = rest(form);
-      val args_ex = expand_forms_lisp1(args, menv);
+      val args_ex = expand_forms_lisp1(dot_to_apply(args, t), menv);
 
       if (args == args_ex)
         return form;
@@ -3547,12 +3574,18 @@ tail:
          also handles: prog1, call, if, and, or,
          unwind-protect, return, dwim, set, inc, dec,
          push, pop, flip, and with-saved-vars. */
-      val args = rest(form);
+      val form_ex = dot_to_apply(form, nil);
+      val sym_ex = first(form_ex);
+      val args = rest(form_ex);
       val args_ex = expand_forms(args, menv);
 
-      if (args == args_ex)
+      if (form_ex == form && args_ex == args)
         return form;
-      return rlcp(cons(sym, args_ex), form);
+
+      if (args_ex == args)
+        return form_ex;
+
+      return rlcp(cons(sym_ex, args_ex), form);
     }
     abort();
   }
@@ -4551,6 +4584,7 @@ void eval_init(void)
   list_s = intern(lit("list"), user_package);
   append_s = intern(lit("append"), user_package);
   apply_s = intern(lit("apply"), user_package);
+  sys_apply_s = intern(lit("apply"), system_package);
   iapply_s = intern(lit("iapply"), user_package);
   gen_s = intern(lit("gen"), user_package);
   gun_s = intern(lit("gun"), user_package);
@@ -4735,7 +4769,11 @@ void eval_init(void)
   reg_fun(intern(lit("mapdo"), user_package), func_n1v(mapdov));
   reg_fun(intern(lit("window-map"), user_package), func_n4(window_map));
   reg_fun(intern(lit("window-mappend"), user_package), func_n4(window_mappend));
-  reg_fun(apply_s, func_n1v(applyv));
+  {
+    val apply_f = func_n1v(applyv);
+    reg_fun(apply_s, apply_f);
+    reg_fun(sys_apply_s, apply_f);
+  }
   reg_fun(iapply_s, func_n1v(iapply));
   reg_fun(call_s, call_f);
   reg_fun(intern(lit("reduce-left"), user_package), func_n4o(reduce_left, 2));
diff --git a/txr.1 b/txr.1
index 75b1a309..1e045362 100644
--- a/txr.1
+++ b/txr.1
@@ -10034,24 +10034,17 @@ namely that the meta-numbers and meta-symbols of the
 .code op
 operator can be used in the dot position).
 
-If the dot position of a compound form is an atom, then the following
-equivalence applies:
+If the dot position of a compound form is an atom, then the behavior maybe
+understood according to the following transformations:
 
 .cblk
-  (f a b c ... . x)   <-->  (apply (fun f) a b c ... x)
+  (f a b c ... . x)  -->  (apply (fun f) a b c ... x)
+  [f a b c ... . x]  -->  [apply f a b c ... x]
 .cble
 
 Effectively, the dot notation constitutes a shorthand for
 .codn apply .
 
-The DWIM brackets are similar, except that the first position is an arbitrary
-expression which is evaluated according to the same rules as the remaining
-positions. The first expression must evaluate to a function, or else to some
-other object for which the DWIM syntax is defined, such as a vector, string,
-list or hash. Operators are not supported.   The dotted syntax for application
-of additional arguments from a list or vector is supported in the DWIM
-brackets just like in the parentheses.
-
 Examples:
 
 .cblk
@@ -10079,20 +10072,34 @@ Examples:
   (call (op list 1 . @1) 2) ;; yields 2
 .cble
 
-A special consideration is given to symbol macros which occur in the
-dot position. If a symbol macro occurs in this position, then it must not
-expand to a compound form, otherwise a macro-expansion-time error results.
-If it expands to an atom, then it is substituted by that atom, and
-the semantics of the function call proceeds as described above.
+Note that the atom in the dot position of a function call may
+be a symbol macro. Since the semantics works as if by
+transformation to an apply form in which the original dot
+position atom is an ordinary argument, the symbol macro
+may produce a compound form.
 
-.cblk
-  (symacrolet ((x (list 1 2)))
-    (list 1 . x))  ;; error: x expands to form (list 1 2)
+Thus:
 
+.cblk
   (symacrolet ((x 2))
     (list 1 . x))  ;; yields (1 . 2)
+
+  (symacrolet ((x (list 1 2)))
+    (list 1 . x))  ;; (yields (1 . 3))
 .cble
 
+That is to say, the expansion of
+.code x
+is not substituted into the form
+.code "(list 1 . x)"
+but rather the transformation to
+.code apply
+syntax takes place first, and
+so the substitution of
+.code x
+takes place in a form resembling
+.codn "(apply (fun list) 1 x)" .
+
 Dialect Note:
 
 In some other Lisp dialects like ANSI Common Lisp, the improper list syntax may
@@ -40994,6 +41001,40 @@ of these version values, the described behaviors are provided if
 is given an argument which is equal or lower. For instance
 .code -C 103
 selects the behaviors described below for version 105, but not those for 102.
+.IP 137
+Compatibility with \*(TX 137 restores the behavior of not expanding
+symbol macros in the dot position of a function call form. For instance
+if
+.code x
+is a symbol macro, in this compatibility mode it is not recognized
+in a form like
+.codn "(list 1 2 . x)" .
+This preserves the behavior of code which depends on
+.code x
+in such a form to refer to a variable that is being otherwise shadowed by the
+symbol macro. \*(TX 137 compatibility also restores a particular behavior
+of the global and local macro defining operators
+.code defsymacro
+and
+.codn symacrolet :
+in compatibility mode, these operators macro-expand the replacement forms
+of symbol macros at expansion time, and then bind the resulting expanded
+forms to their respective macro symbols. The forms are then potentially
+expanded again when the symbol macros are substituted. This wrong behavior was
+never implied by the documentation. The
+.codn with-slots
+macro is also affected by this, because it is implemented in terms of
+.codn symacrolet .
+Lastly, \*(TX 137 compatibility mode also restores another behavior
+of the dot position in function call forms: if the dot position of a
+function call form produces a sequence that is not a list, that sequence
+is converted to a list so that
+.code (list\ .\ "abc")
+produces
+.codn "(#\ea #\eb #\ec)" .
+After 137, no such treatment is applied to the value and the same form now
+yields
+.strn abc .
 .IP 136
 A request for compatibility with \*(TX 136 or earlier restores the old behavior
 of the