TXR quasiliterals and output vars treated as Lisp.

* eval.c (format_field): Function moved here from match.c,
along with the introduction of a new behavior: if a meta-expr
occurs among the modifiers, its constituent expression
is evaluated in its place. This allows for @{a @[expr]}
which was previously not allowed in Lisp quasiliterals,
but worked in TXR quasiliterals due to the treatment
of @ by txeval.
(subst_vars): Static function turns external, so code in
match.c can call it instead of the subst_vars in that module.
For that purpose, it needs to take a filter argument and
process filters, like the match.c subst_vars.
(op_quasi_lit): Pass nil as filter argument to subst_vars.

* eval.h (format_field, subst_vars): Declared.

* match.c (format_field): Function removed, moved to
eval.c and slightly changed.
(subst_vars): Renamed to tx_subst_vars. By default, now just a
wrapper for subst_vars. In compatibility mode, invokes the
old logic.
(do_txeval, do_output_line): Call tx_subst_vars rather than
subst_vars.

* match.h (format_field): Declaration removed.

* parser.y (expr): Grammar production removed: no longer
referenced.
(o_var): Braced variable case now parsed as n_expr, and
expanded as expr by default, since this is Lisp now. In
compatibility mode, expanded using expand_meta.
Also SYMTOK case must be subject to expansion; an output
var can now be a symbol macro.
(expand_meta): Expand a quasi-literal as Lisp, except
in compatibility mode.

* txr.1: Bit of a documentation update. Existing doc isn't
totally clear.

6 files changed, 209 insertions, 174 deletions

diff --git a/eval.c b/eval.c
index 9edc6ae6..a63f369b 100644
--- a/eval.c
+++ b/eval.c
@@ -53,6 +53,7 @@
 #include "lisplib.h"
 #include "struct.h"
 #include "cadr.h"
+#include "filter.h"
 #include "eval.h"
 
 #define max(a, b) ((a) > (b) ? (a) : (b))
@@ -2114,7 +2115,101 @@ static val op_handler_bind(val form, val env)
   return result;
 }
 
-static val subst_vars(val forms, val env)
+val format_field(val obj, val modifier, val filter, val eval_fun)
+{
+  val n = zero, sep = lit(" ");
+  val plist = nil;
+  val str;
+
+  for (; modifier; pop(&modifier)) {
+    val item = first(modifier);
+    if (regexp(item)) {
+      uw_throw(query_error_s, lit("bad field format: regex modifier in output"));
+    } else if (keywordp(item)) {
+      plist = modifier;
+      break;
+    } else if ((!opt_compat || opt_compat > 128) &&
+               consp(item) && car(item) == expr_s)
+    {
+      item = cdr(item);
+      goto eval;
+    } else if (consp(item) && car(item) == dwim_s) {
+      val arg_expr = second(item);
+
+      if (consp(arg_expr) && car(arg_expr) == range_s) {
+        val from = funcall1(eval_fun, second(arg_expr));
+        val to = funcall1(eval_fun, third(arg_expr));
+
+        obj = sub(obj, from, to);
+      } else {
+         val arg = funcall1(eval_fun, arg_expr);
+         if (bignump(arg) || fixnump(arg)) {
+           obj = ref(obj, arg);
+         } else if (rangep(arg)) {
+           obj = sub(obj, from(arg), to(arg));
+         } else {
+           uw_throwf(query_error_s, lit("format_field: bad index: ~s"),
+                     arg, nao);
+         }
+      }
+    } else eval: {
+      val v = funcall1(eval_fun, item);
+      if (fixnump(v))
+        n = v;
+      else if (stringp(v))
+        sep = v;
+      else
+        uw_throwf(query_error_s,
+                  lit("bad field format: bad modifier object: ~s"),
+                  item, nao);
+    }
+  }
+
+  if (listp(obj))
+    str = cat_str(mapcar(func_n1(tostringp), obj), sep);
+  else
+    str = if3(stringp(obj), obj, tostringp(obj));
+
+  {
+    val filter_sym = getplist(plist, filter_k);
+
+    if (filter_sym) {
+      filter = get_filter(filter_sym);
+
+      if (!filter) {
+        uw_throwf(query_error_s,
+                  lit("bad field format: ~s specifies unknown filter"),
+                  filter_sym, nao);
+      }
+    }
+
+    if (filter)
+      str = filter_string_tree(filter, str);
+  }
+
+  {
+    val right = lt(n, zero);
+    val width = if3(lt(n, zero), neg(n), n);
+    val diff = minus(width, length_str(str));
+
+    if (le(diff, zero))
+      return str;
+
+    if (ge(length_str(str), width))
+      return str;
+
+    {
+      val padding = mkstring(diff, chr(' '));
+
+      return if3(right,
+                 cat_str(list(padding, str, nao), nil),
+                 cat_str(list(str, padding, nao), nil));
+    }
+  }
+}
+
+
+val subst_vars(val forms, val env, val filter)
 {
   list_collect_decl(out, iter);
 
@@ -2136,18 +2231,18 @@ static val subst_vars(val forms, val env)
           str = tostringp(str);
 
         if (modifiers) {
-          forms = cons(format_field(str, modifiers, nil,
+          forms = cons(format_field(str, modifiers, filter,
                                     curry_123_1(func_n3(eval), env, form)),
                        rest(forms));
         } else {
           if (listp(str))
             str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
-          forms = cons(str, rest(forms));
+          forms = cons(filter_string_tree(filter, str), rest(forms));
         }
 
         continue;
       } else if (sym == quasi_s) {
-        val nested = subst_vars(rest(form), env);
+        val nested = subst_vars(rest(form), env, filter);
         iter = list_collect_append(iter, nested);
         forms = cdr(forms);
         continue;
@@ -2157,10 +2252,10 @@ static val subst_vars(val forms, val env)
           str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
         else if (!stringp(str))
           str = tostringp(str);
-        forms = cons(str, rest(forms));
+        forms = cons(filter_string_tree(filter, tostringp(str)), rest(forms));
         continue;
       } else {
-        val nested = subst_vars(form, env);
+        val nested = subst_vars(form, env, filter);
         iter = list_collect_append(iter, nested);
         forms = cdr(forms);
         continue;
@@ -2179,7 +2274,7 @@ static val subst_vars(val forms, val env)
 
 static val op_quasi_lit(val form, val env)
 {
-  return cat_str(subst_vars(rest(form), env), nil);
+  return cat_str(subst_vars(rest(form), env, nil), nil);
 }
 
 static val op_with_saved_vars(val form, val env)
diff --git a/eval.h b/eval.h
index 67254820..5fc4c516 100644
--- a/eval.h
+++ b/eval.h
@@ -59,6 +59,8 @@ val apply_intrinsic(val fun, val args);
 val eval_progn(val forms, val env, val ctx_form);
 val eval(val form, val env, val ctx_form);
 val eval_intrinsic(val form, val env);
+val format_field(val string_or_list, val modifier, val filter, val eval_fun);
+val subst_vars(val forms, val env, val filter);
 val load(val target);
 val expand(val form, val menv);
 val expand_forms(val forms, val menv);
diff --git a/match.c b/match.c
index 999d971b..68f228ec 100644
--- a/match.c
+++ b/match.c
@@ -1384,164 +1384,86 @@ static val match_line_completely(match_line_ctx c)
   return result;
 }
 
-
-val format_field(val obj, val modifier, val filter, val eval_fun)
-{
-  val n = zero, sep = lit(" ");
-  val plist = nil;
-  val str;
-
-  for (; modifier; pop(&modifier)) {
-    val item = first(modifier);
-    if (regexp(item)) {
-      uw_throw(query_error_s, lit("bad field format: regex modifier in output"));
-    } else if (keywordp(item)) {
-      plist = modifier;
-      break;
-    } else if (consp(item) && car(item) == dwim_s) {
-      val arg_expr = second(item);
-
-      if (consp(arg_expr) && car(arg_expr) == range_s) {
-        val from = funcall1(eval_fun, second(arg_expr));
-        val to = funcall1(eval_fun, third(arg_expr));
-
-        obj = sub(obj, from, to);
-      } else {
-         val arg = funcall1(eval_fun, arg_expr);
-         if (bignump(arg) || fixnump(arg)) {
-           obj = ref(obj, arg);
-         } else if (rangep(arg)) {
-           obj = sub(obj, from(arg), to(arg));
-         } else {
-           uw_throwf(query_error_s, lit("format_field: bad index: ~s"),
-                     arg, nao);
-         }
-      }
-    } else {
-      val v = funcall1(eval_fun, item);
-      if (fixnump(v))
-        n = v;
-      else if (stringp(v))
-        sep = v;
-      else
-        uw_throwf(query_error_s,
-                  lit("bad field format: bad modifier object: ~s"),
-                  item, nao);
-    }
-  }
-
-  if (listp(obj))
-    str = cat_str(mapcar(func_n1(tostringp), obj), sep);
-  else
-    str = if3(stringp(obj), obj, tostringp(obj));
-
-  {
-    val filter_sym = getplist(plist, filter_k);
-
-    if (filter_sym) {
-      filter = get_filter(filter_sym);
-
-      if (!filter) {
-        uw_throwf(query_error_s,
-                  lit("bad field format: ~s specifies unknown filter"),
-                  filter_sym, nao);
-      }
-    }
-
-    if (filter)
-      str = filter_string_tree(filter, str);
-  }
-
-  {
-    val right = lt(n, zero);
-    val width = if3(lt(n, zero), neg(n), n);
-    val diff = minus(width, length_str(str));
-
-    if (le(diff, zero))
-      return str;
-
-    if (ge(length_str(str), width))
-      return str;
-
-    {
-      val padding = mkstring(diff, chr(' '));
-
-      return if3(right,
-                 cat_str(list(padding, str, nao), nil),
-                 cat_str(list(str, padding, nao), nil));
-    }
-  }
-}
-
-static val subst_vars(val spec, val bindings, val filter)
+static val tx_subst_vars(val spec, val bindings, val filter)
 {
-  list_collect_decl(out, iter);
-  uw_env_begin;
+  if (opt_compat && opt_compat <= 128) {
+    list_collect_decl(out, iter);
+    uw_env_begin;
 
-  uw_set_match_context(cons(spec, bindings));
+    uw_set_match_context(cons(spec, bindings));
 
-  while (spec) {
-    val elem = first(spec);
+    while (spec) {
+      val elem = first(spec);
 
-    if (consp(elem)) {
-      val sym = first(elem);
+      if (consp(elem)) {
+        val sym = first(elem);
 
-      if (sym == var_s) {
-        val expr = second(elem);
-        val modifiers = third(elem);
-        val str = txeval(spec, expr, bindings);
+        if (sym == var_s) {
+          val expr = second(elem);
+          val modifiers = third(elem);
+          val str = txeval(spec, expr, bindings);
 
-        /* If the object is a list, we let format_field deal with the
-           conversion to text, because the modifiers influence how
-           it is done. */
-        if (!stringp(str) && !listp(str))
-          str = tostringp(str);
+          /* If the object is a list, we let format_field deal with the
+             conversion to text, because the modifiers influence how
+             it is done. */
+          if (!stringp(str) && !listp(str))
+            str = tostringp(str);
 
-        if (modifiers) {
-          spec = cons(format_field(str, modifiers, filter,
-                                   curry_123_2(func_n3(txeval), spec, bindings)),
-                      rest(spec));
-        } else {
-          if (listp(str))
-            str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
+          if (modifiers) {
+            spec = cons(format_field(str, modifiers, filter,
+                                     curry_123_2(func_n3(txeval), spec, bindings)),
+                        rest(spec));
+          } else {
+            if (listp(str))
+              str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
 
-          spec = cons(filter_string_tree(filter, str), rest(spec));
-        }
+            spec = cons(filter_string_tree(filter, str), rest(spec));
+          }
 
-        continue;
-      } else if (sym == quasi_s) {
-        val nested = subst_vars(rest(elem), bindings, filter);
-        iter = list_collect_append(iter, nested);
-        spec = cdr(spec);
-        continue;
-      } else if (sym == expr_s) {
-        if (opt_compat && opt_compat < 100) {
-          val result = eval_with_bindings(rest(elem), spec, bindings, elem);
-          spec = cons(filter_string_tree(filter, tostringp(result)), rest(spec));
           continue;
+        } else if (sym == quasi_s) {
+          val nested = tx_subst_vars(rest(elem), bindings, filter);
+          iter = list_collect_append(iter, nested);
+          spec = cdr(spec);
+          continue;
+        } else if (sym == expr_s) {
+          if (opt_compat && opt_compat < 100) {
+            val result = eval_with_bindings(rest(elem), spec, bindings, elem);
+            spec = cons(filter_string_tree(filter, tostringp(result)), rest(spec));
+            continue;
+          } else {
+            val str = eval_with_bindings(rest(elem), spec, bindings, elem);
+            if (listp(str))
+              str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
+            else if (!stringp(str))
+              str = tostringp(str);
+            spec = cons(filter_string_tree(filter, tostringp(str)), rest(spec));
+            continue;
+          }
         } else {
-          val str = eval_with_bindings(rest(elem), spec, bindings, elem);
-          if (listp(str))
-            str = cat_str(mapcar(func_n1(tostringp), str), lit(" "));
-          else if (!stringp(str))
-            str = tostringp(str);
-          spec = cons(filter_string_tree(filter, tostringp(str)), rest(spec));
+          val nested = tx_subst_vars(elem, bindings, filter);
+          iter = list_collect_append(iter, nested);
+          spec = cdr(spec);
           continue;
         }
-      } else {
-        val nested = subst_vars(elem, bindings, filter);
-        iter = list_collect_append(iter, nested);
-        spec = cdr(spec);
-        continue;
       }
+
+      iter = list_collect(iter, elem);
+      spec = cdr(spec);
     }
 
-    iter = list_collect(iter, elem);
-    spec = cdr(spec);
-  }
+    uw_env_end;
+    return out;
+  } else {
+    val saved_de = set_dyn_env(make_env(bindings, nil, nil));
+    val out;
 
-  uw_env_end;
-  return out;
+    uw_set_match_context(cons(spec, bindings));
+    out = subst_vars(spec, nil, filter);
+    set_dyn_env(saved_de);
+
+    return out;
+  }
 }
 
 static val do_txeval(val spec, val form, val bindings, val allow_unbound)
@@ -1570,12 +1492,12 @@ static val do_txeval(val spec, val form, val bindings, val allow_unbound)
     } else if (consp(form)) {
       val sym = first(form);
       if (sym == quasi_s) {
-        ret = cat_str(subst_vars(rest(form), bindings, nil), nil);
+        ret = cat_str(tx_subst_vars(rest(form), bindings, nil), nil);
       } else if (sym == quasilist_s) {
         val iter;
         list_collect_decl (out, tail);
         for (iter = rest(form); iter != nil; iter = cdr(iter))
-          tail = list_collect(tail, subst_vars(cdr(car(iter)), bindings, nil));
+          tail = list_collect(tail, tx_subst_vars(cdr(car(iter)), bindings, nil));
         ret = out;
       } else if (sym == var_s) {
         ret = eval_with_bindings(second(form), spec, bindings, form);
@@ -1770,8 +1692,8 @@ static void do_output_line(val bindings, val specline, val filter, val out)
         val directive = first(elem);
 
         if (directive == var_s) {
-          val str = cat_str(subst_vars(cons(elem, nil),
-                                       bindings, filter), nil);
+          val str = cat_str(tx_subst_vars(cons(elem, nil),
+                                          bindings, filter), nil);
           if (str == nil)
             sem_error(specline, lit("bad substitution: ~a"),
                       second(elem), nao);
@@ -1884,8 +1806,8 @@ static void do_output_line(val bindings, val specline, val filter, val out)
             format(out, lit("~a"),
                    eval_with_bindings(rest(elem), elem, bindings, elem), nao);
           } else {
-            val str = cat_str(subst_vars(cons(elem, nil),
-                                         bindings, filter), nil);
+            val str = cat_str(tx_subst_vars(cons(elem, nil),
+                                            bindings, filter), nil);
             if (str == nil)
               sem_error(specline, lit("bad substitution: ~a"),
                         second(elem), nao);
diff --git a/match.h b/match.h
index d8f075a3..7d6e4ee4 100644
--- a/match.h
+++ b/match.h
@@ -27,7 +27,6 @@
 extern val text_s, choose_s, gather_s, do_s, require_s;
 extern val close_s, load_s, include_s, mod_s, modlast_s, line_s;
 extern val counter_k, env_k, var_k;
-val format_field(val string_or_list, val modifier, val filter, val eval_fun);
 val match_filter(val name, val arg, val other_args);
 val match_fun(val name, val args, val input, val files);
 val include(val specline);
diff --git a/parser.y b/parser.y
index 7fb42fec..fe3d22aa 100644
--- a/parser.y
+++ b/parser.y
@@ -119,7 +119,7 @@ int yyparse(scanner_t *, parser_t *);
 %type <val> if_clause elif_clauses_opt else_clause_opt
 %type <val> line elems_opt elems clause_parts_h additional_parts_h
 %type <val> text texts elem var var_op modifiers vector hash struct range
-%type <val> list exprs exprs_opt expr n_exprs r_exprs i_expr n_expr n_exprs_opt
+%type <val> list exprs exprs_opt n_exprs r_exprs i_expr n_expr n_exprs_opt
 %type <val> out_clauses out_clauses_opt out_clause
 %type <val> repeat_clause repeat_parts_opt o_line
 %type <val> o_elems_opt o_elems o_elem o_var q_var rep_elem rep_parts_opt
@@ -711,10 +711,20 @@ modifiers : NUMBER              { $$ = cons($1, nil); }
                                                  nil), $1); }
           ;
 
-o_var : SYMTOK                  { $$ = list(var_s, symhlpr($1, nil), nao);
+o_var : SYMTOK                  { val expr = symhlpr($1, nil);
+                                  if (!opt_compat || opt_compat > 128)
+                                    expr = expand(expr, nil);
+                                  $$ = list(var_s, expr, nao);
                                   rl($$, num(parser->lineno)); }
-      | '{' expr exprs_opt '}'
-                                { $$ = list(var_s, $2, $3, nao);
+      | '{' n_expr n_exprs_opt '}'
+                                { if (opt_compat && opt_compat <= 128)
+                                  { $$ = list(var_s,
+                                              expand_meta($2, nil),
+                                              expand_meta($3, nil), nao); }
+                                  else
+                                  { $$ = list(var_s,
+                                              expand($2, nil),
+                                              expand($3, nil), nao); }
                                   rl($$, num(parser->lineno)); }
       | SYMTOK error            { $$ = nil;
                                   yybadtok(yychar, lit("variable spec")); }
@@ -786,9 +796,6 @@ list : '(' n_exprs ')'          { $$ = rl($2, num($1)); }
 exprs : n_exprs                 { $$ = rlcp(expand_meta($1, nil), $1); }
       ;
 
-expr : n_expr                   { $$ = rlcp(expand_meta($1, nil), $1); }
-     ;
-
 exprs_opt : exprs               { $$ = $1; }
           | /* empty */         { $$ = nil; }
           ;
@@ -1318,19 +1325,23 @@ static val expand_meta(val form, val menv)
   menv = default_arg(menv, make_env(nil, nil, nil));
 
   if ((sym = car(form)) == quasi_s) {
-    list_collect_decl (out, ptail);
+    if (opt_compat && opt_compat <= 128) {
+      list_collect_decl (out, ptail);
 
-    for (; consp(form); form = cdr(form)) {
-      val subform = car(form);
-      if (consp(subform) && car(subform) == expr_s)
-        ptail = list_collect(ptail, expand_meta(subform, menv));
-      else
-        ptail = list_collect(ptail, subform);
-    }
+      for (; consp(form); form = cdr(form)) {
+        val subform = car(form);
+        if (consp(subform) && car(subform) == expr_s)
+          ptail = list_collect(ptail, expand_meta(subform, menv));
+        else
+          ptail = list_collect(ptail, subform);
+      }
 
-    ptail = list_collect_nconc(ptail, form);
+      ptail = list_collect_nconc(ptail, form);
+
+      return rlcp(out, form);
+    }
 
-    return rlcp(out, form);
+    return expand(form, nil);
   }
 
   if ((sym = car(form)) == expr_s) {
diff --git a/txr.1 b/txr.1
index 003ace07..19b9f58d 100644
--- a/txr.1
+++ b/txr.1
@@ -2555,8 +2555,7 @@ escape for encoding a literal
 .code @
 character.  Quasiliterals support the full output variable
 syntax. Expressions within variables substitutions follow the evaluation rules
-of \*(TL when the quasiliteral occurs in \*(TL, and the rules of
-the \*(TX pattern language when the quasiliteral occurs in the pattern language.
+of \*(TL.  This hasn't always been the case: see the COMPATIBILITY section.
 
 Quasiliterals can be split into multiple lines in the same way as ordinary
 string literals.
@@ -38863,6 +38862,13 @@ of these version values, the described behaviors are provided if
 is given an argument which is equal or lower. For instance
 .code -C 103
 selects the behaviors described below for version 105, but not those for 102.
+.IP 128
+Compatibility with \*(TX 128 or earlier brings back the behavior that
+expressions in quasiliterals are evaluated according to \*(TX evaluation
+rules for quasiliterals which occur in the \*(TX pattern language.
+Similarly, expressions in
+.code @(output)
+blocks are treated \*(TX pattern language expressions.
 .IP 127
 In versions of \*(TX until 127, the functions
 .codn symbol-function ,