Converting expander special case code transformations into

formal macros that are in the top_mb table, make their symbols
fboundp and can be expanded with macroexpand.

* eval.c (mefun_t): New typedef name.
(expand_macro): If the expander is a cobj, then pull out the C function
and call it, otherwise realize the interpreted macro as before.
(me_gen, me_delay): New static functions, replace expand_gen
and expand_delay.
(expand_qquote): Renamed to me_quote.
(expand_gen, expand_delay): Renamed to me_gen and me_delay,
with an interface adjustment and moved.
(expand_op): Renamed to me_op.
(expand): Removed qquote, gen, delay, op, and do handling, since
these operators are now macros.
Removed the unnecessary expansion of with-saved-vars.
(reg_op, reg_fun): Assert that the symbol is not nil, to catch
initialization order issues.  One just showed up: op_do was
interned in match.c, which is initialized later.
(reg_mac): New static function
(eval_init): Intern do_s, because match.c hasn't done it yet
at this point. Register me_gen, me_delay, me_op (twice) and me_qquote
as intrinsic macros.

* txr.1: Documented those operators as macros.

3 files changed, 128 insertions, 65 deletions

diff --git a/ChangeLog b/ChangeLog
index 2bfdb1e5..2af57217 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,33 @@
 2014-02-26  Kaz Kylheku  <kaz@kylheku.com>
 
+	Converting expander special case code transformations into
+	formal macros that are in the top_mb table, make their symbols
+	fboundp and can be expanded with macroexpand.
+
+	* eval.c (mefun_t): New typedef name.
+	(expand_macro): If the expander is a cobj, then pull out the C function
+	and call it, otherwise realize the interpreted macro as before.
+	(me_gen, me_delay): New static functions, replace expand_gen
+	and expand_delay.
+	(expand_qquote): Renamed to me_quote.
+	(expand_gen, expand_delay): Renamed to me_gen and me_delay,
+	with an interface adjustment and moved.
+	(expand_op): Renamed to me_op.
+	(expand): Removed qquote, gen, delay, op, and do handling, since
+	these operators are now macros.
+	Removed the unnecessary expansion of with-saved-vars.
+	(reg_op, reg_fun): Assert that the symbol is not nil, to catch
+	initialization order issues.  One just showed up: op_do was
+	interned in match.c, which is initialized later.
+	(reg_mac): New static function
+	(eval_init): Intern do_s, because match.c hasn't done it yet
+	at this point. Register me_gen, me_delay, me_op (twice) and me_qquote
+	as intrinsic macros.
+
+	* txr.1: Documented those operators as macros.
+
+2014-02-26  Kaz Kylheku  <kaz@kylheku.com>
+
 	* eval.c (reg_op): New static function.
 	(eval_init): Register operators with reg_op instead of direct sethash
 	calls.
diff --git a/eval.c b/eval.c
index 36878b64..8be374d6 100644
--- a/eval.c
+++ b/eval.c
@@ -65,6 +65,7 @@
 #include "eval.h"
 
 typedef val (*opfun_t)(val, val);
+typedef val (*mefun_t)(val, val);
 
 struct c_var {
   val *loc;
@@ -1278,19 +1279,24 @@ static val op_defmacro(val form, val env)
 
 static val expand_macro(val form, val expander, val menv)
 {
-  debug_enter;
-  val name = car(form);
-  val args = rest(form);
-  val env = car(cdr(expander));
-  val params = car(cdr(cdr(expander)));
-  val body = cdr(cdr(cdr(expander)));
-  val saved_de = set_dyn_env(make_env(nil, nil, dyn_env));
-  val exp_env = bind_macro_params(env, menv, params, args, nil, form);
-  debug_frame(name, args, nil, env, nil, nil, nil);
-  debug_return(eval_progn(body, exp_env, body));
-  debug_end;
-  set_dyn_env(saved_de); /* not reached but shuts up compiler */
-  debug_leave;
+  if (cobjp(expander)) {
+    mefun_t fp = (mefun_t) cptr_get(expander);
+    return fp(form, menv);
+  } else {
+    debug_enter;
+    val name = car(form);
+    val args = rest(form);
+    val env = car(cdr(expander));
+    val params = car(cdr(cdr(expander)));
+    val body = cdr(cdr(cdr(expander)));
+    val saved_de = set_dyn_env(make_env(nil, nil, dyn_env));
+    val exp_env = bind_macro_params(env, menv, params, args, nil, form);
+    debug_frame(name, args, nil, env, nil, nil, nil);
+    debug_return(eval_progn(body, exp_env, body));
+    debug_end;
+    set_dyn_env(saved_de); /* not reached but shuts up compiler */
+    debug_leave;
+  }
 }
 
 static val expand_macrolet(val form, val menv)
@@ -1932,6 +1938,23 @@ static val op_with_saved_vars(val form, val env)
   return result;
 }
 
+static val me_gen(val form, val menv)
+{
+  (void) menv;
+  return list(generate_s,
+              list(lambda_s, nil, second(form), nao),
+              list(lambda_s, nil, third(form), nao), nao);
+}
+
+static val me_delay(val form, val menv)
+{
+  (void) menv;
+  return list(cons_s,
+              cons(quote_s, cons(promise_s, nil)),
+              cons(lambda_s, cons(nil, rest(form))), nao);
+}
+
+
 val expand_forms(val form, val menv)
 {
   if (atom(form)) {
@@ -2027,6 +2050,11 @@ static val expand_qquote(val qquoted_form, val menv)
   abort();
 }
 
+static val me_qquote(val form, val menv)
+{
+  return expand_qquote(second(form), menv);
+}
+
 static val expand_vars(val vars, val menv, val form, val *spec_p)
 {
   val sym;
@@ -2095,20 +2123,6 @@ static val expand_quasi(val quasi_forms, val menv)
   }
 }
 
-static val expand_gen(val args)
-{
-  return list(generate_s,
-              list(lambda_s, nil, first(args), nao),
-              list(lambda_s, nil, second(args), nao), nao);
-}
-
-static val expand_delay(val args)
-{
-  return list(cons_s,
-              cons(quote_s, cons(promise_s, nil)),
-              cons(lambda_s, cons(nil, args)), nao);
-}
-
 static val format_op_arg(val num)
 {
   return format(nil, lit("arg-~,02s-"), num, nao);
@@ -2208,8 +2222,9 @@ static val supplement_op_syms(val ssyms, val max)
   return outsyms;
 }
 
-static val expand_op(val sym, val body, val menv)
+static val me_op(val form, val menv)
 {
+  cons_bind (sym, body, form);
   val body_ex = if3(sym == op_s, expand_forms(body, menv), expand(body, menv));
   val rest_gensym = gensym(lit("rest-"));
   cons_bind (syms, body_trans, transform_op(body_ex, nil, rest_gensym));
@@ -2395,8 +2410,6 @@ tail:
       return rlcp(cons(sym, cons(params_ex, cons(expr_ex, body_ex))), form);
     } else if (sym == quote_s || sym == fun_s) {
       return form;
-    } else if (sym == qquote_s) {
-      return expand_qquote(second(form), menv);
     } else if (sym == for_s || sym == for_star_s) {
       val vars = second(form);
       val cond = third(form);
@@ -2443,14 +2456,6 @@ tail:
       if (quasi == quasi_ex)
         return form;
       return rlcp(cons(sym, quasi_ex), form);
-    } else if (sym == gen_s) {
-      form = expand_gen(rest(form));
-      goto tail;
-    } else if (sym == delay_s) {
-      form = expand_delay(rest(form));
-      goto tail;
-    } else if (sym == op_s || sym == do_s) {
-      return expand_op(sym, rest(form), menv);
     } else if (sym == catch_s) {
       return expand_catch(rest(form), menv);
     } else if (sym == regex_s || regexp(sym)) {
@@ -2460,17 +2465,6 @@ tail:
       val args_ex = expand_forms(args, menv);
       val result = eval_progn(args_ex, make_env(nil, nil, nil), args);
       return cons(quote_s, cons(result, nil));
-    } else if (sym == with_saved_vars_s) {
-      /* We should never have to expand a machine-generated with-saved-vars
-       * produced by the expander itself. This is for the sake of someone
-       * testing with-saved-vars in isolation.
-       */
-      val vars = first(form);
-      val expr = second(form);
-      val expr_ex = expand(expr, menv);
-      if (expr == expr_ex)
-        return form;
-      return cons(vars, cons(expr_ex, nil));
     } else if (sym == macrolet_s) {
       return expand_macrolet(form, menv);
     } else if (sym == symacrolet_s) {
@@ -2485,7 +2479,7 @@ tail:
       /* funtion call
          also handles: progn, prog1, call, if, and, or, 
          unwind-protect, return, dwim, set, inc, dec,
-         push, pop, flip. */
+         push, pop, flip, and with-saved-vars. */
       val args = rest(form);
       val args_ex = expand_forms(args, menv);
 
@@ -2898,14 +2892,22 @@ static val env_hash(void)
 
 static void reg_op(val sym, opfun_t fun)
 {
+  assert (sym != 0);
   sethash(op_table, sym, cptr((mem_t *) fun));
 }
 
 static void reg_fun(val sym, val fun)
 {
+  assert (sym != 0);
   sethash(top_fb, sym, cons(sym, fun));
 }
 
+static void reg_mac(val sym, mefun_t fun)
+{
+  assert (sym != 0);
+  sethash(top_mb, sym, cptr((mem_t *) fun));
+}
+
 static void c_var_mark(val obj)
 {
   struct c_var *cv = (struct c_var *) obj->co.handle;
@@ -3029,6 +3031,7 @@ void eval_init(void)
   delay_s = intern(lit("delay"), user_package);
   promise_s = intern(lit("promise"), system_package);
   op_s = intern(lit("op"), user_package);
+  do_s = intern(lit("do"), user_package);
   rest_s = intern(lit("rest"), user_package);
   hash_lit_s = intern(lit("hash-construct"), system_package);
   hash_construct_s = intern(lit("hash-construct"), user_package);
@@ -3087,6 +3090,12 @@ void eval_init(void)
   reg_op(catch_s, op_catch);
   reg_op(with_saved_vars_s, op_with_saved_vars);
 
+  reg_mac(gen_s, me_gen);
+  reg_mac(delay_s, me_delay);
+  reg_mac(op_s, me_op);
+  reg_mac(do_s, me_op);
+  reg_mac(qquote_s, me_qquote);
+
   reg_fun(cons_s, func_n2(cons));
   reg_fun(intern(lit("make-lazy-cons"), user_package), func_n1(make_lazy_cons));
   reg_fun(intern(lit("lcons-fun"), user_package), func_n1(lcons_fun));
diff --git a/txr.1 b/txr.1
index 820e5d09..309a5cd9 100644
--- a/txr.1
+++ b/txr.1
@@ -4895,7 +4895,7 @@ of the variable a.
 Note that TXR Lisp does not have a distinct quote and backquote read syntax.
 There is only one quote, which supports unquoting. However, there is an
 underlying expression syntax which distinguishes them: see the documentation
-quote and qquote operator.
+quote operator and sys:qquote macro operator.
 
 A quoted form which contains no unquotes codifies an ordinary quote.
 
@@ -5236,12 +5236,36 @@ Example:
 
 
 
-.SH CONTROL FLOW AND SEQUENCING
+.SH OPERATOR AND FUNCTION LIBRARY
+
+A compound expression with a symbol as its first element, if
+intended to be evaluated, denotes either an operator invocation or a function
+call. This depends on whether the symbol names an operator or a function.
+
+When the form is an operator invocation, the interpretation of the meaning of
+that form is under the complete control of that operator. 
+
+Functions calls do not control the evaluation of the function call forms which
+invoke them; function call forms are evaluated according to very regular rules
+that apply to all function calls.
+
+Some operators are macros. There exist predefined macros in the library, and
+macro operators can also be user-defined using the macro-defining operator
+defmacro.  Operators that are not macros are called special operators. 
 
-When the first element of a compound expression is an operator symbol,
-the interpretation of the meaning of that form is under the complete control
-of that operator. The following sections list all of the operators available
-in TXR Lisp.
+Macro operators work as functions which are given the source code of the form.
+They analyze the form, and translate it to another form which is substituted in
+their place.   This happens during a code walking phase called the expansion
+phase, which is applied to Lisp code prior to evaluation. All macros are
+expanded in the expansion phase, resulting in code which contains only
+function calls and the executable forms of the operators.
+
+(Special operators can also perform code transformations during the expansion
+phase, but that is not considered macroexpansion, but rather an adjustment
+of the representation of the operator into an required executable form.)
+
+The following sections list all of the special operators, macros
+and functions in TXR Lisp.
 
 In these sections Syntax is indicated using these conventions:
 
@@ -5275,6 +5299,8 @@ alternative1 | alternative2 | ... | alternativeN
 Multiple syntactic variations allowed in one place are
 indicated as bar-separated items.
 
+.SH CONTROL FLOW AND SEQUENCING
+
 .SS Operators progn and prog1
 
 .TP
@@ -7947,7 +7973,7 @@ Description:
 The repeat function produces an infinite lazy list formed by the repeatedly
 cycled catenation of the argument lists.
 
-.SS Operator gen
+.SS Macro gen
 
 .TP
 Syntax:
@@ -7957,9 +7983,9 @@ Syntax:
 .TP
 Description:
 
-The gen operator produces a lazy list, in a manner similar to the generate
-function. Whereas the generate function takes functional arguments, the gen
-operator takes two expressions, which is often more convenient.
+The gen macro operator produces a lazy list, in a manner similar to the
+generate function. Whereas the generate function takes functional arguments,
+the gen operator takes two expressions, which is often more convenient.
 
 The return value of gen is a lazy list. When the lazy list is accessed, for
 instance with the functions car and cdr, it produces items on demand. Prior to
@@ -7988,7 +8014,7 @@ Example:
   Output:
   1 2 3
 
-.SS Operator delay
+.SS Macro delay
 
 .TP
 Syntax:
@@ -10773,7 +10799,7 @@ value is then used as the data item in the intersection hash.
 
 .SH PARTIAL EVALUATION AND COMBINATORS
 
-.SS Operators op and do
+.SS Macros op and do
 
 .TP
 Syntax:
@@ -10784,7 +10810,7 @@ Syntax:
 .TP
 Description:
 
-The op and do operators are similar. 
+The op and do macro operators are similar. 
 
 Like the lambda operator, the op operator creates an anonymous function.
 The difference is that the arguments of the function are implicit, or
@@ -12906,7 +12932,7 @@ Example:
 
   (quote (+ 2 2)) ;; yields (+ 2 2), not 4.
 
-.SS Operator sys:qquote
+.SS Macro sys:qquote
 
 .TP
 Syntax:
@@ -12916,7 +12942,7 @@ Syntax:
 .TP
 Description:
 
-The qquote (quasi-quote) operator implements a notation for convenient
+The qquote (quasi-quote) macro operator implements a notation for convenient
 list construction.  If <form> is an atom, or a list structure which
 does not contain any unquote or splice operators, then (qquote <form>)
 is equivalent to (quote <form>).