The code expander becomes a public API.

The functions sys:expand, sys:expand* and
sys:expand-with-free-refs are now in the usr package and
documented for public use.

* eval.c (eval_init): Move registrations of the symbools
expand, expand* and expand-with-free-refs from the
system package to the user package.

* share/txr/stdlib/awk.tl (sys:awk-mac-let, awk): Uses of
sys:expand drop the sys: prefix.

* share/txr/stdlib/op.tl (sys:op-alpha-rename): Likewise.

* share/txr/stdlib/place.tl (call-upudate-expander,
call-clobber-expander, call-delete-expander, sys:placelet-1):
Likewise.

* tests/011/macros-2.txr, tests/012/struct.tl: Likewise.

* txr.1: Documented expand, expand* and expand-with-free-refs.

7 files changed, 318 insertions, 29 deletions

diff --git a/eval.c b/eval.c
index e28e1164..05c07ce8 100644
--- a/eval.c
+++ b/eval.c
@@ -6456,9 +6456,9 @@ void eval_init(void)
   reg_var(load_path_s, nil);
   reg_symacro(intern(lit("self-load-path"), user_package), load_path_s);
   reg_var(load_recursive_s, nil);
-  reg_fun(intern(lit("expand"), system_package), func_n2o(no_warn_expand, 1));
-  reg_fun(intern(lit("expand*"), system_package), func_n2o(expand, 1));
-  reg_fun(intern(lit("expand-with-free-refs"), system_package),
+  reg_fun(intern(lit("expand"), user_package), func_n2o(no_warn_expand, 1));
+  reg_fun(intern(lit("expand*"), user_package), func_n2o(expand, 1));
+  reg_fun(intern(lit("expand-with-free-refs"), user_package),
           func_n3o(expand_with_free_refs, 1));
   reg_fun(intern(lit("macro-form-p"), user_package), func_n2o(macro_form_p, 1));
   reg_fun(intern(lit("macroexpand-1"), user_package),
diff --git a/share/txr/stdlib/awk.tl b/share/txr/stdlib/awk.tl
index 4e603f9e..cb08eeb4 100644
--- a/share/txr/stdlib/awk.tl
+++ b/share/txr/stdlib/awk.tl
@@ -375,9 +375,9 @@
                   (let* ((style (car form))
                          (ix (pinc (qref ,awc nranges)))
                          (rng-temp (gensym))
-                         (from-expr-ex (sys:expand from-expr e))
+                         (from-expr-ex (expand from-expr e))
                          (from-expr-val (gensym))
-                         (to-expr-ex (sys:expand to-expr e))
+                         (to-expr-ex (expand to-expr e))
                          (to-expr-val (gensym))
                          (vec-temp (qref ,awc rng-vec-temp))
                          (emul-broken (and (plusp sys:compat) (<= sys:compat 177)))
@@ -393,10 +393,8 @@
                                 (to-expr-ex te-fv te-ff te-ev te-ef)
                                 (from-expr-orig to-expr-orig))
                                (list
-                                 (sys:expand-with-free-refs from-expr e
-                                                            ,awc.outer-env)
-                                 (sys:expand-with-free-refs to-expr e
-                                                            ,awc.outer-env)
+                                 (expand-with-free-refs from-expr e ,awc.outer-env)
+                                 (expand-with-free-refs to-expr e ,awc.outer-env)
                                  (list (cadr form) (caddr form)))
                       (sys:awk-code-move-check ,awc ',aws-sym
                                                form from-expr-orig
@@ -479,7 +477,7 @@
       (let* ((p-actions-xform-unex (mapcar (aret ^(when (sys:awk-test ,@1 rec)
                                                     ,*@rest))
                                            awc.cond-actions))
-             (p-actions-xform (sys:expand
+             (p-actions-xform (expand
                                 ^(sys:awk-mac-let ,awc ,aws-sym
                                    ,*p-actions-xform-unex)
                                 (sys:awk-fun-shadowing-env outer-env))))
diff --git a/share/txr/stdlib/op.tl b/share/txr/stdlib/op.tl
index d496249c..3c3d85b0 100644
--- a/share/txr/stdlib/op.tl
+++ b/share/txr/stdlib/op.tl
@@ -71,7 +71,7 @@
                            ,*(if do-nested-metas
                                ^((,(slot ctx 'meta) ((quote arg)) arg))))
                   ,op-args)))
-    (sys:expand code e)))
+    (expand code e)))
 
 (defun sys:op-expand (f e args)
   (let* ((ctx (make-struct 'sys:op-ctx ^(form ,f)))
diff --git a/share/txr/stdlib/place.tl b/share/txr/stdlib/place.tl
index 26ca1d94..6e85d3d8 100644
--- a/share/txr/stdlib/place.tl
+++ b/share/txr/stdlib/place.tl
@@ -148,7 +148,7 @@
          (expander (get-update-expander place))
          (sys:*pl-env* env)
          (expansion [expander getter setter place body])
-         (expansion-ex (sys:expand expansion env)))
+         (expansion-ex (expand expansion env)))
     (sys:propagate-ancestor expansion-ex place getter setter)))
 
 (defun call-clobber-expander (ssetter unex-place env body)
@@ -157,7 +157,7 @@
          (expander (get-clobber-expander place))
          (sys:*pl-env* env)
          (expansion [expander ssetter place body])
-         (expansion-ex (sys:expand expansion env)))
+         (expansion-ex (expand expansion env)))
     (sys:propagate-ancestor expansion-ex place ssetter)))
 
 (defun call-delete-expander (deleter unex-place env body)
@@ -166,7 +166,7 @@
          (expander (get-delete-expander place))
          (sys:*pl-env* env)
          (expansion [expander deleter place body])
-         (expansion-ex (sys:expand expansion env)))
+         (expansion-ex (expand expansion env)))
     (sys:propagate-ancestor expansion-ex place deleter)))
 
 (defmacro with-update-expander ((getter setter) unex-place env body)
@@ -874,8 +874,8 @@
                       ,tmp-body)))
         (call-update-expander pl-getter pl-setter place env
           ^(macrolet ((,tmp-place () ^(,',pl-getter)))
-             ,(sys:expand ^(symacrolet ((,sym (,tmp-place)))
-                             ,*body) env))))
+             ,(expand ^(symacrolet ((,sym (,tmp-place)))
+                         ,*body) env))))
       (remhash *place-update-expander* tmp-place))))
 
 (defmacro placelet* (sym-place-pairs . body)
diff --git a/tests/011/macros-2.txr b/tests/011/macros-2.txr
index a6693c6b..0d7cc607 100644
--- a/tests/011/macros-2.txr
+++ b/tests/011/macros-2.txr
@@ -22,7 +22,7 @@
 
    (prinl
      (ignwarn
-       (sys:expand
+       (expand
          '(whilst ((< i 100))
             (if (< (inc i) 20)
               continue)
diff --git a/tests/012/struct.tl b/tests/012/struct.tl
index 99b2a3eb..9de3f832 100644
--- a/tests/012/struct.tl
+++ b/tests/012/struct.tl
@@ -13,7 +13,7 @@
 (test ^#S(bar b ,(+ 2 2))
       #S(bar a 103 b 4))
 
-(test (sys:expand '^#S(bar b ,(+ 2 2)))
+(test (expand '^#S(bar b ,(+ 2 2)))
       (sys:make-struct-lit 'bar (list 'b (+ 2 2))))
 
 (defvar s (eval ^#S(bar b ,(+ 2 2))))
@@ -24,36 +24,36 @@
 (test s
       #S(bar a 100 b 4))
 
-(test (ignwarn (sys:expand 'a.b.c.d))
+(test (ignwarn (expand 'a.b.c.d))
       (slot (slot (slot a 'b)
                   'c) 'd))
 
-(test (sys:expand 's.a)
+(test (expand 's.a)
       (slot s 'a))
-(test (sys:expand 's.[a])
+(test (expand 's.[a])
       [(slot s 'a)])
-(test (sys:expand 's.[a b c])
+(test (expand 's.[a b c])
       [(slot s 'a) b c])
 
 (set *gensym-counter* 0)
-(stest (ignwarn (sys:expand 's.(a)))
+(stest (ignwarn (expand 's.(a)))
        "(call (slot s 'a)\n                     \
        \      s)")
 (set *gensym-counter* 0)
-(stest (ignwarn (sys:expand 's.(a b c)))
+(stest (ignwarn (expand 's.(a b c)))
        "(call (slot s 'a)\n                     \
        \      s b c)")
-(test (sys:expand 's.[a].d)
+(test (expand 's.[a].d)
       (slot [(slot s 'a)] 'd))
-(test (sys:expand 's.[a b c].d)
+(test (expand 's.[a b c].d)
       (slot [(slot s 'a) b c] 'd))
 (set *gensym-counter* 0)
-(stest (ignwarn (sys:expand 's.(a).d))
+(stest (ignwarn (expand 's.(a).d))
        "(slot (call (slot s 'a)\n               \
        \            s)\n                        \
        \      'd)")
 (set *gensym-counter* 0)
-(stest (ignwarn (sys:expand 's.(a b c).d))
+(stest (ignwarn (expand 's.(a b c).d))
        "(slot (call (slot s 'a)\n               \
        \            s b c)\n                    \
        \      'd)")
@@ -63,7 +63,7 @@
 (test (new foo) #S(foo a 42))
 
 (set *gensym-counter* 0)
-(stest (sys:expand '(defstruct (boa x y) nil
+(stest (expand '(defstruct (boa x y) nil
                       (x 0) (y 0)))
        "(sys:make-struct-type 'boa '() '()\n                                            \
        \                      '(x y) () (lambda (#:g0008)\n                             \
diff --git a/txr.1 b/txr.1
index 4dbb8516..dad42dd7 100644
--- a/txr.1
+++ b/txr.1
@@ -31264,6 +31264,297 @@ one or more argument forms to be treated in a Lisp-1 context, in situations
 when such a macro needs to itself expand the material, rather than merely
 insert it as-is into the output code template.
 
+.coNP Functions @ expand and @ *expand
+.synb
+.mets (expand < form <> [ env ])
+.mets (expand* < form <> [ env ])
+.syne
+.desc
+The functions
+.code expand
+and
+.code expand*
+both perform a complete expansion of
+.meta form
+in the macro-environment
+.metn env ,
+and return that expansion.
+
+If
+.meta env
+is omitted, the expansion takes place in the global environment in
+which only global macros are visible.
+
+The returned object is a structure that
+is devoid of any macro calls. Also, all
+.code macrolet
+and
+.code symacrolet
+blocks in form
+.meta form
+are removed in the returned structure, replaced by their fully
+expanded bodies.
+
+The difference between
+.code expand
+and
+.code expand*
+is that
+.code expand
+suppresses any warning exceptions that are issued during expansion.
+
+.coNP Function @ expand-with-free-refs
+.synb
+.mets (expand-with-free-refs < form >> [ inner-env <> [ outer-env ]])
+.syne
+.desc
+The
+.code expand-with-free-refs
+form performs a full expansion of
+.metn form ,
+as if by the
+.code expand
+function and returns a list containing that expansion, plus four additional
+items which provide information about variable and function references which
+occur in
+.metn form .
+
+If both
+.meta inner-env
+and
+.meta outer-env
+are provided, then it is expected that
+.meta inner-env
+is lexically nested within
+.metn outer-env .
+
+Note: it is not required that
+.meta outer-env
+be the immediate parent of
+.metn inner-env .
+
+Note: a common usage situation is that
+.meta outer-env
+is the environment of the invocation of a "parent" macro which generates a form
+that contains local macros. The bodies of those local macros use
+.codn expand-with-free-refs ,
+specifying their own environment as
+.meta inner-env
+and that of their generating "parent" as
+.metn outer-env .
+
+In detail, the five items of the returned list are
+.cblk
+.meti >> ( expansion < fv-inner < ff-inner < fv-outer << ff-outer )
+.cble
+whose descriptions are:
+.RS
+.meIP < expansion
+The full expansion of
+.metn form ,
+containing no macro invocations, or
+.code symacrolet
+or
+.code macrolet
+forms.
+.meIP < fv-inner
+A list of the free variables which occur in
+.meta form
+relative to the
+.meta inner-env
+environment. That is to say, variables that are not bound inside
+.meta form
+and are not also bound in
+.metn inner-env .
+If
+.meta inner-env
+is omitted, then these are the absolutely free variables
+occurring in
+.metn form .
+.meIP < ff-inner
+Exactly like
+.meta fv-inner
+but informing about function bindings rather than variables.
+.meIP < fv-outer
+A list of the variables which which occur in
+.meta form
+which would be free if the environments between
+.meta inner-env
+and
+.meta outer-env
+(including the former, excluding the latter)
+were removed from consideration. A more detailed description of this semantics
+is given below. If
+.meta outer-env
+is omitted, then these are the absolutely free variables
+occurring in
+.metn form ,
+ignoring the
+.metn inner-env .
+.meIP < ff-outer
+Exactly like
+.meta fv-outer
+but informing about function bindings rather than variables.
+.RE
+
+.IP
+The semantics of the treatment of
+.meta inner-env
+and
+.meta outer-env
+in the calculation of
+.meta fv-outer
+and
+.meta ff-outer
+is as follows.  A new environment
+.meta diff-env
+is calculated from these two environments, and
+.meta form
+is expanded in this environment. Variables and functions occurring in
+.meta form
+which are not bound in
+.meta diff-env
+are listed as
+.meta fv-outer
+and
+.metn ff-outer .
+
+This
+.meta diff-env
+is calculated as follows. First
+.meta diff-env
+is initialized as a copy of
+.metn outer-env .
+Then, all environments below
+.meta outer-env
+down to
+.meta inner-env
+are examined for bindings which shadow bindings in
+.metn diff-env .
+Those shadows are removed from
+.metn diff-env .
+Therefore, what remains in
+.meta diff-env
+are those bindings from
+.meta outer-env
+that are
+.I not
+shadowed by the environments between
+.meta inner-env
+and
+.metn outer-env .
+
+.TP* Example:
+
+Suppose that
+.code mac
+is a macro which somehow has access to the two indicated lexical environments
+in the following code snippet:
+
+.cblk
+  (let (a c) ;; <- outer-env
+    (let (b)
+      (let (c) ;; <- inner-env
+        (mac (list a b c d))))
+.cble
+
+Suppose that
+.code mac
+invokes the
+.code expand-with-free-refs
+function, passing in the
+.code "(list a b c d)"
+argument form as
+.code form
+and two macro-time environment objects corresponding to the indicated
+environments.
+
+The return value of
+.code expand-with-free-refs
+shall be:
+
+.cblk
+  ((list a b c d) (a d) (list) (b c d) (list))
+.cble
+
+The
+.meta fv-inner
+list is
+.code "(d)"
+because this is the only variable that occurs in
+.code "(list a b c d)"
+which is free with regard to
+.metn inner-env .
+The
+.codn a ,
+.code b
+and
+.code c
+variables are not listed because they appear bound inside
+.metn inner-env .
+
+The reported
+.meta fv-outer
+list is
+.code "(b c d)"
+because the form is considered against
+.meta diff-env
+which is formed by removing the shadowing bindings from
+.metn outer-env .
+The difference between
+.code "(a c)"
+and
+.code "(b c)"
+is
+.code a
+and so the form is considered in an environment containing the binding
+.code a
+which leaves
+.code "(b c d)"
+free.
+
+Note: this information is useful because a set difference can be calculated
+between the two reported sets. The set difference between the
+.meta fv-outer
+variables
+.code "(b c d)"
+and the
+.meta fv-inner
+variables
+.code "(d)"
+is
+.codn "(b c)" .
+
+That set difference
+.code "(b c)"
+is significant because it precisely informs about the
+.I bound
+variables which occur in
+.code "(list a b c d)"
+which appear bound in
+.metn inner-env ,
+but are not bound due to a binding coming from
+.metn outer-env .
+
+The variable
+.code d
+is not listed in
+.code "(b c)"
+because it is not a bound variable.
+The variable
+.code a
+is not in
+.code "(b c)"
+because though it is bound in
+.metn inner-env ,
+that binding comes from
+.metn outer-env .
+
+The upshot of this logic is that it allows a macro to inspect a form in order
+to discover the identities of the variables and functions which are used inside
+that form, whose definitions come from a specific, bounded scope surrounding
+that form.
+
 .coNP Functions @ lexical-var-p and @ lexical-fun-p
 .synb
 .mets (lexical-var-p < env << form )