bugfix: fixnum crackdown.

The purpose of this commit is to address certain situations in
which code is wrongly relying on a cnum value being in the
fixnum range (NUM_MIN to NUM_MAX), so that num_fast can safely
be used on it.

One wrong pattern is that c_num is applied to some Lisp value,
and that value (or one derived from it arithmetically) is then
passed to num_fast. The problem is that c_num succeeds on
integers outside of the fixnum range.  Some bignum values
convert to a cnum successfully. Thus either num has to be used
instead of num_fast, or else the original c_num attempt must
be replaced with something that will fail if the original
value isn't a fixnum. (In the latter case, any arithmetic on
the fixnum cannot produce value outside of that range).

* buf.c (buf_put_bytes): The size argument here is not
guaranteed to be in fixnum range: use num.

* combi.c (perm_init_common): Throw if the sequence length
isn't a fixnum. Thus the num_fast in perm_while_fun is
correct, since the ci value is bounded by k, which is bounded
by n.

* hash.c (hash_grow): Remove dubious assertion which aborts
the run-time if the hash table doubling overflows.  Simply
don't allow the modulus to grow beyond NUM_MAX.  If doubling
it makes it larger than NUM_MAX, then just don't grow the
table. We need the modulus to be in fixnum range, so that
uses of num_fast on the modulus value elsewhere are correct.
(group_by, group_reduce): Use c_fixnum rather than c_num to
extract a value that is later assumed to be a fixnum.

* lib.c (c_fixnum): New function.
(nreverse, reverse, remove_if, less, window_map_list,
sort_vec, unique): Use c_fixnum rather than c_num to extract a
value that is later assumed to be a fixnum.
(string_extend): Use c_fixnum rather than c_num to extract a
value that is later assumed to be a fixnum. Cap the string
allocation size to fixnum range rather than INT_PTR_MAX.
(cmp_str): The wcscmp function could return values outside
of the fixnum range, so we must use num, not num_fast.

* lib.h (c_fixnum): Declared.

5 files changed, 54 insertions, 26 deletions

diff --git a/buf.c b/buf.c
index b22ff2b2..6a5ed15b 100644
--- a/buf.c
+++ b/buf.c
@@ -228,7 +228,7 @@ static void buf_put_bytes(val buf, val pos, mem_t *ptr, cnum size, val self)
 {
   struct buf *b = buf_handle(buf, self);
   cnum p = buf_check_index(pos, self);
-  val req_len = plus(pos, num_fast(size));
+  val req_len = plus(pos, num(size));
   if (gt(req_len, b->len))
     buf_do_set_len(buf, b, req_len, nil, self);
   memcpy(b->data + p, ptr, size);
diff --git a/combi.c b/combi.c
index 51bbdc9b..c749b300 100644
--- a/combi.c
+++ b/combi.c
@@ -100,6 +100,10 @@ static val perm_init_common(val p, val k_null)
   val n = length(p);
   val k = or2(k_null, n);
 
+  if (!fixnump(n))
+    uw_throwf(error_s, lit("perm: sequence length ~s is out of fixnum range"),
+              n, nao);
+
   if (gt(k, n)) {
     return nil;
   } else {
diff --git a/hash.c b/hash.c
index c7603f55..7198c2dc 100644
--- a/hash.c
+++ b/hash.c
@@ -539,9 +539,12 @@ static void hash_grow(struct hash *h, val hash)
 {
   cnum i;
   cnum new_modulus = 2 * h->modulus;
-  val new_table = vector(num_fast(new_modulus), nil);
+  val new_table;
 
-  bug_unless (new_modulus > h->modulus);
+  if (new_modulus > NUM_MAX)
+    return;
+
+  new_table = vector(num_fast(new_modulus), nil);
 
   for (i = 0; i < h->modulus; i++) {
     val conses = vecref(h->table, num_fast(i));
@@ -1096,12 +1099,13 @@ val hash_list(val keys, struct args *hashv_args)
 
 val group_by(val func, val seq, struct args *hashv_args)
 {
+  val self = lit("group-by");
   val hash = hashv(hashv_args);
 
   if (vectorp(seq)) {
     cnum i, len;
 
-    for (i = 0, len = c_num(length(seq)); i < len; i++) {
+    for (i = 0, len = c_fixnum(length(seq), self); i < len; i++) {
       val v = vecref(seq, num_fast(i));
       pushhash(hash, funcall1(func, v), v);
     }
@@ -1126,12 +1130,13 @@ val group_by(val func, val seq, struct args *hashv_args)
 val group_reduce(val hash, val by_fun, val reduce_fun, val seq,
                  val initval, val filter_fun)
 {
+  val self = lit("group-reduce");
   initval = default_null_arg(initval);
 
   if (vectorp(seq)) {
     cnum i, len;
 
-    for (i = 0, len = c_num(length(seq)); i < len; i++) {
+    for (i = 0, len = c_fixnum(length(seq), self); i < len; i++) {
       val v = vecref(seq, num_fast(i));
       val key = funcall1(by_fun, v);
       val new_p;
diff --git a/lib.c b/lib.c
index 657b39f3..fbc46683 100644
--- a/lib.c
+++ b/lib.c
@@ -1054,6 +1054,7 @@ loc list_collect_revappend(loc ptail, val obj)
 
 val nreverse(val in)
 {
+  val self = lit("nreverse");
   seq_info_t si = seq_info(in);
 
   switch (si.kind) {
@@ -1076,7 +1077,7 @@ val nreverse(val in)
   case SEQ_VECLIKE:
     {
       val in = si.obj;
-      cnum len = c_num(length(in));
+      cnum len = c_fixnum(length(in), self);
       cnum i;
 
       for (i = 0; i < len / 2; i++) {
@@ -1095,6 +1096,7 @@ val nreverse(val in)
 
 val reverse(val seq_in)
 {
+  val self = lit("reverse");
   seq_info_t si = seq_info(seq_in);
 
   switch (si.kind) {
@@ -1118,7 +1120,7 @@ val reverse(val seq_in)
 
     {
       val obj = copy(si.obj);
-      cnum len = c_num(length(si.obj));
+      cnum len = c_fixnum(length(si.obj), self);
       cnum i;
 
       for (i = 0; i < len / 2; i++) {
@@ -1624,7 +1626,7 @@ static val rem_impl(val (*eqfun)(val, val), val name,
     {
       val out = mkustring(zero);
       val str = seq_in;
-      cnum len = c_num(length_str(str)), i;
+      cnum len = c_fixnum(length_str(str), name), i;
 
       for (i = 0; i < len; i++) {
         val elem = chr_str(str, num_fast(i));
@@ -1640,7 +1642,7 @@ static val rem_impl(val (*eqfun)(val, val), val name,
     {
       val out = vector(zero, nil);
       val vec = seq_in;
-      cnum len = c_num(length_vec(vec)), i;
+      cnum len = c_fixnum(length_vec(vec), name), i;
 
       for (i = 0; i < len; i++) {
         val elem = vecref(vec, num_fast(i));
@@ -1659,6 +1661,7 @@ static val rem_impl(val (*eqfun)(val, val), val name,
 
 val remove_if(val pred, val seq_in, val keyfun_in)
 {
+  val self = lit("remove-if");
   val keyfun = default_null_arg(keyfun_in);
 
   switch (type(seq_in)) {
@@ -1692,7 +1695,7 @@ val remove_if(val pred, val seq_in, val keyfun_in)
     {
       val out = mkustring(zero);
       val str = seq_in;
-      cnum len = c_num(length_str(str)), i;
+      cnum len = c_fixnum(length_str(str), self), i;
 
       for (i = 0; i < len; i++) {
         val elem = chr_str(str, num_fast(i));
@@ -1708,7 +1711,7 @@ val remove_if(val pred, val seq_in, val keyfun_in)
     {
       val out = vector(zero, nil);
       val vec = seq_in;
-      cnum len = c_num(length_vec(vec)), i;
+      cnum len = c_fixnum(length_vec(vec), self), i;
 
       for (i = 0; i < len; i++) {
         val elem = vecref(vec, num_fast(i));
@@ -3057,6 +3060,17 @@ cnum c_num(val num)
   }
 }
 
+cnum c_fixnum(val num, val self)
+{
+  switch (type(num)) {
+  case CHR: case NUM:
+    return coerce(cnum, num) >> TAG_SHIFT;
+  default:
+    type_mismatch(lit("~a: ~s is not fixnum integer or character"),
+                  self, num, nao);
+  }
+}
+
 val flo(double n)
 {
   val obj = make_obj();
@@ -3524,27 +3538,27 @@ val string_extend(val str, val tail)
   type_check(str, STR);
   {
     val self = lit("string-extend");
-    cnum len = c_num(length_str(str));
-    cnum oalloc = c_num(str->st.alloc), alloc = oalloc;
+    cnum len = c_fixnum(length_str(str), self);
+    cnum oalloc = c_fixnum(str->st.alloc, self), alloc = oalloc;
     cnum delta, needed;
 
     if (stringp(tail))
-      delta = c_num(length_str(tail));
+      delta = c_fixnum(length_str(tail), self);
     else if (chrp(tail))
       delta = 1;
     else if (integerp(tail))
-      delta = c_num(tail);
+      delta = c_fixnum(tail, self);
     else
       uw_throwf(error_s, lit("~s: tail ~s bad type"), self, str, nao);
 
-    if (INT_PTR_MAX - delta - 1 < len)
+    if (NUM_MAX - delta - 1 < len)
       uw_throwf(error_s, lit("~s: overflow"), self, nao);
 
     needed = len + delta + 1;
 
     if (needed > alloc) {
-      if (alloc >= (INT_PTR_MAX - INT_PTR_MAX / 5))
-        alloc = INT_PTR_MAX;
+      if (alloc >= (NUM_MAX - NUM_MAX / 5))
+        alloc = NUM_MAX;
       else
         alloc = max(alloc + alloc / 4, needed);
 
@@ -4399,7 +4413,7 @@ val cmp_str(val astr, val bstr)
    case TYPE_PAIR(STR, STR):
    case TYPE_PAIR(LIT, STR):
    case TYPE_PAIR(STR, LIT):
-     return num_fast(wcscmp(c_str(astr), c_str(bstr)));
+     return num(wcscmp(c_str(astr), c_str(bstr)));
    case TYPE_PAIR(LSTR, LIT):
    case TYPE_PAIR(LSTR, STR):
    case TYPE_PAIR(LIT, LSTR):
@@ -4614,6 +4628,7 @@ static void less_tab_init(void)
 
 val less(val left, val right)
 {
+  val self = lit("less");
   type_t l_type, r_type;
 
   if (left == right)
@@ -4692,8 +4707,8 @@ tail:
   case VEC:
     {
       cnum i;
-      cnum lenl = c_num(length_vec(left));
-      cnum lenr = c_num(length_vec(right));
+      cnum lenl = c_fixnum(length_vec(left), self);
+      cnum lenr = c_fixnum(length_vec(right), self);
       cnum len = min(lenl, lenr);
 
       for (i = 0; i < len; i++) {
@@ -7936,7 +7951,8 @@ static cnum calc_win_size(cnum ra)
 
 static val window_map_list(val range, val boundary, val fun, val list, val app)
 {
-  cnum i, j, ra = c_num(range), ws = calc_win_size(ra);
+  val self = lit("window-map");
+  cnum i, j, ra = c_fixnum(range, self), ws = calc_win_size(ra);
   val iter;
   args_decl (args, ws);
   list_collect_decl (out, ptail);
@@ -7963,7 +7979,7 @@ static val window_map_list(val range, val boundary, val fun, val list, val app)
     args->arg[i] = car(iter);
 
   for (j = ra; i < ws; i++)
-    args->arg[i] = ref(boundary, num_fast(j++));
+    args->arg[i] = ref(boundary, num(j++));
 
   for (;;) {
     args_decl (args_cp, ws);
@@ -7984,7 +8000,7 @@ static val window_map_list(val range, val boundary, val fun, val list, val app)
       args->arg[i] = car(iter);
       iter = cdr(iter);
     } else {
-      args->arg[i] = ref(boundary, num_fast(j++));
+      args->arg[i] = ref(boundary, num(j++));
     }
   }
 
@@ -8244,7 +8260,8 @@ static void quicksort(val vec, val lessfun, val keyfun, cnum from, cnum to)
 
 static void sort_vec(val vec, val lessfun, val keyfun)
 {
-  cnum len = c_num(length(vec));
+  val self = lit("sort");
+  cnum len = c_fixnum(length(vec), self);
   quicksort(vec, lessfun, keyfun, 0, len);
 }
 
@@ -8363,6 +8380,7 @@ val sort_group(val seq, val keyfun, val lessfun)
 
 val unique(val seq, val keyfun, struct args *hashv_args)
 {
+  val self = lit("unique");
   val hash = hashv(hashv_args);
   val kf = default_arg(keyfun, identity_f);
 
@@ -8371,7 +8389,7 @@ val unique(val seq, val keyfun, struct args *hashv_args)
   if (vectorp(seq) || stringp(seq)) {
     cnum i, len;
 
-    for (i = 0, len = c_num(length(seq)); i < len; i++) {
+    for (i = 0, len = c_fixnum(length(seq), self); i < len; i++) {
       val new_p;
       val v = ref(seq, num_fast(i));
 
diff --git a/lib.h b/lib.h
index 24d04695..a4138bda 100644
--- a/lib.h
+++ b/lib.h
@@ -646,6 +646,7 @@ val improper_plist_to_alist(val list, val boolean_keys);
 val num(cnum val);
 val flo(double val);
 cnum c_num(val num);
+cnum c_fixnum(val num, val self);
 double c_flo(val num);
 val fixnump(val num);
 val bignump(val num);