math: two bad edge cases in double_uintptr_t conversion.

This fixes two failing test cases introduced in the
parent commit.

* arith.c (c_dbl_unum): Here, what is wrong that if the
incoming value is a CHR or NUM, we just convert it to a signed
cnum, and return that value. The problem with this is that
negative values are supposed to be out of range for
double_uintptr_t. We now check for negative and route to
the out-of-range error.

* mpi/mpi.c (s_mp_in_big_range): Here, the edge case of
handling the most negative two's complement value is
incorrectly coded. We replace the logic by a simple test for
that exact special case. If a negative bignum being tested
whether it fits into the signed double_intptr_t, then
we check whether its mantissa has the 0x80..00 bit pattern.
That is the only value greater than 0x7F..FF that is
still in range, so we return 1 for that case. We remove the
bogus subtraction (top - neg). After handling the above
special value, we just need to look whether the most
significant word of the bignum is 0x7F...FF or lower.

2 files changed, 30 insertions, 4 deletions

diff --git a/arith.c b/arith.c
index d3f2b186..c195b7b0 100644
--- a/arith.c
+++ b/arith.c
@@ -247,18 +247,24 @@ dbl_ucnum c_dbl_unum(val n)
 {
   switch (type(n)) {
   case CHR: case NUM:
-    return coerce(cnum, n) >> TAG_SHIFT;
+    {
+      dbl_cnum cn = coerce(cnum, n) >> TAG_SHIFT;
+      if (cn >= 0)
+        return cn;
+      break;
+    }
   case BGNUM:
     if (mp_in_double_uintptr_range(mp(n))) {
       double_uintptr_t out;
       mp_get_double_uintptr(mp(n), &out);
       return out;
     }
-    uw_throwf(error_s, lit("~s is out of unsigned ~a bit range"),
-              n, num_fast(SIZEOF_DOUBLE_INTPTR * CHAR_BIT), nao);
+    break;
   default:
     type_mismatch(lit("~s is not an integer"), n, nao);
   }
+  uw_throwf(error_s, lit("~s is out of unsigned ~a bit range"),
+            n, num_fast(SIZEOF_DOUBLE_INTPTR * CHAR_BIT), nao);
 }
 
 #endif
diff --git a/mpi/mpi.c b/mpi/mpi.c
index b1eb7e49..291f3d10 100644
--- a/mpi/mpi.c
+++ b/mpi/mpi.c
@@ -582,10 +582,30 @@ static int s_mp_in_big_range(mp_int *mp, double_uintptr_t lim, int unsig)
   if (nd > ptrnd)
     return 0;
 
+  if (neg) {
+    mp_digit *dp = DIGITS(mp);
+    const mp_digit Ox8__0 = MP_DIGIT_MAX ^ (MP_DIGIT_MAX >> 1);
+
+    switch (ptrnd) {
+    case 1:
+      if (dp[0] == Ox8__0)
+        return 1;
+      break;
+    case 2:
+      if (dp[0] == 0 && dp[1] == Ox8__0)
+        return 1;
+      break;
+    case 4:
+      if (dp[0] == 0 && dp[1] == 0 && dp[2] == 0 && dp[3] == Ox8__0)
+        return 1;
+      break;
+    }
+  }
+
   {
     mp_digit top = DIGITS(mp)[ptrnd - 1];
     lim >>= ((ptrnd - 1) * MP_DIGIT_BIT);
-    return (top - neg) <= lim;
+    return top <= lim;
   }
 }