From d94373b4c627b204f365b25c3033eb0ac9ffcb84 Mon Sep 17 00:00:00 2001
From: Kaz Kylheku <kaz@kylheku.com>
Date: Sun, 29 Apr 2018 07:54:51 -0700
Subject: bugfix: include most negative two's in cnum range.

The INT_PTR_MIN value isn't the smallest value actually
representable in cnum, because it is just the additive
inverse of INT_PTR_MAX. In two's complement, the INT_PTR_MIN-1
value is still representable. But we are excluding it.
If a Lisp integer has the value INT_PTR_MIN-1, the c_num
function will fail to convert it to a cnum.
This situation occurs in FFI, where code may expect that the
Lisp value #x-80000000 can convert to an external 32 bit
integer type.  This will be done by way of a conversion to
cnum first via c_num (see ffi_i32_put for instance, which
calls c_i32 which relies on c_num).

* arith.c (INT_PTR_MAX_SUCC_MP): New static variable.
This holds a bignum equivalent to INT_PTR_MAX + 1.
(in_int_ptr_range): We now check whether the value
against the range [-(INT_PTR_MAX + 1), (INT_PTR_MAX + 1)]
and then check all possible return values. The MP_LT
case is clearly in range, and MP_GT is out of the range.
The interesting situation is MP_EQ: in that case
we just test the sign to see whether we are looking
at -(INT_PTR_MAX + 1).
(int_flo): INT_PTR_MIN is referenced in this function, so
we use INT_PTR_MIN - 1 instead. This allows that value
to be handled via the simple bignum(n) case.
(arith_init): Initialize INT_PTR_MAX_SUCC_MP. We cannot
initialize it to INT_PTR_MAX + 1 directly because that
expression overflows: insted we use INT_PTR_MIN - 1 and
then flip the resulting bignum's sign.
(arith_free_all): Call mp_clear on the new variable to
release its digit buffer.

* ffi.c (make_ffi_type_enum): Use INT_PTR_MIN - 1 as the
initial value of the highest variable, to accurately
calculate the range of the enum values if they contain
INT_PTR_MIN - 1.
---
 arith.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

(limited to 'arith.c')

diff --git a/arith.c b/arith.c
index f0c7e168..3f2c5d56 100644
--- a/arith.c
+++ b/arith.c
@@ -50,6 +50,7 @@
 #define ABS(A) ((A) < 0 ? -(A) : (A))
 
 static mp_int NUM_MAX_MP, INT_PTR_MAX_MP, UINT_PTR_MAX_MP;
+static mp_int INT_PTR_MAX_SUCC_MP;
 
 val make_bignum(void)
 {
@@ -149,7 +150,17 @@ val normalize(val bignum)
 
 val in_int_ptr_range(val bignum)
 {
-  return (mp_cmp_mag(mp(bignum), &INT_PTR_MAX_MP) == MP_GT) ? nil : t;
+  switch (mp_cmp_mag(mp(bignum), &INT_PTR_MAX_SUCC_MP)) {
+  default:
+  case MP_GT:
+    return nil;
+  case MP_EQ:
+    if (mp_cmp_z(mp(bignum)) == MP_GT)
+      return nil;
+    /* fallthrough */
+  case MP_LT:
+    return t;
+  }
 }
 
 static val in_uint_ptr_range(val bignum)
@@ -2180,7 +2191,7 @@ val int_flo(val f)
 {
   double d = c_flo(f);
 
-  if (d >= INT_PTR_MAX && d <= INT_PTR_MIN) {
+  if (d >= INT_PTR_MAX && d <= INT_PTR_MIN - 1) {
     cnum n = d;
     if (n < NUM_MIN || n > NUM_MAX)
       return bignum(n);
@@ -3079,6 +3090,9 @@ void arith_init(void)
   mp_set_intptr(&INT_PTR_MAX_MP, INT_PTR_MAX);
   mp_init(&UINT_PTR_MAX_MP);
   mp_set_uintptr(&UINT_PTR_MAX_MP, -1);
+  mp_init(&INT_PTR_MAX_SUCC_MP);
+  mp_set_intptr(&INT_PTR_MAX_SUCC_MP, INT_PTR_MIN - 1);
+  mp_neg(&INT_PTR_MAX_SUCC_MP, &INT_PTR_MAX_SUCC_MP);
   log2_init();
 
   reg_varl(intern(lit("*flo-dig*"), user_package), num_fast(DBL_DIG));
@@ -3118,4 +3132,5 @@ void arith_free_all(void)
   mp_clear(&NUM_MAX_MP);
   mp_clear(&INT_PTR_MAX_MP);
   mp_clear(&UINT_PTR_MAX_MP);
+  mp_clear(&INT_PTR_MAX_SUCC_MP);
 }
-- 
cgit v1.2.3