From 835fbf4248cce666199f41a118befd1063fdea2e Mon Sep 17 00:00:00 2001
From: Kaz Kylheku <kaz@kylheku.com>
Date: Mon, 14 Feb 2022 06:35:58 -0800
Subject: Fix some issues found by ubsan.

* lib.h (num_fast): Under HAVE_UBSAN, avoid left shifting a
negative value, which is diagnosed by ubsan (by default), but
is otherwise a documented GCC extension that behaves sanely in
the manner one would expect on two's complement machines.

* mpi/mpi.c (s_mp_mul_2d): Fix instance of undefined shift
equal to the width of the type. This occurs in the case when
d == 0.  In that case, we try to calculate a 32 or 64 bit full
mask by shifting 1 to the left that many times and then
subtracting 1. However, the entire code for the fractional
shift is not necessary in that case and may be skipped. I'm
also removing the unnecesary s_mp_clamp call. If the number is
clamped on entry into the function, it stays clamped.
---
 lib.h     |  4 ++++
 mpi/mpi.c | 46 +++++++++++++++++++++++-----------------------
 regex.c   |  3 ++-
 3 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/lib.h b/lib.h
index 5d095108..d2d7a486 100644
--- a/lib.h
+++ b/lib.h
@@ -491,7 +491,11 @@ INLINE wchar_t *litptr(val obj)
 
 INLINE val num_fast(cnum n)
 {
+#if HAVE_UBSAN
+  return coerce(val, (n * (1 << TAG_SHIFT)) | TAG_NUM);
+#else
   return coerce(val, (n << TAG_SHIFT) | TAG_NUM);
+#endif
 }
 
 INLINE mp_int *mp(val bign)
diff --git a/mpi/mpi.c b/mpi/mpi.c
index 34003253..5aa3d06f 100644
--- a/mpi/mpi.c
+++ b/mpi/mpi.c
@@ -3385,35 +3385,35 @@ mp_err s_mp_mul_2d(mp_int *mp, mp_digit d)
     return res;
 
   dp = DIGITS(mp); used = USED(mp);
-  d %= DIGIT_BIT;
 
-  mask = (convert(mp_digit, 1) << d) - 1;
+  if ((d %= DIGIT_BIT) != 0) {
+    mask = (convert(mp_digit, 1) << d) - 1;
 
-  /* If the shift requires another digit, make sure we've got one to
-     work with */
-  if ((dp[used - 1] >> (DIGIT_BIT - d)) & mask) {
-    if ((res = s_mp_grow(mp, used + 1)) != MP_OKAY)
-      return res;
-    dp = DIGITS(mp);
-  }
+    /* If the shift requires another digit, make sure we've got one to
+       work with */
+    if ((dp[used - 1] >> (DIGIT_BIT - d)) & mask) {
+      if ((res = s_mp_grow(mp, used + 1)) != MP_OKAY)
+        return res;
+      dp = DIGITS(mp);
+    }
 
-  /* Do the shifting... */
-  save = 0;
-  for (ix = 0; ix < used; ix++) {
-    next = (dp[ix] >> (DIGIT_BIT - d)) & mask;
-    dp[ix] = (dp[ix] << d) | save;
-    save = next;
-  }
+    /* Do the shifting... */
+    save = 0;
+    for (ix = 0; ix < used; ix++) {
+      next = (dp[ix] >> (DIGIT_BIT - d)) & mask;
+      dp[ix] = (dp[ix] << d) | save;
+      save = next;
+    }
 
-  /* If, at this point, we have a nonzero carryout into the next
-   * digit, we'll increase the size by one digit, and store it...
-   */
-  if (save) {
-    dp[used] = save;
-    USED(mp) += 1;
+    /* If, at this point, we have a nonzero carryout into the next
+     * digit, we'll increase the size by one digit, and store it...
+     */
+    if (save) {
+      dp[used] = save;
+      USED(mp) += 1;
+    }
   }
 
-  s_mp_clamp(mp);
   return MP_OKAY;
 }
 
diff --git a/regex.c b/regex.c
index 213ac98f..1e8034d3 100644
--- a/regex.c
+++ b/regex.c
@@ -300,7 +300,8 @@ static void L0_fill_range(cset_L0_t *L0, wchar_t ch0, wchar_t ch1)
 
 static int L0_contains(cset_L0_t *L0, wchar_t ch)
 {
-  return ((*L0)[CHAR_SET_INDEX(ch)] & (1 << CHAR_SET_BIT(ch))) != 0;
+  return ((*L0)[CHAR_SET_INDEX(ch)] &
+          (convert(bitcell_t, 1) << CHAR_SET_BIT(ch))) != 0;
 }
 
 static int L1_full(cset_L1_t *L1)
-- 
cgit v1.2.3