Fix some issues found by ubsan.

* lib.h (num_fast): Under HAVE_UBSAN, avoid left shifting a
negative value, which is diagnosed by ubsan (by default), but
is otherwise a documented GCC extension that behaves sanely in
the manner one would expect on two's complement machines.

* mpi/mpi.c (s_mp_mul_2d): Fix instance of undefined shift
equal to the width of the type. This occurs in the case when
d == 0.  In that case, we try to calculate a 32 or 64 bit full
mask by shifting 1 to the left that many times and then
subtracting 1. However, the entire code for the fractional
shift is not necessary in that case and may be skipped. I'm
also removing the unnecesary s_mp_clamp call. If the number is
clamped on entry into the function, it stays clamped.

1 files changed, 23 insertions, 23 deletions

diff --git a/mpi/mpi.c b/mpi/mpi.c
index 34003253..5aa3d06f 100644
--- a/mpi/mpi.c
+++ b/mpi/mpi.c
@@ -3385,35 +3385,35 @@ mp_err s_mp_mul_2d(mp_int *mp, mp_digit d)
     return res;
 
   dp = DIGITS(mp); used = USED(mp);
-  d %= DIGIT_BIT;
 
-  mask = (convert(mp_digit, 1) << d) - 1;
+  if ((d %= DIGIT_BIT) != 0) {
+    mask = (convert(mp_digit, 1) << d) - 1;
 
-  /* If the shift requires another digit, make sure we've got one to
-     work with */
-  if ((dp[used - 1] >> (DIGIT_BIT - d)) & mask) {
-    if ((res = s_mp_grow(mp, used + 1)) != MP_OKAY)
-      return res;
-    dp = DIGITS(mp);
-  }
+    /* If the shift requires another digit, make sure we've got one to
+       work with */
+    if ((dp[used - 1] >> (DIGIT_BIT - d)) & mask) {
+      if ((res = s_mp_grow(mp, used + 1)) != MP_OKAY)
+        return res;
+      dp = DIGITS(mp);
+    }
 
-  /* Do the shifting... */
-  save = 0;
-  for (ix = 0; ix < used; ix++) {
-    next = (dp[ix] >> (DIGIT_BIT - d)) & mask;
-    dp[ix] = (dp[ix] << d) | save;
-    save = next;
-  }
+    /* Do the shifting... */
+    save = 0;
+    for (ix = 0; ix < used; ix++) {
+      next = (dp[ix] >> (DIGIT_BIT - d)) & mask;
+      dp[ix] = (dp[ix] << d) | save;
+      save = next;
+    }
 
-  /* If, at this point, we have a nonzero carryout into the next
-   * digit, we'll increase the size by one digit, and store it...
-   */
-  if (save) {
-    dp[used] = save;
-    USED(mp) += 1;
+    /* If, at this point, we have a nonzero carryout into the next
+     * digit, we'll increase the size by one digit, and store it...
+     */
+    if (save) {
+      dp[used] = save;
+      USED(mp) += 1;
+    }
   }
 
-  s_mp_clamp(mp);
   return MP_OKAY;
 }