vim-patch:9.0.2114: overflow detection not accurate when adding digits (#28271)

Problem:  overflow detection not accurate when adding digits
Solution: Use a helper function

Use a helper function to better detect overflows before adding integer
digits to a long or an integer variable respectively. Signal the
overflow to the caller function.

closes: vim/vim#13539

22cbc8a4e1

Co-authored-by: Christian Brabandt <cb@256bit.org>

src/nvim/input.c

@@ -14,6 +14,7 @@
 #include "nvim/highlight_defs.h"
 #include "nvim/input.h"
 #include "nvim/keycodes.h"
+#include "nvim/math.h"
 #include "nvim/mbyte.h"
 #include "nvim/memory.h"
 #include "nvim/message.h"
@@ -21,6 +22,7 @@
 #include "nvim/os/input.h"
 #include "nvim/state_defs.h"
 #include "nvim/ui.h"
+#include "nvim/vim_defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "input.c.generated.h"  // IWYU pragma: export
@@ -180,10 +182,9 @@ int get_number(int colon, bool *mouse_used)
     ui_cursor_goto(msg_row, msg_col);
     int c = safe_vgetc();
     if (ascii_isdigit(c)) {
-      if (n > INT_MAX / 10) {
+      if (vim_append_digit_int(&n, c - '0') == FAIL) {
         return 0;
       }
-      n = n * 10 + c - '0';
       msg_putchar(c);
       typed++;
     } else if (c == K_DEL || c == K_KDEL || c == K_BS || c == Ctrl_H) {

src/nvim/math.c

@@ -1,14 +1,16 @@
 // uncrustify:off
 #include <math.h>
 // uncrustify:on
+#include <limits.h>
 #include <stdint.h>
 #include <string.h>
 
-#ifdef _MSC_VER
+#ifdef HAVE_BITSCANFORWARD64
 # include <intrin.h>  // Required for _BitScanForward64
 #endif
 
 #include "nvim/math.h"
+#include "nvim/vim_defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "math.c.generated.h"
@@ -74,3 +76,14 @@ int xctz(uint64_t x)
   return count;
 #endif
 }
+
+/// For overflow detection, add a digit safely to an int value.
+int vim_append_digit_int(int *value, int digit)
+{
+  int x = *value;
+  if (x > ((INT_MAX - digit) / 10)) {
+    return FAIL;
+  }
+  *value = x * 10 + digit;
+  return OK;
+}

src/nvim/normal.c

@@ -48,6 +48,7 @@
 #include "nvim/mapping.h"
 #include "nvim/mark.h"
 #include "nvim/mark_defs.h"
+#include "nvim/math.h"
 #include "nvim/mbyte.h"
 #include "nvim/memline.h"
 #include "nvim/memline_defs.h"
@@ -2639,11 +2640,10 @@ static bool nv_z_get_count(cmdarg_T *cap, int *nchar_arg)
     if (nchar == K_DEL || nchar == K_KDEL) {
       n /= 10;
     } else if (ascii_isdigit(nchar)) {
-      if (n > INT_MAX / 10) {
+      if (vim_append_digit_int(&n, nchar - '0') == FAIL) {
         clearopbeep(cap->oap);
         break;
       }
-      n = n * 10 + (nchar - '0');
     } else if (nchar == CAR) {
       win_setheight(n);
       break;