vim-patch:8.1.1720: crash with very long %[] pattern

Problem:    Crash with very long %[] pattern. (Reza Mirzazade farkhani)
Solution:   Check for reg_toolong. (closes vim/vim#4703)
2a5b52758b

src/nvim/regexp.c

@@ -2058,10 +2058,14 @@ static char_u *regatom(int *flagp)
             EMSG2_RET_NULL(_(e_missing_sb),
                 reg_magic == MAGIC_ALL);
           br = regnode(BRANCH);
-          if (ret == NULL)
+          if (ret == NULL) {
             ret = br;
-          else
+          } else {
             regtail(lastnode, br);
+            if (reg_toolong) {
+              return NULL;
+            }
+          }
 
           ungetchr();
           one_exactly = TRUE;
@@ -2083,6 +2087,9 @@ static char_u *regatom(int *flagp)
           for (br = ret; br != lastnode; ) {
             if (OP(br) == BRANCH) {
               regtail(br, lastbranch);
+              if (reg_toolong) {
+                return NULL;
+              }
               br = OPERAND(br);
             } else
               br = regnext(br);

src/nvim/testdir/test_regexp_utf8.vim

@@ -183,3 +183,12 @@ func Test_large_class()
   call assert_equal(1, "\u3042" =~# '[\u3000-\u4000]')
   set re=0
 endfunc
+
+func Test_optmatch_toolong()
+  set re=1
+  " Can only handle about 8000 characters.
+  let pat = '\\%[' .. repeat('x', 9000) .. ']'
+  call assert_fails('call match("abc def", "' .. pat .. '")', 'E339:')
+  set re=0
+endfunc
+