vim-patch:9.1.1204: MS-Windows: crash when passing long string to expand() (#32902)

Problem:  MS-Windows: crash when passing long string to expand() with
          'wildignorecase'.
Solution: Use the same buflen as unix_expandpath() in dos_expandpath().
          Remove an unnecessary STRLEN() while at it (zeertzjq).

closes: vim/vim#16896

00a749bd90
(cherry picked from commit ec8fc28743)

.cirrus.yml

@@ -6,7 +6,7 @@ freebsd_task:
   name: FreeBSD
   only_if: $BRANCH != "master"
   freebsd_instance:
-    image_family: freebsd-14-0
+    image_family: freebsd-14-2
   timeout_in: 30m
   install_script:
     - pkg install -y cmake gmake ninja unzip wget gettext python git

CMakeLists.txt

@@ -145,8 +145,8 @@ endif()
 # version string, else they are combined with the result of `git describe`.
 set(NVIM_VERSION_MAJOR 0)
 set(NVIM_VERSION_MINOR 10)
-set(NVIM_VERSION_PATCH 4)
-set(NVIM_VERSION_PRERELEASE "") # for package maintainers
+set(NVIM_VERSION_PATCH 5)
+set(NVIM_VERSION_PRERELEASE "-dev") # for package maintainers
 
 # API level
 set(NVIM_API_LEVEL 12)        # Bump this after any API change.

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -343,6 +343,10 @@ function STHighlighter:process_response(response, client, version)
     return
   end
 
+  if not api.nvim_buf_is_valid(self.bufnr) then
+    return
+  end
+
   -- if we have a response to a delta request, update the state of our tokens
   -- appropriately. if it's a full response, just use that
   local tokens ---@type integer[]
@@ -382,8 +386,10 @@ function STHighlighter:process_response(response, client, version)
   current_result.highlights = highlights
   current_result.namespace_cleared = false
 
-  -- redraw all windows displaying buffer
+  -- redraw all windows displaying buffer (if still valid)
+  if api.nvim_buf_is_valid(self.bufnr) then
     api.nvim__redraw({ buf = self.bufnr, valid = true })
+  end
 end
 
 --- on_win handler for the decoration provider (see |nvim_set_decoration_provider|)

runtime/lua/vim/lsp/sync.lua

@@ -212,7 +212,8 @@ end
 ---@param lastline integer
 ---@param new_lastline integer
 ---@param offset_encoding string
----@return vim.lsp.sync.Range, vim.lsp.sync.Range
+---@return vim.lsp.sync.Range prev_end_range
+---@return vim.lsp.sync.Range curr_end_range
 local function compute_end_range(
   prev_lines,
   curr_lines,
@@ -222,6 +223,16 @@ local function compute_end_range(
   new_lastline,
   offset_encoding
 )
+  -- A special case for the following `firstline == new_lastline` case where lines are deleted.
+  -- Even if the buffer has become empty, nvim behaves as if it has an empty line with eol.
+  if #curr_lines == 1 and curr_lines[1] == '' then
+    local prev_line = prev_lines[lastline - 1]
+    return {
+      line_idx = lastline - 1,
+      byte_idx = #prev_line + 1,
+      char_idx = compute_line_length(prev_line, offset_encoding) + 1,
+    }, { line_idx = 1, byte_idx = 1, char_idx = 1 }
+  end
   -- If firstline == new_lastline, the first change occurred on a line that was deleted.
   -- In this case, the last_byte...
   if firstline == new_lastline then

runtime/lua/vim/treesitter/highlighter.lua

@@ -240,7 +240,12 @@ end
 ---@return vim.treesitter.highlighter.Query
 function TSHighlighter:get_query(lang)
   if not self._queries[lang] then
-    self._queries[lang] = TSHighlighterQuery.new(lang)
+    local success, result = pcall(TSHighlighterQuery.new, lang)
+    if not success then
+      self:destroy()
+      error(result)
+    end
+    self._queries[lang] = result
   end
 
   return self._queries[lang]

runtime/windows_icon.rc

@@ -0,0 +1,4 @@
+// NOTE: this resource file *must* be in the same folder as the icon.
+// Otherwise, absolute paths would need to be used.
+// see https://learn.microsoft.com/en-us/windows/win32/menurc/icon-resource
+NEOVIM_ICON ICON "neovim.ico"

src/nvim/CMakeLists.txt

@@ -708,6 +708,12 @@ target_sources(main_lib INTERFACE
   ${EXTERNAL_SOURCES}
   ${EXTERNAL_HEADERS})
 
+if(WIN32)
+  # add windows resource file pointing to the neovim icon
+  # this makes the icon appear for the neovim exe and associated filetypes
+  target_sources(nvim_bin PRIVATE ${NVIM_RUNTIME_DIR}/windows_icon.rc)
+endif()
+
 target_sources(nlua0 PUBLIC ${NLUA0_SOURCES})
 
 target_link_libraries(nvim_bin PRIVATE main_lib PUBLIC libuv)

src/nvim/change.c

@@ -342,8 +342,8 @@ static void changed_common(buf_T *buf, linenr_T lnum, colnr_T col, linenr_T lnum
           && (last < wp->w_topline
               || (wp->w_topline >= lnum
                   && wp->w_topline < lnume
-                  && win_linetabsize(wp, wp->w_topline, ml_get_buf(buf, wp->w_topline), MAXCOL)
-                  <= (wp->w_skipcol + sms_marker_overlap(wp, -1))))) {
+                  && (linetabsize_eol(wp, wp->w_topline)
+                      <= wp->w_skipcol + sms_marker_overlap(wp, -1))))) {
         wp->w_skipcol = 0;
       }
 

src/nvim/cursor.c

@@ -129,7 +129,7 @@ static int coladvance2(win_T *wp, pos_T *pos, bool addspaces, bool finetune, col
         && wp->w_width_inner != 0
         && wcol >= (colnr_T)width
         && width > 0) {
-      csize = linetabsize(wp, pos->lnum);
+      csize = linetabsize_eol(wp, pos->lnum);
       if (csize > 0) {
         csize--;
       }

src/nvim/memline.c

@@ -1880,6 +1880,11 @@ static char *ml_get_buf_impl(buf_T *buf, linenr_T lnum, bool will_change)
   static int recursive = 0;
   static char questions[4];
 
+  if (buf->b_ml.ml_mfp == NULL) {       // there are no lines
+    buf->b_ml.ml_line_len = 1;
+    return "";
+  }
+
   if (lnum > buf->b_ml.ml_line_count) {  // invalid line number
     if (recursive == 0) {
       // Avoid giving this message for a recursive call, may happen when
@@ -1899,11 +1904,6 @@ errorret:
     lnum = 1;
   }
 
-  if (buf->b_ml.ml_mfp == NULL) {       // there are no lines
-    buf->b_ml.ml_line_len = 1;
-    return "";
-  }
-
   // See if it is the same line as requested last time.
   // Otherwise may need to flush last used line.
   // Don't use the last used line when 'swapfile' is reset, need to load all

src/nvim/message.c

@@ -1576,7 +1576,7 @@ int msg_outtrans_len(const char *msgstr, int len, int attr)
 
   // When drawing over the command line no need to clear it later or remove
   // the mode message.
-  if (msg_row >= cmdline_row && msg_col == 0) {
+  if (msg_silent == 0 && len > 0 && msg_row >= cmdline_row && msg_col == 0) {
     clear_cmdline = false;
     mode_displayed = false;
   }

src/nvim/move.c

@@ -1208,9 +1208,7 @@ static void cursor_correct_sms(win_T *wp)
   int width2 = width1 + win_col_off2(wp);
   int so_cols = so == 0 ? 0 : width1 + (so - 1) * width2;
   int space_cols = (wp->w_height_inner - 1) * width2;
-  int size = so == 0 ? 0 : win_linetabsize(wp, wp->w_topline,
-                                           ml_get_buf(wp->w_buffer, wp->w_topline),
-                                           (colnr_T)MAXCOL);
+  int size = so == 0 ? 0 : linetabsize_eol(wp, wp->w_topline);
 
   if (wp->w_topline == 1 && wp->w_skipcol == 0) {
     so_cols = 0;               // Ignore 'scrolloff' at top of buffer.
@@ -1226,9 +1224,10 @@ static void cursor_correct_sms(win_T *wp)
     so_cols -= width1;
   }
 
-  // If there is no marker or we have non-zero scrolloff, just ignore it.
-  int overlap = (wp->w_skipcol == 0 || so_cols != 0) ? 0 : sms_marker_overlap(wp, -1);
-  int top = wp->w_skipcol + overlap + so_cols;
+  int overlap = wp->w_skipcol == 0
+                ? 0 : sms_marker_overlap(wp, wp->w_width_inner - width2);
+  // If we have non-zero scrolloff, ignore marker overlap.
+  int top = wp->w_skipcol + (so_cols != 0 ? so_cols : overlap);
   int bot = wp->w_skipcol + width1 + (wp->w_height_inner - 1) * width2 - so_cols;
 
   validate_virtcol(wp);
@@ -1249,10 +1248,22 @@ static void cursor_correct_sms(win_T *wp)
 
   if (col != wp->w_virtcol) {
     wp->w_curswant = col;
-    coladvance(wp, wp->w_curswant);
+    int rc = coladvance(wp, wp->w_curswant);
     // validate_virtcol() marked various things as valid, but after
     // moving the cursor they need to be recomputed
     wp->w_valid &= ~(VALID_WROW|VALID_WCOL|VALID_CHEIGHT|VALID_CROW|VALID_VIRTCOL);
+    if (rc == FAIL && wp->w_skipcol > 0
+        && wp->w_cursor.lnum < wp->w_buffer->b_ml.ml_line_count) {
+      validate_virtcol(wp);
+      if (wp->w_virtcol < wp->w_skipcol + overlap) {
+        // Cursor still not visible: move it to the next line instead.
+        wp->w_cursor.lnum++;
+        wp->w_cursor.col = 0;
+        wp->w_cursor.coladd = 0;
+        wp->w_curswant = 0;
+        wp->w_valid &= ~VALID_VIRTCOL;
+      }
+    }
   }
 }
 
@@ -1365,8 +1376,7 @@ bool scrolldown(win_T *wp, linenr_T line_count, int byfold)
           wp->w_topline = first;
         } else {
           if (do_sms) {
-            int size = win_linetabsize(wp, wp->w_topline,
-                                       ml_get_buf(wp->w_buffer, wp->w_topline), MAXCOL);
+            int size = linetabsize_eol(wp, wp->w_topline);
             if (size > width1) {
               wp->w_skipcol = width1;
               size -= width1;
@@ -1449,7 +1459,7 @@ bool scrollup(win_T *wp, linenr_T line_count, bool byfold)
     const colnr_T prev_skipcol = wp->w_skipcol;
 
     if (do_sms) {
-      size = linetabsize(wp, wp->w_topline);
+      size = linetabsize_eol(wp, wp->w_topline);
     }
 
     // diff mode: first consume "topfill"
@@ -1492,7 +1502,7 @@ bool scrollup(win_T *wp, linenr_T line_count, bool byfold)
           wp->w_topfill = win_get_fill(wp, lnum);
           wp->w_skipcol = 0;
           if (todo > 1 && do_sms) {
-            size = linetabsize(wp, wp->w_topline);
+            size = linetabsize_eol(wp, wp->w_topline);
           }
         }
       }
@@ -1563,7 +1573,7 @@ void adjust_skipcol(void)
   }
 
   validate_virtcol(curwin);
-  int overlap = sms_marker_overlap(curwin, -1);
+  int overlap = sms_marker_overlap(curwin, curwin->w_width_inner - width2);
   while (curwin->w_skipcol > 0
          && curwin->w_virtcol < curwin->w_skipcol + overlap + scrolloff_cols) {
     // scroll a screen line down
@@ -1584,8 +1594,7 @@ void adjust_skipcol(void)
 
   // Avoid adjusting for 'scrolloff' beyond the text line height.
   if (scrolloff_cols > 0) {
-    int size = win_linetabsize(curwin, curwin->w_topline,
-                               ml_get(curwin->w_topline), (colnr_T)MAXCOL);
+    int size = linetabsize_eol(curwin, curwin->w_topline);
     size = width1 + width2 * ((size - width1 + width2 - 1) / width2);
     while (col > size) {
       col -= width2;

src/nvim/normal.c

@@ -5197,7 +5197,7 @@ void nv_g_home_m_cmd(cmdarg_T *cap)
     // When ending up below 'smoothscroll' marker, move just beyond it so
     // that skipcol is not adjusted later.
     if (curwin->w_skipcol > 0 && curwin->w_cursor.lnum == curwin->w_topline) {
-      int overlap = sms_marker_overlap(curwin, -1);
+      int overlap = sms_marker_overlap(curwin, curwin->w_width_inner - width2);
       if (overlap > 0 && i == curwin->w_skipcol) {
         i += overlap;
       }

src/nvim/path.c

@@ -619,7 +619,6 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
   FUNC_ATTR_NONNULL_ALL
 {
   int start_len = gap->ga_len;
-  size_t len;
   bool starstar = false;
   static int stardepth = 0;  // depth for "**" expansion
 
@@ -631,9 +630,9 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
     }
   }
 
-  // Make room for file name.  When doing encoding conversion the actual
-  // length may be quite a bit longer, thus use the maximum possible length.
-  char *buf = xmalloc(MAXPATHL);
+  // Make room for file name (a bit too much to stay on the safe side).
+  const size_t buflen = strlen(path) + MAXPATHL;
+  char *buf = xmalloc(buflen);
 
   // Find the first part in the path name that contains a wildcard.
   // When EW_ICASE is set every letter is considered to be a wildcard.
@@ -662,10 +661,10 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
                ) {
       e = p;
     }
-    len = (size_t)(utfc_ptr2len(path_end));
-    memcpy(p, path_end, len);
-    p += len;
-    path_end += len;
+    int charlen = utfc_ptr2len(path_end);
+    memcpy(p, path_end, (size_t)charlen);
+    p += charlen;
+    path_end += charlen;
   }
   e = p;
   *e = NUL;
@@ -719,13 +718,14 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
     return 0;
   }
 
+  size_t len = (size_t)(s - buf);
   // If "**" is by itself, this is the first time we encounter it and more
   // is following then find matches without any directory.
   if (!didstar && stardepth < 100 && starstar && e - s == 2
       && *path_end == '/') {
-    STRCPY(s, path_end + 1);
+    vim_snprintf(s, buflen - len, "%s", path_end + 1);
     stardepth++;
-    do_path_expand(gap, buf, (size_t)(s - buf), flags, true);
+    do_path_expand(gap, buf, len, flags, true);
     stardepth--;
   }
   *s = NUL;
@@ -737,6 +737,7 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
     const char *name;
     scandir_next_with_dots(NULL);  // initialize
     while (!got_int && (name = scandir_next_with_dots(&dir)) != NULL) {
+      len = (size_t)(s - buf);
       if ((name[0] != '.'
            || starts_with_dot
            || ((flags & EW_DODOT)
@@ -744,21 +745,22 @@ static size_t do_path_expand(garray_T *gap, const char *path, size_t wildoff, in
                && (name[1] != '.' || name[2] != NUL)))
           && ((regmatch.regprog != NULL && vim_regexec(&regmatch, name, 0))
               || ((flags & EW_NOTWILD)
-                  && path_fnamencmp(path + (s - buf), name, (size_t)(e - s)) == 0))) {
-        STRCPY(s, name);
-        len = strlen(buf);
+                  && path_fnamencmp(path + len, name, (size_t)(e - s)) == 0))) {
+        len += (size_t)vim_snprintf(s, buflen - len, "%s", name);
+        if (len + 1 >= buflen) {
+          continue;
+        }
 
         if (starstar && stardepth < 100) {
           // For "**" in the pattern first go deeper in the tree to
           // find matches.
-          STRCPY(buf + len, "/**");  // NOLINT
-          STRCPY(buf + len + 3, path_end);
+          vim_snprintf(buf + len, buflen - len, "/**%s", path_end);  // NOLINT
           stardepth++;
           do_path_expand(gap, buf, len + 1, flags, true);
           stardepth--;
         }
 
-        STRCPY(buf + len, path_end);
+        vim_snprintf(buf + len, buflen - len, "%s", path_end);
         if (path_has_exp_wildcard(path_end)) {      // handle more wildcards
           // need to expand another component of the path
           // remove backslashes for the remaining components only

src/nvim/plines.c

@@ -74,11 +74,19 @@ int linetabsize_col(int startvcol, char *s)
 
 /// Return the number of cells line "lnum" of window "wp" will take on the
 /// screen, taking into account the size of a tab and inline virtual text.
+/// Doesn't count the size of 'listchars' "eol".
 int linetabsize(win_T *wp, linenr_T lnum)
 {
   return win_linetabsize(wp, lnum, ml_get_buf(wp->w_buffer, lnum), MAXCOL);
 }
 
+/// Like linetabsize(), but counts the size of 'listchars' "eol".
+int linetabsize_eol(win_T *wp, linenr_T lnum)
+{
+  return linetabsize(wp, lnum)
+         + ((wp->w_p_list && wp->w_p_lcs_chars.eol != NUL) ? 1 : 0);
+}
+
 static const uint32_t inline_filter[4] = {[kMTMetaInline] = kMTFilterSelect };
 
 /// Prepare the structure passed to charsize functions.

src/nvim/plines.h

@@ -81,6 +81,7 @@ static inline int win_linetabsize(win_T *wp, linenr_T lnum, char *line, colnr_T
   REAL_FATTR_NONNULL_ALL REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_ALWAYS_INLINE;
 
 /// Like linetabsize_str(), but for a given window instead of the current one.
+/// Doesn't count the size of 'listchars' "eol".
 ///
 /// @param wp
 /// @param line

src/nvim/strings.c

@@ -1677,8 +1677,6 @@ int vim_vsnprintf_typval(char *str, size_t str_m, const char *fmt, va_list ap_st
       }
 
       switch (fmt_spec) {
-      case 'b':
-      case 'B':
       case 'd':
       case 'u':
       case 'o':
@@ -1802,6 +1800,13 @@ int vim_vsnprintf_typval(char *str, size_t str_m, const char *fmt, va_list ap_st
           if (ptr_arg) {
             arg_sign = 1;
           }
+        } else if (fmt_spec == 'b' || fmt_spec == 'B') {
+          uarg = (tvs
+                  ? (unsigned long long)tv_nr(tvs, &arg_idx)  // NOLINT(runtime/int)
+                  : (skip_to_arg(ap_types, ap_start, &ap, &arg_idx,
+                                 &arg_cur, fmt),
+                     va_arg(ap, unsigned long long)));  // NOLINT(runtime/int)
+          arg_sign = (uarg != 0);
         } else if (fmt_spec == 'd') {
           // signed
           switch (length_modifier) {

src/nvim/syntax.c

@@ -1649,7 +1649,7 @@ static int syn_current_attr(const bool syncing, const bool displaying, bool *con
                        ? !(spp->sp_flags & HL_CONTAINED)
                        : in_id_list(cur_si,
                                     cur_si->si_cont_list, &spp->sp_syn,
-                                    spp->sp_flags & HL_CONTAINED)))) {
+                                    spp->sp_flags)))) {
               // If we already tried matching in this line, and
               // there isn't a match before next_match_col, skip
               // this item.
@@ -2774,7 +2774,7 @@ static keyentry_T *match_keyword(char *keyword, hashtab_T *ht, stateitem_T *cur_
           : (cur_si == NULL
              ? !(kp->flags & HL_CONTAINED)
              : in_id_list(cur_si, cur_si->si_cont_list,
-                          &kp->k_syn, kp->flags & HL_CONTAINED))) {
+                          &kp->k_syn, kp->flags))) {
         return kp;
       }
     }
@@ -3927,7 +3927,7 @@ static void syn_incl_toplevel(int id, int *flagsp)
   if ((*flagsp & HL_CONTAINED) || curwin->w_s->b_syn_topgrp == 0) {
     return;
   }
-  *flagsp |= HL_CONTAINED;
+  *flagsp |= HL_CONTAINED | HL_INCLUDED_TOPLEVEL;
   if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER) {
     // We have to alloc this, because syn_combine_list() will free it.
     int16_t *grp_list = xmalloc(2 * sizeof(*grp_list));
@@ -4968,16 +4968,13 @@ static int get_id_list(char **const arg, const int keylen, int16_t **const list,
           break;
         }
         if (name[1] == 'A') {
-          id = SYNID_ALLBUT + current_syn_inc_tag;
+          id = SYNID_ALLBUT;
         } else if (name[1] == 'T') {
-          if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER) {
-            id = curwin->w_s->b_syn_topgrp;
+          id = SYNID_TOP;
         } else {
-            id = SYNID_TOP + current_syn_inc_tag;
-          }
-        } else {
-          id = SYNID_CONTAINED + current_syn_inc_tag;
+          id = SYNID_CONTAINED;
         }
+        id += current_syn_inc_tag;
       } else if (name[1] == '@') {
         if (skip) {
           id = -1;
@@ -5095,8 +5092,8 @@ static int16_t *copy_id_list(const int16_t *const list)
 /// @param cur_si     current item or NULL
 /// @param list       id list
 /// @param ssp        group id and ":syn include" tag of group
-/// @param contained  group id is contained
-static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, int contained)
+/// @param flags      group flags
+static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, int flags)
 {
   int retval;
   int16_t id = ssp->id;
@@ -5114,8 +5111,7 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
     // cur_si->si_idx is -1 for keywords, these never contain anything.
     if (cur_si->si_idx >= 0 && in_id_list(NULL, ssp->cont_in_list,
                                           &(SYN_ITEMS(syn_block)[cur_si->si_idx].sp_syn),
-                                          SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags &
-                                          HL_CONTAINED)) {
+                                          SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags)) {
       return true;
     }
   }
@@ -5127,9 +5123,14 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
   // If list is ID_LIST_ALL, we are in a transparent item that isn't
   // inside anything.  Only allow not-contained groups.
   if (list == ID_LIST_ALL) {
-    return !contained;
+    return !(flags & HL_CONTAINED);
   }
 
+  // Is this top-level (i.e. not 'contained') in the file it was declared in?
+  // For included files, this is different from HL_CONTAINED, which is set
+  // unconditionally.
+  bool toplevel = !(flags & HL_CONTAINED) || (flags & HL_INCLUDED_TOPLEVEL);
+
   // If the first item is "ALLBUT", return true if "id" is NOT in the
   // contains list.  We also require that "id" is at the same ":syn include"
   // level as the list.
@@ -5142,12 +5143,12 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
       }
     } else if (item < SYNID_CONTAINED) {
       // TOP: accept all not-contained groups in the same file
-      if (item - SYNID_TOP != ssp->inc_tag || contained) {
+      if (item - SYNID_TOP != ssp->inc_tag || !toplevel) {
         return false;
       }
     } else {
       // CONTAINED: accept all contained groups in the same file
-      if (item - SYNID_CONTAINED != ssp->inc_tag || !contained) {
+      if (item - SYNID_CONTAINED != ssp->inc_tag || toplevel) {
         return false;
       }
     }
@@ -5168,7 +5169,7 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
       // cluster that includes itself (indirectly)
       if (scl_list != NULL && depth < 30) {
         depth++;
-        int r = in_id_list(NULL, scl_list, ssp, contained);
+        int r = in_id_list(NULL, scl_list, ssp, flags);
         depth--;
         if (r) {
           return retval;

src/nvim/syntax.h

@@ -26,6 +26,7 @@ enum {
   HL_TRANS_CONT        = 0x10000,  ///< transparent item without contains arg
   HL_CONCEAL           = 0x20000,  ///< can be concealed
   HL_CONCEALENDS       = 0x40000,  ///< can be concealed
+  HL_INCLUDED_TOPLEVEL = 0x80000,  ///< toplevel item in included syntax, allowed by contains=TOP
 };
 
 #define SYN_GROUP_STATIC(s) syn_check_group(S_LEN(s))

src/nvim/tag.c

@@ -3014,6 +3014,8 @@ static int jumpto_tag(const char *lbuf_arg, int forceit, bool keep_help)
       secure = 1;
       sandbox++;
       curwin->w_cursor.lnum = 1;  // start command in line 1
+      curwin->w_cursor.col = 0;
+      curwin->w_cursor.coladd = 0;
       do_cmdline_cmd(pbuf);
       retval = OK;
 

test/functional/legacy/messages_spec.lua

@@ -89,6 +89,34 @@ describe('messages', function()
     ]])
   end)
 
+  -- oldtest: Test_mode_cleared_after_silent_message()
+  it('mode is cleared properly after slient message', function()
+    screen = Screen.new(60, 10)
+    screen:attach()
+    exec([[
+      edit XsilentMessageMode.txt
+      call setline(1, 'foobar')
+      autocmd TextChanged * silent update
+    ]])
+    finally(function()
+      os.remove('XsilentMessageMode.txt')
+    end)
+
+    feed('v')
+    screen:expect([[
+      ^foobar                                                      |
+      {1:~                                                           }|*8
+      {5:-- VISUAL --}                                                |
+    ]])
+
+    feed('d')
+    screen:expect([[
+      ^oobar                                                       |
+      {1:~                                                           }|*8
+                                                                  |
+    ]])
+  end)
+
   describe('more prompt', function()
     before_each(function()
       command('set more')

test/functional/lua/buffer_updates_spec.lua

@@ -300,6 +300,24 @@ describe('lua buffer event callbacks: on_lines', function()
     n.assert_alive()
   end)
 
+  it('no invalid lnum error for closed memline in on_detach #31251', function()
+    eq(vim.NIL, exec_lua('return _G.did_detach'))
+    exec_lua([[
+      vim.api.nvim_buf_set_lines(0, 0, -1, false, { '' })
+      local bufname = 'buf2'
+      local buf = vim.api.nvim_create_buf(false, true)
+      vim.api.nvim_buf_set_name(buf, bufname)
+      vim.bo[buf].bufhidden = 'wipe'
+      vim.cmd('vertical diffsplit '..bufname)
+      vim.api.nvim_buf_attach(0, false, { on_detach = function()
+        vim.cmd("redraw")
+        _G.did_detach = true
+      end})
+      vim.cmd.bdelete()
+    ]])
+    eq(true, exec_lua('return _G.did_detach'))
+  end)
+
   it('#12718 lnume', function()
     api.nvim_buf_set_lines(0, 0, -1, true, { '1', '2', '3' })
     exec_lua([[

test/functional/plugin/lsp/incremental_sync_spec.lua

@@ -170,7 +170,7 @@ describe('incremental synchronization', function()
       }
       test_edit({ 'a' }, { 'rb' }, expected_text_changes, 'utf-16', '\n')
     end)
-    it('deleting a line', function()
+    it('deleting the first line', function()
       local expected_text_changes = {
         {
           range = {
@@ -183,11 +183,49 @@ describe('incremental synchronization', function()
               line = 1,
             },
           },
-          rangeLength = 12,
+          rangeLength = 6,
           text = '',
         },
       }
-      test_edit({ 'hello world' }, { 'dd' }, expected_text_changes, 'utf-16', '\n')
+      test_edit({ 'hello', 'world' }, { 'ggdd' }, expected_text_changes, 'utf-16', '\n')
+    end)
+    it('deleting the last line', function()
+      local expected_text_changes = {
+        {
+          range = {
+            ['start'] = {
+              character = 0,
+              line = 1,
+            },
+            ['end'] = {
+              character = 0,
+              line = 2,
+            },
+          },
+          rangeLength = 6,
+          text = '',
+        },
+      }
+      test_edit({ 'hello', 'world' }, { '2ggdd' }, expected_text_changes, 'utf-16', '\n')
+    end)
+    it('deleting all lines', function()
+      local expected_text_changes = {
+        {
+          range = {
+            ['start'] = {
+              character = 0,
+              line = 0,
+            },
+            ['end'] = {
+              character = 5,
+              line = 1,
+            },
+          },
+          rangeLength = 11,
+          text = '',
+        },
+      }
+      test_edit({ 'hello', 'world' }, { 'ggdG' }, expected_text_changes, 'utf-16', '\n')
     end)
     it('deleting an empty line', function()
       local expected_text_changes = {

test/old/testdir/runtest.vim

@@ -13,6 +13,9 @@
 " For csh:
 "     setenv TEST_FILTER Test_channel
 "
+" If the environment variable $TEST_SKIP_PAT is set then test functions
+" matching this pattern will be skipped.  It's the opposite of $TEST_FILTER.
+"
 " While working on a test you can make $TEST_NO_RETRY non-empty to not retry:
 "     export TEST_NO_RETRY=yes
 "
@@ -92,7 +95,12 @@ set encoding=utf-8
 " REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for
 " the test_name.vim file itself. Replace it here with a more restrictive one,
 " so we still catch mistakes.
-let s:test_script_fname = expand('%')
+if has("win32")
+  " replace any '/' directory separators by '\\'
+  let s:test_script_fname = substitute(expand('%'), '/', '\\', 'g')
+else
+  let s:test_script_fname = expand('%')
+endif
 au! SwapExists * call HandleSwapExists()
 func HandleSwapExists()
   if exists('g:ignoreSwapExists')
@@ -431,13 +439,17 @@ func FinishTesting()
 
   if s:done == 0
     if s:filtered > 0
+      if $TEST_FILTER != ''
         let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'"
+      else
+        let message = "ALL tests match $TEST_SKIP_PAT: '" .. $TEST_SKIP_PAT .. "'"
+      endif
     else
       let message = 'NO tests executed'
     endif
   else
     if s:filtered > 0
-      call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER")
+      call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER and $TEST_SKIP_PAT")
     endif
     let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test')
   endif
@@ -530,6 +542,12 @@ endif
 
 " Execute the tests in alphabetical order.
 for g:testfunc in sort(s:tests)
+  if $TEST_SKIP_PAT != '' && g:testfunc =~ $TEST_SKIP_PAT
+    call add(s:messages, g:testfunc .. ' matches $TEST_SKIP_PAT')
+    let s:filtered += 1
+    continue
+  endif
+
   " Silence, please!
   set belloff=all
   let prev_error = ''

test/old/testdir/test_expand_func.vim

@@ -143,4 +143,11 @@ func Test_expand_wildignore()
   set wildignore&
 endfunc
 
+" Passing a long string to expand with 'wildignorecase' should not crash Vim.
+func Test_expand_long_str()
+  set wildignorecase
+  call expand('a'->repeat(99999))
+  set wildignorecase&
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

test/old/testdir/test_messages.vim

@@ -386,6 +386,29 @@ func Test_message_not_cleared_after_mode()
   call StopVimInTerminal(buf)
 endfunc
 
+func Test_mode_cleared_after_silent_message()
+  CheckRunVimInTerminal
+
+  let lines =<< trim END
+    edit XsilentMessageMode.txt
+    call setline(1, 'foobar')
+    autocmd TextChanged * silent update
+  END
+  call writefile(lines, 'XsilentMessageMode', 'D')
+  let buf = RunVimInTerminal('-S XsilentMessageMode', {'rows': 10})
+
+  call term_sendkeys(buf, 'v')
+  call TermWait(buf)
+  call VerifyScreenDump(buf, 'Test_mode_cleared_after_silent_message_1', {})
+
+  call term_sendkeys(buf, 'd')
+  call TermWait(buf)
+  call VerifyScreenDump(buf, 'Test_mode_cleared_after_silent_message_2', {})
+
+  call StopVimInTerminal(buf)
+  call delete('XsilentMessageMode.txt')
+endfunc
+
 " Test verbose message before echo command
 func Test_echo_verbose_system()
   CheckRunVimInTerminal

test/old/testdir/test_scroll_opt.vim

@@ -1219,4 +1219,59 @@ func Test_smooth_long_scrolloff()
   call StopVimInTerminal(buf)
 endfunc
 
+func Test_smoothscroll_listchars_eol()
+  call NewWindow(10, 40)
+  setlocal list listchars=eol:$ scrolloff=0 smoothscroll
+  call setline(1, repeat('-', 40))
+  call append(1, repeat(['foobar'], 10))
+
+  normal! G
+  call assert_equal(2, line('w0'))
+  call assert_equal(0, winsaveview().skipcol)
+
+  exe "normal! \<C-Y>"
+  call assert_equal(1, line('w0'))
+  call assert_equal(40, winsaveview().skipcol)
+
+  exe "normal! \<C-Y>"
+  call assert_equal(1, line('w0'))
+  call assert_equal(0, winsaveview().skipcol)
+
+  exe "normal! \<C-Y>"
+  call assert_equal(1, line('w0'))
+  call assert_equal(0, winsaveview().skipcol)
+
+  exe "normal! \<C-E>"
+  call assert_equal(1, line('w0'))
+  call assert_equal(40, winsaveview().skipcol)
+
+  exe "normal! \<C-E>"
+  call assert_equal(2, line('w0'))
+  call assert_equal(0, winsaveview().skipcol)
+
+  for ve in ['', 'all', 'onemore']
+    let &virtualedit = ve
+    normal! gg
+    call assert_equal(1, line('w0'))
+    call assert_equal(0, winsaveview().skipcol)
+
+    exe "normal! \<C-E>"
+    redraw  " redrawing should not cause another scroll
+    call assert_equal(1, line('w0'))
+    call assert_equal(40, winsaveview().skipcol)
+
+    exe "normal! \<C-E>"
+    redraw
+    call assert_equal(2, line('w0'))
+    call assert_equal(0, winsaveview().skipcol)
+
+    if ve != 'all'
+      call assert_equal([0, 2, 1, 0], getpos('.'))
+    endif
+  endfor
+
+  set virtualedit&
+  bwipe!
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

test/old/testdir/test_syntax.vim

@@ -949,7 +949,7 @@ func Test_syn_contained_transparent()
 endfunc
 
 func Test_syn_include_contains_TOP()
-  let l:case = "TOP in included syntax means its group list name"
+  let l:case = "TOP in included syntax refers to top level of that included syntax"
   new
   syntax include @INCLUDED syntax/c.vim
   syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
@@ -964,6 +964,18 @@ func Test_syn_include_contains_TOP()
   bw!
 endfunc
 
+func Test_syn_include_contains_TOP_excluding()
+  new
+  syntax include @INCLUDED syntax/c.vim
+  syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
+
+  call setline(1,  ['```c', '#if 0', 'int', '#else', 'int', '#if', '#endif', '```' ])
+  let l:expected = ["cCppOutElse", "cConditional"]
+  eval AssertHighlightGroups(6, 1, l:expected, 1)
+  syntax clear
+  bw!
+endfunc
+
 " This was using freed memory
 func Test_WinEnter_synstack_synID()
   autocmd WinEnter * call synstack(line("."), col("."))

test/old/testdir/test_tagjump.vim

@@ -1623,4 +1623,39 @@ func Test_tagbsearch()
   set tags& tagbsearch&
 endfunc
 
+" Test tag guessing with very short names
+func Test_tag_guess_short()
+  call writefile(["!_TAG_FILE_ENCODING\tutf-8\t//",
+        \ "y\tXf\t/^y()/"],
+        \ 'Xt', 'D')
+  set tags=Xt cpoptions+=t
+  call writefile(['', 'int * y () {}', ''], 'Xf', 'D')
+
+  let v:statusmsg = ''
+  let @/ = ''
+  ta y
+  call assert_match('E435:', v:statusmsg)
+  call assert_equal(2, line('.'))
+  call assert_match('<y', @/)
+
+  set tags& cpoptions-=t
+endfunc
+
+func Test_tag_excmd_with_nostartofline()
+  call writefile(["!_TAG_FILE_ENCODING\tutf-8\t//",
+        \ "f\tXfile\tascii"],
+        \ 'Xtags', 'D')
+  call writefile(['f', 'foobar'], 'Xfile', 'D')
+
+  set nostartofline
+  new Xfile
+  setlocal tags=Xtags
+  normal! G$
+  " This used to cause heap-buffer-overflow
+  tag f
+
+  bwipe!
+  set startofline&
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

test/unit/strings_spec.lua

@@ -1,6 +1,7 @@
 local t = require('test.unit.testutil')
 local itp = t.gen_itp(it)
 
+local child_call_once = t.child_call_once
 local cimport = t.cimport
 local eq = t.eq
 local ffi = t.ffi
@@ -8,6 +9,12 @@ local to_cstr = t.to_cstr
 
 local strings = cimport('stdlib.h', './src/nvim/strings.h', './src/nvim/memory.h')
 
+local UVARNUM_TYPE
+
+child_call_once(function()
+  UVARNUM_TYPE = ffi.typeof('uvarnumber_T')
+end)
+
 describe('vim_strsave_escaped()', function()
   local vim_strsave_escaped = function(s, chars)
     local res = strings.vim_strsave_escaped(to_cstr(s), to_cstr(chars))
@@ -140,13 +147,22 @@ end)
 
 describe('vim_snprintf()', function()
   local function a(expected, buf, bsize, fmt, ...)
-    eq(#expected, strings.vim_snprintf(buf, bsize, fmt, ...))
+    local args = { ... }
+    local ctx = string.format('snprintf(buf, %d, "%s"', bsize, fmt)
+    for _, x in ipairs(args) do
+      ctx = ctx .. ', ' .. tostring(x)
+    end
+    ctx = ctx .. string.format(') = %s', expected)
+    eq(#expected, strings.vim_snprintf(buf, bsize, fmt, ...), ctx)
     if bsize > 0 then
       local actual = ffi.string(buf, math.min(#expected + 1, bsize))
-      eq(expected:sub(1, bsize - 1) .. '\0', actual)
+      eq(expected:sub(1, bsize - 1) .. '\0', actual, ctx)
     end
   end
 
+  local function uv(n)
+    return ffi.cast(UVARNUM_TYPE, n)
+  end
   local function i(n)
     return ffi.cast('int', n)
   end
@@ -181,7 +197,7 @@ describe('vim_snprintf()', function()
       a('  1234567', buf, bsize, '%9ld', l(1234567))
       a('1234567  ', buf, bsize, '%-9ld', l(1234567))
       a('deadbeef', buf, bsize, '%x', u(0xdeadbeef))
-      a('001100', buf, bsize, '%06b', u(12))
+      a('001100', buf, bsize, '%06b', uv(12))
       a('one two', buf, bsize, '%s %s', 'one', 'two')
       a('1.234000', buf, bsize, '%f', 1.234)
       a('1.234000e+00', buf, bsize, '%e', 1.234)
@@ -223,10 +239,10 @@ describe('vim_snprintf()', function()
       a('three one two', buf, bsize, '%3$s %1$s %2$s', 'one', 'two', 'three')
       a('1234567', buf, bsize, '%1$d', i(1234567))
       a('deadbeef', buf, bsize, '%1$x', u(0xdeadbeef))
-      a('001100', buf, bsize, '%2$0*1$b', i(6), u(12))
-      a('001100', buf, bsize, '%1$0.*2$b', u(12), i(6))
+      a('001100', buf, bsize, '%2$0*1$b', i(6), uv(12))
+      a('001100', buf, bsize, '%1$0.*2$b', uv(12), i(6))
       a('one two', buf, bsize, '%1$s %2$s', 'one', 'two')
-      a('001100', buf, bsize, '%06b', u(12))
+      a('001100', buf, bsize, '%06b', uv(12))
       a('two one', buf, bsize, '%2$s %1$s', 'one', 'two')
       a('1.234000', buf, bsize, '%1$f', 1.234)
       a('1.234000e+00', buf, bsize, '%1$e', 1.234)

test/unit/testutil.lua

@@ -146,11 +146,16 @@ local function filter_complex_blocks(body)
         or string.find(line, 'value_init_')
         or string.find(line, 'UUID_NULL') -- static const uuid_t UUID_NULL = {...}
         or string.find(line, 'inline _Bool')
+        -- used by musl libc headers on 32-bit arches via __REDIR marco
+        or string.find(line, '__typeof__')
         -- used by macOS headers
         or string.find(line, 'typedef enum : ')
         or string.find(line, 'mach_vm_range_recipe')
       )
     then
+      -- Remove GCC's extension keyword which is just used to disable warnings.
+      line = string.gsub(line, '__extension__', '')
+
       result[#result + 1] = line
     end
   end