vim-patch:8.2.1969: Vim9: map() may change the list or dict item type

Problem:    Vim9: map() may change the list or dict item type.
Solution:   Add mapnew().

ea696852e7

Co-authored-by: Bram Moolenaar <Bram@vim.org>

runtime/doc/builtin.txt

@@ -4065,6 +4065,8 @@ map({expr1}, {expr2})                                                    *map()*
 		{expr1} must be a |List|, |Blob| or |Dictionary|.
 		Replace each item in {expr1} with the result of evaluating
 		{expr2}.  For a |Blob| each byte is replaced.
+		If the item type changes you may want to use |mapnew()| to
+		create a new List or Dictionary.
 
 		{expr2} must be a |string| or |Funcref|.
 
@@ -4205,6 +4207,11 @@ mapcheck({name} [, {mode} [, {abbr}]])                              *mapcheck()*
 <		This avoids adding the "_vv" mapping when there already is a
 		mapping for "_v" or for "_vvv".
 
+mapnew({expr1}, {expr2})                                              *mapnew()*
+		Like |map()| but instead of replacing items in {expr1} a new
+		List or Dictionary is created and returned.  {expr1} remains
+		unchanged.
+
 mapset({mode}, {abbr}, {dict})                                        *mapset()*
 		Restore a mapping from a dictionary returned by |maparg()|.
 		{mode} and {abbr} should be the same as for the call to

runtime/doc/usr_41.txt

@@ -660,6 +660,7 @@ List manipulation:					*list-functions*
 	deepcopy()		make a full copy of a List
 	filter()		remove selected items from a List
 	map()			change each List item
+	mapnew()		make a new List with changed items
 	reduce()		reduce a List to a value
 	slice()			take a slice of a List
 	sort()			sort a List
@@ -690,6 +691,7 @@ Dictionary manipulation:				*dict-functions*
 	extendnew()		make a new Dictionary and append items
 	filter()		remove selected entries from a Dictionary
 	map()			change each Dictionary entry
+	mapnew()		make a new Dictionary with changed items
 	keys()			get List of Dictionary keys
 	values()		get List of Dictionary values
 	items()			get List of Dictionary key-value pairs

runtime/lua/vim/_meta/vimfn.lua

@@ -4922,6 +4922,8 @@ function vim.fn.log10(expr) end
 --- {expr1} must be a |List|, |Blob| or |Dictionary|.
 --- Replace each item in {expr1} with the result of evaluating
 --- {expr2}.  For a |Blob| each byte is replaced.
+--- If the item type changes you may want to use |mapnew()| to
+--- create a new List or Dictionary.
 ---
 --- {expr2} must be a |string| or |Funcref|.
 ---
@@ -5078,6 +5080,15 @@ function vim.fn.maparg(name, mode, abbr, dict) end
 --- @return any
 function vim.fn.mapcheck(name, mode, abbr) end
 
+--- Like |map()| but instead of replacing items in {expr1} a new
+--- List or Dictionary is created and returned.  {expr1} remains
+--- unchanged.
+---
+--- @param expr1 any
+--- @param expr2 any
+--- @return any
+function vim.fn.mapnew(expr1, expr2) end
+
 --- Restore a mapping from a dictionary returned by |maparg()|.
 --- {mode} and {abbr} should be the same as for the call to
 --- |maparg()|. *E460*

src/nvim/eval.c

@@ -295,6 +295,13 @@ static partial_T *vvlua_partial;
 /// v: hashtab
 #define vimvarht  vimvardict.dv_hashtab
 
+/// Enum used by filter(), map() and mapnew()
+typedef enum {
+  FILTERMAP_FILTER,
+  FILTERMAP_MAP,
+  FILTERMAP_MAPNEW,
+} filtermap_T;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "eval.c.generated.h"
 #endif
@@ -1838,7 +1845,7 @@ void *eval_for_line(const char *arg, bool *errp, exarg_T *eap, evalarg_T *const
 
           // Make a copy, so that the iteration still works when the
           // blob is changed.
-          tv_blob_copy(&tv, &btv);
+          tv_blob_copy(tv.vval.v_blob, &btv);
           fi->fi_blob = btv.vval.v_blob;
         }
         tv_clear(&tv);
@@ -5018,33 +5025,51 @@ void assert_error(garray_T *gap)
 }
 
 /// Implementation of map() and filter().
-void filter_map(typval_T *argvars, typval_T *rettv, int map)
+static void filter_map(typval_T *argvars, typval_T *rettv, filtermap_T filtermap)
 {
   list_T *l = NULL;
   dict_T *d = NULL;
   blob_T *b = NULL;
   int rem = false;
-  char *ermsg = map ? "map()" : "filter()";
+  const char *const ermsg = (filtermap == FILTERMAP_MAP ? "map()"
-  const char *const arg_errmsg = (map
+                             : filtermap == FILTERMAP_MAPNEW ? "mapnew()" : "filter()");
+  const char *const arg_errmsg = (filtermap == FILTERMAP_MAP
                                   ? N_("map() argument")
-                                  : N_("filter() argument"));
+                                  : (filtermap == FILTERMAP_MAPNEW
+                                     ? N_("mapnew() argument")
+                                     : N_("filter() argument")));
 
-  // Always return the first argument, also on failure.
+  // map() and filter() return the first argument, also on failure.
+  if (filtermap != FILTERMAP_MAPNEW) {
     tv_copy(&argvars[0], rettv);
+  }
 
   if (argvars[0].v_type == VAR_BLOB) {
+    if (filtermap == FILTERMAP_MAPNEW) {
+      rettv->v_type = VAR_BLOB;
+      rettv->vval.v_blob = NULL;
+    }
     if ((b = argvars[0].vval.v_blob) == NULL) {
       return;
     }
   } else if (argvars[0].v_type == VAR_LIST) {
+    if (filtermap == FILTERMAP_MAPNEW) {
+      rettv->v_type = VAR_LIST;
+      rettv->vval.v_list = NULL;
+    }
     if ((l = argvars[0].vval.v_list) == NULL
-        || (!map
+        || (filtermap == FILTERMAP_FILTER
             && value_check_lock(tv_list_locked(l), arg_errmsg, TV_TRANSLATE))) {
       return;
     }
   } else if (argvars[0].v_type == VAR_DICT) {
+    if (filtermap == FILTERMAP_MAPNEW) {
+      rettv->v_type = VAR_DICT;
+      rettv->vval.v_dict = NULL;
+    }
     if ((d = argvars[0].vval.v_dict) == NULL
-        || (!map && value_check_lock(d->dv_lock, arg_errmsg, TV_TRANSLATE))) {
+        || (filtermap == FILTERMAP_FILTER
+            && value_check_lock(d->dv_lock, arg_errmsg, TV_TRANSLATE))) {
       return;
     }
   } else {
@@ -5069,10 +5094,17 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map)
     typval_T save_key;
     prepare_vimvar(VV_KEY, &save_key);
     if (argvars[0].v_type == VAR_DICT) {
+      const VarLockStatus prev_lock = d->dv_lock;
+      dict_T *d_ret = NULL;
+
+      if (filtermap == FILTERMAP_MAPNEW) {
+        tv_dict_alloc_ret(rettv);
+        d_ret = rettv->vval.v_dict;
+      }
+
       vimvars[VV_KEY].vv_type = VAR_STRING;
 
-      const VarLockStatus prev_lock = d->dv_lock;
+      if (filtermap != FILTERMAP_FILTER && d->dv_lock == VAR_UNLOCKED) {
-      if (map && d->dv_lock == VAR_UNLOCKED) {
         d->dv_lock = VAR_LOCKED;
       }
       hashtab_T *ht = &d->dv_hashtab;
@@ -5083,19 +5115,34 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map)
           todo--;
 
           dictitem_T *di = TV_DICT_HI2DI(hi);
-          if (map
+          if (filtermap != FILTERMAP_FILTER
               && (value_check_lock(di->di_tv.v_lock, arg_errmsg, TV_TRANSLATE)
                   || var_check_ro(di->di_flags, arg_errmsg, TV_TRANSLATE))) {
             break;
           }
 
           vimvars[VV_KEY].vv_str = xstrdup(di->di_key);
-          int r = filter_map_one(&di->di_tv, expr, map, &rem);
+          typval_T newtv;
+          int r = filter_map_one(&di->di_tv, expr, filtermap, &newtv, &rem);
           tv_clear(&vimvars[VV_KEY].vv_tv);
           if (r == FAIL || did_emsg) {
+            tv_clear(&newtv);
             break;
           }
-          if (!map && rem) {
+          if (filtermap == FILTERMAP_MAP) {
+            // map(): replace the dict item value
+            tv_clear(&di->di_tv);
+            newtv.v_lock = VAR_UNLOCKED;
+            di->di_tv = newtv;
+          } else if (filtermap == FILTERMAP_MAPNEW) {
+            // mapnew(): add the item value to the new dict
+            r = tv_dict_add_tv(d_ret, di->di_key, strlen(di->di_key), &newtv);
+            tv_clear(&newtv);
+            if (r == FAIL) {
+              break;
+            }
+          } else if (filtermap == FILTERMAP_FILTER && rem) {
+            // filter(false): remove the item from the dict
             if (var_check_fixed(di->di_flags, arg_errmsg, TV_TRANSLATE)
                 || var_check_ro(di->di_flags, arg_errmsg, TV_TRANSLATE)) {
               break;
@@ -5107,24 +5154,36 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map)
       hash_unlock(ht);
       d->dv_lock = prev_lock;
     } else if (argvars[0].v_type == VAR_BLOB) {
+      blob_T *b_ret = b;
+
+      if (filtermap == FILTERMAP_MAPNEW) {
+        tv_blob_copy(b, rettv);
+        b_ret = rettv->vval.v_blob;
+      }
+
       vimvars[VV_KEY].vv_type = VAR_NUMBER;
 
       for (int i = 0; i < b->bv_ga.ga_len; i++) {
-        typval_T tv;
-        tv.v_type = VAR_NUMBER;
         const varnumber_T val = tv_blob_get(b, i);
-        tv.vval.v_number = val;
+        typval_T tv = {
+          .v_type = VAR_NUMBER,
+          .v_lock = VAR_UNLOCKED,
+          .vval.v_number = val,
+        };
         vimvars[VV_KEY].vv_nr = idx;
-        if (filter_map_one(&tv, expr, map, &rem) == FAIL || did_emsg) {
+        typval_T newtv;
+        if (filter_map_one(&tv, expr, filtermap, &newtv, &rem) == FAIL
+            || did_emsg) {
           break;
         }
-        if (tv.v_type != VAR_NUMBER && tv.v_type != VAR_BOOL) {
+        if (newtv.v_type != VAR_NUMBER && newtv.v_type != VAR_BOOL) {
+          tv_clear(&newtv);
           emsg(_(e_invalblob));
-          return;
+          break;
         }
-        if (map) {
+        if (filtermap != FILTERMAP_FILTER) {
-          if (tv.vval.v_number != val) {
+          if (newtv.vval.v_number != val) {
-            tv_blob_set(b, i, (uint8_t)tv.vval.v_number);
+            tv_blob_set(b_ret, i, (uint8_t)newtv.vval.v_number);
           }
         } else if (rem) {
           char *const p = argvars[0].vval.v_blob->bv_ga.ga_data;
@@ -5136,24 +5195,42 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map)
       }
     } else {
       assert(argvars[0].v_type == VAR_LIST);
-      vimvars[VV_KEY].vv_type = VAR_NUMBER;
 
       const VarLockStatus prev_lock = tv_list_locked(l);
-      if (map && tv_list_locked(l) == VAR_UNLOCKED) {
+      list_T *l_ret = NULL;
+
+      if (filtermap == FILTERMAP_MAPNEW) {
+        tv_list_alloc_ret(rettv, kListLenUnknown);
+        l_ret = rettv->vval.v_list;
+      }
+
+      vimvars[VV_KEY].vv_type = VAR_NUMBER;
+
+      if (filtermap != FILTERMAP_FILTER && tv_list_locked(l) == VAR_UNLOCKED) {
         tv_list_set_lock(l, VAR_LOCKED);
       }
       for (listitem_T *li = tv_list_first(l); li != NULL;) {
-        if (map
+        if (filtermap != FILTERMAP_FILTER
             && value_check_lock(TV_LIST_ITEM_TV(li)->v_lock, arg_errmsg,
                                 TV_TRANSLATE)) {
           break;
         }
         vimvars[VV_KEY].vv_nr = idx;
-        if (filter_map_one(TV_LIST_ITEM_TV(li), expr, map, &rem) == FAIL
+        typval_T newtv;
+        if (filter_map_one(TV_LIST_ITEM_TV(li), expr, filtermap, &newtv, &rem) == FAIL
             || did_emsg) {
           break;
         }
-        if (!map && rem) {
+        if (filtermap == FILTERMAP_MAP) {
+          // map(): replace the list item value
+          tv_clear(TV_LIST_ITEM_TV(li));
+          newtv.v_lock = VAR_UNLOCKED;
+          *TV_LIST_ITEM_TV(li) = newtv;
+        } else if (filtermap == FILTERMAP_MAPNEW) {
+          // mapnew(): append the list item value
+          tv_list_append_owned_tv(l_ret, newtv);
+        }
+        if (filtermap == FILTERMAP_FILTER && rem) {
           li = tv_list_item_remove(l, li);
         } else {
           li = TV_LIST_ITEM_NEXT(l, li);
@@ -5170,30 +5247,32 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map)
   }
 }
 
-static int filter_map_one(typval_T *tv, typval_T *expr, int map, int *remp)
+/// Handle one item for map() and filter().
-  FUNC_ATTR_NONNULL_ARG(1, 2)
+/// Sets v:val to "tv".  Caller must set v:key.
+///
+/// @param tv     original value
+/// @param expr   callback
+/// @param newtv  for map() an mapnew(): new value
+/// @param remp   for filter(): remove flag
+static int filter_map_one(typval_T *tv, typval_T *expr, const filtermap_T filtermap,
+                          typval_T *newtv, int *remp)
+  FUNC_ATTR_NONNULL_ALL
 {
-  typval_T rettv;
   typval_T argv[3];
   int retval = FAIL;
 
   tv_copy(tv, &vimvars[VV_VAL].vv_tv);
   argv[0] = vimvars[VV_KEY].vv_tv;
   argv[1] = vimvars[VV_VAL].vv_tv;
-  if (eval_expr_typval(expr, argv, 2, &rettv) == FAIL) {
+  if (eval_expr_typval(expr, argv, 2, newtv) == FAIL) {
     goto theend;
   }
-  if (map) {
+  if (filtermap == FILTERMAP_FILTER) {
-    // map(): replace the list item value.
-    tv_clear(tv);
-    rettv.v_lock = VAR_UNLOCKED;
-    *tv = rettv;
-  } else {
     bool error = false;
 
     // filter(): when expr is zero remove the item
-    *remp = (tv_get_number_chk(&rettv, &error) == 0);
+    *remp = (tv_get_number_chk(newtv, &error) == 0);
-    tv_clear(&rettv);
+    tv_clear(newtv);
     // On type error, nothing has been removed; return FAIL to stop the
     // loop.  The error message was given by tv_get_number_chk().
     if (error) {
@@ -5206,6 +5285,24 @@ theend:
   return retval;
 }
 
+/// "filter()" function
+void f_filter(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
+{
+  filter_map(argvars, rettv, FILTERMAP_FILTER);
+}
+
+/// "map()" function
+void f_map(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
+{
+  filter_map(argvars, rettv, FILTERMAP_MAP);
+}
+
+/// "mapnew()" function
+void f_mapnew(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
+{
+  filter_map(argvars, rettv, FILTERMAP_MAPNEW);
+}
+
 /// "function()" function
 /// "funcref()" function
 void common_function(typval_T *argvars, typval_T *rettv, bool is_funcref)
@@ -7682,7 +7779,7 @@ int var_item_copy(const vimconv_T *const conv, typval_T *const from, typval_T *c
     }
     break;
   case VAR_BLOB:
-    tv_blob_copy(from, to);
+    tv_blob_copy(from->vval.v_blob, to);
     break;
   case VAR_DICT:
     to->v_type = VAR_DICT;

src/nvim/eval.lua

@@ -6053,6 +6053,8 @@ M.funcs = {
       {expr1} must be a |List|, |Blob| or |Dictionary|.
       Replace each item in {expr1} with the result of evaluating
       {expr2}.  For a |Blob| each byte is replaced.
+      If the item type changes you may want to use |mapnew()| to
+      create a new List or Dictionary.
 
       {expr2} must be a |string| or |Funcref|.
 
@@ -6215,6 +6217,18 @@ M.funcs = {
     params = { { 'name', 'string' }, { 'mode', 'string' }, { 'abbr', 'any' } },
     signature = 'mapcheck({name} [, {mode} [, {abbr}]])',
   },
+  mapnew = {
+    args = 2,
+    base = 1,
+    desc = [=[
+      Like |map()| but instead of replacing items in {expr1} a new
+      List or Dictionary is created and returned.  {expr1} remains
+      unchanged.
+    ]=],
+    name = 'mapnew',
+    params = { { 'expr1', 'any' }, { 'expr2', 'any' } },
+    signature = 'mapnew({expr1}, {expr2})',
+  },
   mapset = {
     args = 3,
     base = 1,

src/nvim/eval/funcs.c

@@ -2097,12 +2097,6 @@ static void findfilendir(typval_T *argvars, typval_T *rettv, int find_what)
   }
 }
 
-/// "filter()" function
-static void f_filter(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
-{
-  filter_map(argvars, rettv, false);
-}
-
 /// "finddir({fname}[, {path}[, {count}]])" function
 static void f_finddir(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
 {
@@ -4488,12 +4482,6 @@ static void f_luaeval(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
   nlua_typval_eval(cstr_as_string((char *)str), &argvars[1], rettv);
 }
 
-/// "map()" function
-static void f_map(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
-{
-  filter_map(argvars, rettv, true);
-}
-
 static void find_some_match(typval_T *const argvars, typval_T *const rettv,
                             const SomeMatchType type)
 {

src/nvim/eval/typval.c

@@ -3249,22 +3249,19 @@ void tv_blob_alloc_ret(typval_T *const ret_tv)
 ///
 /// @param[in]  from  Blob object to copy from.
 /// @param[out]  to  Blob object to copy to.
-void tv_blob_copy(typval_T *const from, typval_T *const to)
+void tv_blob_copy(blob_T *const from, typval_T *const to)
-  FUNC_ATTR_NONNULL_ALL
+  FUNC_ATTR_NONNULL_ARG(2)
 {
-  assert(from->v_type == VAR_BLOB);
-
   to->v_type = VAR_BLOB;
   to->v_lock = VAR_UNLOCKED;
-  if (from->vval.v_blob == NULL) {
+  if (from == NULL) {
     to->vval.v_blob = NULL;
   } else {
     tv_blob_alloc_ret(to);
-    int len = from->vval.v_blob->bv_ga.ga_len;
+    int len = from->bv_ga.ga_len;
 
     if (len > 0) {
-      to->vval.v_blob->bv_ga.ga_data
+      to->vval.v_blob->bv_ga.ga_data = xmemdup(from->bv_ga.ga_data, (size_t)len);
-        = xmemdup(from->vval.v_blob->bv_ga.ga_data, (size_t)len);
     }
     to->vval.v_blob->bv_ga.ga_len = len;
     to->vval.v_blob->bv_ga.ga_maxlen = len;

test/old/testdir/test_filter_map.vim

@@ -111,4 +111,25 @@ func Test_map_and_modify()
   call assert_fails('echo map(d, {k,v -> remove(d, k)})', 'E741:')
 endfunc
 
+func Test_mapnew_dict()
+  let din = #{one: 1, two: 2}
+  let dout = mapnew(din, {k, v -> string(v)})
+  call assert_equal(#{one: 1, two: 2}, din)
+  call assert_equal(#{one: '1', two: '2'}, dout)
+endfunc
+
+func Test_mapnew_list()
+  let lin = [1, 2, 3]
+  let lout = mapnew(lin, {k, v -> string(v)})
+  call assert_equal([1, 2, 3], lin)
+  call assert_equal(['1', '2', '3'], lout)
+endfunc
+
+func Test_mapnew_blob()
+  let bin = 0z123456
+  let bout = mapnew(bin, {k, v -> k == 1 ? 0x99 : v})
+  call assert_equal(0z123456, bin)
+  call assert_equal(0z129956, bout)
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab