vim-patch:8.2.3818: cannot filter or map characters in a string

Problem:    Cannot filter or map characters in a string.
Solution:   Make filter() and map() work on a string. (Naruhiko Nishino,
            closes vim/vim#9327)

c479ce032f

Co-authored-by: rbtnn <naru123456789@gmail.com>

runtime/doc/builtin.txt

@@ -1623,10 +1623,11 @@ filewritable({file})                                            *filewritable()*
 		directory, and we can write to it, the result is 2.
 
 filter({expr1}, {expr2})                                              *filter()*
-		{expr1} must be a |List|, |Blob|, or a |Dictionary|.
+		{expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
 		For each item in {expr1} evaluate {expr2} and when the result
-		is zero remove the item from the |List| or |Dictionary|. For a
-		|Blob| each byte is removed.
+		is zero or false remove the item from the |List| or
+		|Dictionary|.  Similarly for each byte in a |Blob| and each
+		character in a |String|.
 
 		{expr2} must be a |string| or |Funcref|.
 
@@ -1662,14 +1663,16 @@ filter({expr1}, {expr2})                                              *filter()*
 <		If you do not use "val" you can leave it out: >vim
 			call filter(myList, {idx -> idx % 2 == 1})
 <
-		The operation is done in-place.  If you want a |List| or
-		|Dictionary| to remain unmodified make a copy first: >vim
+		For a |List| and a |Dictionary| the operation is done
+		in-place.  If you want it to remain unmodified make a copy
+		first: >vim
 			let l = filter(copy(mylist), 'v:val =~ "KEEP"')
 
-<		Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
-		filtered.  When an error is encountered while evaluating
-		{expr2} no further items in {expr1} are processed.  When
-		{expr2} is a Funcref errors inside a function are ignored,
+<		Returns {expr1}, the |List| or |Dictionary| that was filtered,
+		or a new |Blob| or |String|.
+		When an error is encountered while evaluating {expr2} no
+		further items in {expr1} are processed.
+		When {expr2} is a Funcref errors inside a function are ignored,
 		unless it was defined with the "abort" flag.
 
 finddir({name} [, {path} [, {count}]])                               *finddir()*
@@ -4062,15 +4065,18 @@ luaeval({expr} [, {expr}])                                           *luaeval()*
 		to Vim data structures. See |lua-eval| for more details.
 
 map({expr1}, {expr2})                                                    *map()*
-		{expr1} must be a |List|, |Blob| or |Dictionary|.
-		Replace each item in {expr1} with the result of evaluating
-		{expr2}.  For a |Blob| each byte is replaced.
+		{expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
+		When {expr1} is a |List|| or |Dictionary|, replace each
+		item in {expr1} with the result of evaluating {expr2}.
+		For a |Blob| each byte is replaced.
+		For a |String|, each character, including composing
+		characters, is replaced.
 		If the item type changes you may want to use |mapnew()| to
 		create a new List or Dictionary.
 
-		{expr2} must be a |string| or |Funcref|.
+		{expr2} must be a |String| or |Funcref|.
 
-		If {expr2} is a |string|, inside {expr2} |v:val| has the value
+		If {expr2} is a |String|, inside {expr2} |v:val| has the value
 		of the current item.  For a |Dictionary| |v:key| has the key
 		of the current item and for a |List| |v:key| has the index of
 		the current item.  For a |Blob| |v:key| has the index of the
@@ -4100,14 +4106,15 @@ map({expr1}, {expr2})                                                    *map()*
 <		If you do not use "key" you can use a short name: >vim
 			call map(myDict, {_, val -> 'item: ' .. val})
 <
-		The operation is done in-place.  If you want a |List| or
-		|Dictionary| to remain unmodified make a copy first: >vim
+		The operation is done in-place for a |List| and |Dictionary|.
+		If you want it to remain unmodified make a copy first: >vim
 			let tlist = map(copy(mylist), ' v:val .. "\t"')
 
-<		Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
-		filtered.  When an error is encountered while evaluating
-		{expr2} no further items in {expr1} are processed.  When
-		{expr2} is a Funcref errors inside a function are ignored,
+<		Returns {expr1}, the |List| or |Dictionary| that was filtered,
+		or a new |Blob| or |String|.
+		When an error is encountered while evaluating {expr2} no
+		further items in {expr1} are processed.
+		When {expr2} is a Funcref errors inside a function are ignored,
 		unless it was defined with the "abort" flag.
 
 maparg({name} [, {mode} [, {abbr} [, {dict}]]])                       *maparg()*

runtime/lua/vim/_meta/vimfn.lua

@@ -2005,10 +2005,11 @@ function vim.fn.filereadable(file) end
 --- @return 0|1
 function vim.fn.filewritable(file) end
 
---- {expr1} must be a |List|, |Blob|, or a |Dictionary|.
+--- {expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
 --- For each item in {expr1} evaluate {expr2} and when the result
---- is zero remove the item from the |List| or |Dictionary|. For a
---- |Blob| each byte is removed.
+--- is zero or false remove the item from the |List| or
+--- |Dictionary|.  Similarly for each byte in a |Blob| and each
+--- character in a |String|.
 ---
 --- {expr2} must be a |string| or |Funcref|.
 ---
@@ -2044,14 +2045,16 @@ function vim.fn.filewritable(file) end
 --- <If you do not use "val" you can leave it out: >vim
 ---   call filter(myList, {idx -> idx % 2 == 1})
 --- <
---- The operation is done in-place.  If you want a |List| or
---- |Dictionary| to remain unmodified make a copy first: >vim
+--- For a |List| and a |Dictionary| the operation is done
+--- in-place.  If you want it to remain unmodified make a copy
+--- first: >vim
 ---   let l = filter(copy(mylist), 'v:val =~ "KEEP"')
 ---
---- <Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
---- filtered.  When an error is encountered while evaluating
---- {expr2} no further items in {expr1} are processed.  When
---- {expr2} is a Funcref errors inside a function are ignored,
+--- <Returns {expr1}, the |List| or |Dictionary| that was filtered,
+--- or a new |Blob| or |String|.
+--- When an error is encountered while evaluating {expr2} no
+--- further items in {expr1} are processed.
+--- When {expr2} is a Funcref errors inside a function are ignored,
 --- unless it was defined with the "abort" flag.
 ---
 --- @param expr1 any
@@ -4919,15 +4922,18 @@ function vim.fn.log(expr) end
 --- @return any
 function vim.fn.log10(expr) end
 
---- {expr1} must be a |List|, |Blob| or |Dictionary|.
---- Replace each item in {expr1} with the result of evaluating
---- {expr2}.  For a |Blob| each byte is replaced.
+--- {expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
+--- When {expr1} is a |List|| or |Dictionary|, replace each
+--- item in {expr1} with the result of evaluating {expr2}.
+--- For a |Blob| each byte is replaced.
+--- For a |String|, each character, including composing
+--- characters, is replaced.
 --- If the item type changes you may want to use |mapnew()| to
 --- create a new List or Dictionary.
 ---
---- {expr2} must be a |string| or |Funcref|.
+--- {expr2} must be a |String| or |Funcref|.
 ---
---- If {expr2} is a |string|, inside {expr2} |v:val| has the value
+--- If {expr2} is a |String|, inside {expr2} |v:val| has the value
 --- of the current item.  For a |Dictionary| |v:key| has the key
 --- of the current item and for a |List| |v:key| has the index of
 --- the current item.  For a |Blob| |v:key| has the index of the
@@ -4957,14 +4963,15 @@ function vim.fn.log10(expr) end
 --- <If you do not use "key" you can use a short name: >vim
 ---   call map(myDict, {_, val -> 'item: ' .. val})
 --- <
---- The operation is done in-place.  If you want a |List| or
---- |Dictionary| to remain unmodified make a copy first: >vim
+--- The operation is done in-place for a |List| and |Dictionary|.
+--- If you want it to remain unmodified make a copy first: >vim
 ---   let tlist = map(copy(mylist), ' v:val .. "\t"')
 ---
---- <Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
---- filtered.  When an error is encountered while evaluating
---- {expr2} no further items in {expr1} are processed.  When
---- {expr2} is a Funcref errors inside a function are ignored,
+--- <Returns {expr1}, the |List| or |Dictionary| that was filtered,
+--- or a new |Blob| or |String|.
+--- When an error is encountered while evaluating {expr2} no
+--- further items in {expr1} are processed.
+--- When {expr2} is a Funcref errors inside a function are ignored,
 --- unless it was defined with the "abort" flag.
 ---
 --- @param expr1 any

src/nvim/eval.c

@@ -108,6 +108,8 @@ static const char e_dot_can_only_be_used_on_dictionary_str[]
   = N_("E1203: Dot can only be used on a dictionary: %s");
 static const char e_empty_function_name[]
   = N_("E1192: Empty function name");
+static char e_argument_of_str_must_be_list_string_dictionary_or_blob[]
+  = N_("E1250: Argument of %s must be a List, String, Dictionary or Blob");
 
 static char * const namespace_char = "abglstvw";
 
@@ -5072,8 +5074,11 @@ static void filter_map(typval_T *argvars, typval_T *rettv, filtermap_T filtermap
             && value_check_lock(d->dv_lock, arg_errmsg, TV_TRANSLATE))) {
       return;
     }
+  } else if (argvars[0].v_type == VAR_STRING) {
+    rettv->v_type = VAR_STRING;
+    rettv->vval.v_string = NULL;
   } else {
-    semsg(_(e_listdictblobarg), ermsg);
+    semsg(_(e_argument_of_str_must_be_list_string_dictionary_or_blob), ermsg);
     return;
   }
 
@@ -5193,6 +5198,51 @@ static void filter_map(typval_T *argvars, typval_T *rettv, filtermap_T filtermap
         }
         idx++;
       }
+    } else if (argvars[0].v_type == VAR_STRING) {
+      vimvars[VV_KEY].vv_type = VAR_NUMBER;
+
+      garray_T ga;
+      ga_init(&ga, (int)sizeof(char), 80);
+      int len;
+      for (const char *p = tv_get_string(&argvars[0]); *p != NUL; p += len) {
+        len = utfc_ptr2len(p);
+
+        typval_T tv = {
+          .v_type = VAR_STRING,
+          .v_lock = VAR_UNLOCKED,
+          .vval.v_string = xstrnsave(p, (size_t)len),
+        };
+
+        vimvars[VV_KEY].vv_nr = idx;
+        typval_T newtv;
+        if (filter_map_one(&tv, expr, filtermap, &newtv, &rem) == FAIL
+            || did_emsg) {
+          break;
+        }
+        if (did_emsg) {
+          tv_clear(&newtv);
+          tv_clear(&tv);
+          break;
+        } else if (filtermap != FILTERMAP_FILTER) {
+          if (newtv.v_type != VAR_STRING) {
+            tv_clear(&newtv);
+            tv_clear(&tv);
+            emsg(_(e_stringreq));
+            break;
+          } else {
+            ga_concat(&ga, newtv.vval.v_string);
+          }
+        } else if (!rem) {
+          ga_concat(&ga, tv.vval.v_string);
+        }
+
+        tv_clear(&newtv);
+        tv_clear(&tv);
+
+        idx++;
+      }
+      ga_append(&ga, NUL);
+      rettv->vval.v_string = ga.ga_data;
     } else {
       assert(argvars[0].v_type == VAR_LIST);
 

src/nvim/eval.lua

@@ -2539,10 +2539,11 @@ M.funcs = {
     args = 2,
     base = 1,
     desc = [=[
-      {expr1} must be a |List|, |Blob|, or a |Dictionary|.
+      {expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
       For each item in {expr1} evaluate {expr2} and when the result
-      is zero remove the item from the |List| or |Dictionary|. For a
-      |Blob| each byte is removed.
+      is zero or false remove the item from the |List| or
+      |Dictionary|.  Similarly for each byte in a |Blob| and each
+      character in a |String|.
 
       {expr2} must be a |string| or |Funcref|.
 
@@ -2578,14 +2579,16 @@ M.funcs = {
       <If you do not use "val" you can leave it out: >vim
       	call filter(myList, {idx -> idx % 2 == 1})
       <
-      The operation is done in-place.  If you want a |List| or
-      |Dictionary| to remain unmodified make a copy first: >vim
+      For a |List| and a |Dictionary| the operation is done
+      in-place.  If you want it to remain unmodified make a copy
+      first: >vim
       	let l = filter(copy(mylist), 'v:val =~ "KEEP"')
 
-      <Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
-      filtered.  When an error is encountered while evaluating
-      {expr2} no further items in {expr1} are processed.  When
-      {expr2} is a Funcref errors inside a function are ignored,
+      <Returns {expr1}, the |List| or |Dictionary| that was filtered,
+      or a new |Blob| or |String|.
+      When an error is encountered while evaluating {expr2} no
+      further items in {expr1} are processed.
+      When {expr2} is a Funcref errors inside a function are ignored,
       unless it was defined with the "abort" flag.
 
     ]=],
@@ -6050,15 +6053,18 @@ M.funcs = {
     args = 2,
     base = 1,
     desc = [=[
-      {expr1} must be a |List|, |Blob| or |Dictionary|.
-      Replace each item in {expr1} with the result of evaluating
-      {expr2}.  For a |Blob| each byte is replaced.
+      {expr1} must be a |List|, |String|, |Blob| or |Dictionary|.
+      When {expr1} is a |List|| or |Dictionary|, replace each
+      item in {expr1} with the result of evaluating {expr2}.
+      For a |Blob| each byte is replaced.
+      For a |String|, each character, including composing
+      characters, is replaced.
       If the item type changes you may want to use |mapnew()| to
       create a new List or Dictionary.
 
-      {expr2} must be a |string| or |Funcref|.
+      {expr2} must be a |String| or |Funcref|.
 
-      If {expr2} is a |string|, inside {expr2} |v:val| has the value
+      If {expr2} is a |String|, inside {expr2} |v:val| has the value
       of the current item.  For a |Dictionary| |v:key| has the key
       of the current item and for a |List| |v:key| has the index of
       the current item.  For a |Blob| |v:key| has the index of the
@@ -6088,14 +6094,15 @@ M.funcs = {
       <If you do not use "key" you can use a short name: >vim
       	call map(myDict, {_, val -> 'item: ' .. val})
       <
-      The operation is done in-place.  If you want a |List| or
-      |Dictionary| to remain unmodified make a copy first: >vim
+      The operation is done in-place for a |List| and |Dictionary|.
+      If you want it to remain unmodified make a copy first: >vim
       	let tlist = map(copy(mylist), ' v:val .. "\t"')
 
-      <Returns {expr1}, the |List|, |Blob| or |Dictionary| that was
-      filtered.  When an error is encountered while evaluating
-      {expr2} no further items in {expr1} are processed.  When
-      {expr2} is a Funcref errors inside a function are ignored,
+      <Returns {expr1}, the |List| or |Dictionary| that was filtered,
+      or a new |Blob| or |String|.
+      When an error is encountered while evaluating {expr2} no
+      further items in {expr1} are processed.
+      When {expr2} is a Funcref errors inside a function are ignored,
       unless it was defined with the "abort" flag.
 
     ]=],

test/old/testdir/test_filter_map.vim

@@ -84,8 +84,6 @@ endfunc
 func Test_map_filter_fails()
   call assert_fails('call map([1], "42 +")', 'E15:')
   call assert_fails('call filter([1], "42 +")', 'E15:')
-  call assert_fails("let l = map('abc', '\"> \" . v:val')", 'E896:')
-  call assert_fails("let l = filter('abc', '\"> \" . v:val')", 'E896:')
   call assert_fails("let l = filter([1, 2, 3], '{}')", 'E728:')
   call assert_fails("let l = filter({'k' : 10}, '{}')", 'E728:')
   call assert_fails("let l = filter([1, 2], {})", 'E731:')
@@ -138,4 +136,68 @@ func Test_mapnew_blob()
   call assert_equal(0z129956, bout)
 endfunc
 
+func Test_filter_map_string()
+  let s = "abc"
+
+  " filter()
+  call filter(s, '"b" != v:val')
+  call assert_equal(s, s)
+  call assert_equal('ac', filter('abc', '"b" != v:val'))
+  call assert_equal('あいうえお', filter('あxいxうxえxお', '"x" != v:val'))
+  call assert_equal('あa😊💕💕b💕', filter('あxax😊x💕💕b💕x', '"x" != v:val'))
+  call assert_equal('xxxx', filter('あxax😊x💕💕b💕x', '"x" == v:val'))
+  let t = "%),:;>?]}’”†‡…‰,‱‼⁇⁈⁉℃℉,、。〉》」,』】〕〗〙〛,！），．：,；？,］｝"
+  let u = "%):;>?]}’”†‡…‰‱‼⁇⁈⁉℃℉、。〉》」』】〕〗〙〛！），．：；？］｝"
+  call assert_equal(u, filter(t, '"," != v:val'))
+  call assert_equal('', filter('abc', '0'))
+  call assert_equal('ac', filter('abc', { i, x -> "b" != x }))
+  call assert_equal('あいうえお', filter('あxいxうxえxお', { i, x -> "x" != x }))
+  call assert_equal('', filter('abc', { i, x -> v:false }))
+
+  " map()
+  call map(s, 'nr2char(char2nr(v:val) + 2)')
+  call assert_equal(s, s)
+  call assert_equal('cde', map('abc', 'nr2char(char2nr(v:val) + 2)'))
+  call assert_equal('[あ][i][う][え][お]', map('あiうえお', '"[" .. v:val .. "]"'))
+  call assert_equal('[あ][a][😊][,][‱][‼][⁇][⁈][⁉][💕][b][💕][c][💕]', map('あa😊,‱‼⁇⁈⁉💕b💕c💕', '"[" .. v:val .. "]"'))
+  call assert_equal('', map('abc', '""'))
+  call assert_equal('cde', map('abc', { i, x -> nr2char(char2nr(x) + 2) }))
+  call assert_equal('[あ][i][う][え][お]', map('あiうえお', { i, x -> '[' .. x .. ']' }))
+  call assert_equal('', map('abc', { i, x -> '' }))
+
+  " mapnew()
+  call mapnew(s, 'nr2char(char2nr(v:val) + 2)')
+  call assert_equal(s, s)
+  call assert_equal('cde', mapnew('abc', 'nr2char(char2nr(v:val) + 2)'))
+  call assert_equal('[あ][i][う][え][お]', mapnew('あiうえお', '"[" .. v:val .. "]"'))
+  call assert_equal('[あ][a][😊][,][‱][‼][⁇][⁈][⁉][💕][b][💕][c][💕]', mapnew('あa😊,‱‼⁇⁈⁉💕b💕c💕', '"[" .. v:val .. "]"'))
+  call assert_equal('', mapnew('abc', '""'))
+  call assert_equal('cde', mapnew('abc', { i, x -> nr2char(char2nr(x) + 2) }))
+  call assert_equal('[あ][i][う][え][お]', mapnew('あiうえお', { i, x -> '[' .. x .. ']' }))
+  call assert_equal('', mapnew('abc', { i, x -> '' }))
+
+  " map() and filter()
+  call assert_equal('[あ][⁈][a][😊][⁉][💕][💕][b][💕]', map(filter('あx⁈ax😊x⁉💕💕b💕x', '"x" != v:val'), '"[" .. v:val .. "]"'))
+
+  " patterns-composing(\Z)
+  call assert_equal('ॠॠ', filter('ऊॠॡ,ऊॠॡ', {i,x -> x =~ '\Z' .. nr2char(0x0960) }))
+  call assert_equal('àà', filter('càt,càt', {i,x -> x =~ '\Za' }))
+  call assert_equal('ÅÅ', filter('Åström,Åström', {i,x -> x =~ '\Z' .. nr2char(0xc5) }))
+  call assert_equal('öö', filter('Åström,Åström', {i,x -> x =~ '\Z' .. nr2char(0xf6) }))
+  call assert_equal('ऊ@ॡ', map('ऊॠॡ', {i,x -> x =~ '\Z' .. nr2char(0x0960) ? '@' : x }))
+  call assert_equal('c@t', map('càt', {i,x -> x =~ '\Za' ? '@' : x }))
+  call assert_equal('@ström', map('Åström', {i,x -> x =~ '\Z' .. nr2char(0xc5) ? '@' : x }))
+  call assert_equal('Åstr@m', map('Åström', {i,x -> x =~ '\Z' .. nr2char(0xf6) ? '@' : x }))
+
+  " patterns-composing(\%C)
+  call assert_equal('ॠॠ', filter('ऊॠॡ,ऊॠॡ', {i,x -> x =~ nr2char(0x0960) .. '\%C' }))
+  call assert_equal('àà', filter('càt,càt', {i,x -> x =~ 'a' .. '\%C' }))
+  call assert_equal('ÅÅ', filter('Åström,Åström', {i,x -> x =~ nr2char(0xc5) .. '\%C' }))
+  call assert_equal('öö', filter('Åström,Åström', {i,x -> x =~ nr2char(0xf6) .. '\%C' }))
+  call assert_equal('ऊ@ॡ', map('ऊॠॡ', {i,x -> x =~ nr2char(0x0960) .. '\%C' ? '@' : x }))
+  call assert_equal('c@t', map('càt', {i,x -> x =~ 'a' .. '\%C' ? '@' : x }))
+  call assert_equal('@ström', map('Åström', {i,x -> x =~ nr2char(0xc5) .. '\%C' ? '@' : x }))
+  call assert_equal('Åstr@m', map('Åström', {i,x -> x =~ nr2char(0xf6) .. '\%C' ? '@' : x }))
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab