fnamemodify: fix handling of :r after :e #11165

- Test fnamemodify()
- Test handling of `expand("%:e:e:r")`.
- Fix :e:e:r on filenames with insufficiently many extensions

During `fnamemodify()`, ensuring that we don't go before the filename's
tail is insufficient in cases where we've already handled a ":e"
modifier, for example:

```
"path/to/this.file.ext" :e:e:r:r
         ^    ^-------- *fnamep
         +------------- tail
```

This means for a ":r", we'll go before `*fnamep`, and outside the bounds
of the filename. This is both incorrect and causes neovim to exit with
an allocation error.

We exit because we attempt to calculate `s - *fnamep` (line 23948).
Since `s` is before `*fnamep`, we caluclate a negative length, which
ends up being interpreted as an amount to allocate, causing neovim to
exit with ENOMEM (`memory.c:xmalloc`).

We must instead ensure we don't go before `*fnamep` nor `tail`.
The check for `tail` is still relevant, for example:

```
"path/to/this.file.ext" :r:r:r
 ^       ^------------- tail
 +--------------------- *fnamep
```
Here we don't want to go before `tail`.

close #11165

src/nvim/eval.c

@@ -24051,22 +24051,47 @@ repeat:
      * - for second :e: before the current fname
      * - otherwise: The last '.'
      */
-    if (src[*usedlen + 1] == 'e' && *fnamep > tail)
+    const bool is_second_e = *fnamep > tail;
+    if (src[*usedlen + 1] == 'e' && is_second_e) {
       s = *fnamep - 2;
-    else
+    } else {
       s = *fnamep + *fnamelen - 1;
-    for (; s > tail; --s)
-      if (s[0] == '.')
+    }
+
+    for (; s > tail; s--) {
+      if (s[0] == '.') {
         break;
-    if (src[*usedlen + 1] == 'e') {             /* :e */
-      if (s > tail) {
-        *fnamelen += (size_t)(*fnamep - (s + 1));
-        *fnamep = s + 1;
-      } else if (*fnamep <= tail)
+      }
+    }
+    if (src[*usedlen + 1] == 'e') {
+      if (s > tail || (0 && is_second_e && s == tail)) {
+        // we stopped at a '.' (so anchor to &'.' + 1)
+        char_u *newstart = s + 1;
+        size_t distance_stepped_back = *fnamep - newstart;
+        *fnamelen += distance_stepped_back;
+        *fnamep = newstart;
+      } else if (*fnamep <= tail) {
         *fnamelen = 0;
-    } else {                          /* :r */
-      if (s > tail)             /* remove one extension */
+      }
+    } else {
+      // :r - Remove one extension
+      //
+      // Ensure that `s` doesn't go before `*fnamep`,
+      // since then we're taking too many roots:
+      //
+      // "path/to/this.file.ext" :e:e:r:r
+      //          ^    ^-------- *fnamep
+      //          +------------- tail
+      //
+      // Also ensure `s` doesn't go before `tail`,
+      // since then we're taking too many roots again:
+      //
+      // "path/to/this.file.ext" :r:r:r
+      //  ^       ^------------- tail
+      //  +--------------------- *fnamep
+      if (s > MAX(tail, *fnamep)) {
         *fnamelen = (size_t)(s - *fnamep);
+      }
     }
     *usedlen += 2;
   }

test/functional/eval/fnamemodify_spec.lua

@@ -3,8 +3,14 @@ local clear = helpers.clear
 local eq = helpers.eq
 local iswin = helpers.iswin
 local fnamemodify = helpers.funcs.fnamemodify
+local getcwd = helpers.funcs.getcwd
 local command = helpers.command
 local write_file = helpers.write_file
+local alter_slashes = helpers.alter_slashes
+
+local function eq_slashconvert(expected, got)
+  eq(alter_slashes(expected), alter_slashes(got))
+end
 
 describe('fnamemodify()', function()
   setup(function()
@@ -17,7 +23,7 @@ describe('fnamemodify()', function()
     os.remove('Xtest-fnamemodify.txt')
   end)
 
-  it('works', function()
+  it('handles the root path', function()
     local root = helpers.pathroot()
     eq(root, fnamemodify([[/]], ':p:h'))
     eq(root, fnamemodify([[/]], ':p'))
@@ -36,4 +42,115 @@ describe('fnamemodify()', function()
   it(':8 works', function()
     eq('Xtest-fnamemodify.txt', fnamemodify([[Xtest-fnamemodify.txt]], ':8'))
   end)
+
+  it('handles examples from ":help filename-modifiers"', function()
+    local filename = "src/version.c"
+    local cwd = getcwd()
+
+    eq_slashconvert(cwd .. '/src/version.c', fnamemodify(filename, ':p'))
+
+    eq_slashconvert('src/version.c', fnamemodify(filename, ':p:.'))
+    eq_slashconvert(cwd .. '/src', fnamemodify(filename, ':p:h'))
+    eq_slashconvert(cwd .. '', fnamemodify(filename, ':p:h:h'))
+    eq('version.c', fnamemodify(filename, ':p:t'))
+    eq_slashconvert(cwd .. '/src/version', fnamemodify(filename, ':p:r'))
+
+    eq_slashconvert(cwd .. '/src/main.c', fnamemodify(filename, ':s?version?main?:p'))
+
+    local converted_cwd = cwd:gsub('/', '\\')
+    eq(converted_cwd .. '\\src\\version.c', fnamemodify(filename, ':p:gs?/?\\\\?'))
+
+    eq('src', fnamemodify(filename, ':h'))
+    eq('version.c', fnamemodify(filename, ':t'))
+    eq_slashconvert('src/version', fnamemodify(filename, ':r'))
+    eq('version', fnamemodify(filename, ':t:r'))
+    eq('c', fnamemodify(filename, ':e'))
+
+    eq_slashconvert('src/main.c', fnamemodify(filename, ':s?version?main?'))
+  end)
+
+  it('handles advanced examples from ":help filename-modifiers"', function()
+    local filename = "src/version.c.gz"
+
+    eq('gz', fnamemodify(filename, ':e'))
+    eq('c.gz', fnamemodify(filename, ':e:e'))
+    eq('c.gz', fnamemodify(filename, ':e:e:e'))
+
+    eq('c', fnamemodify(filename, ':e:e:r'))
+
+    eq_slashconvert('src/version.c', fnamemodify(filename, ':r'))
+    eq('c', fnamemodify(filename, ':r:e'))
+
+    eq_slashconvert('src/version', fnamemodify(filename, ':r:r'))
+    eq_slashconvert('src/version', fnamemodify(filename, ':r:r:r'))
+  end)
+
+  it('handles :h', function()
+    eq('.', fnamemodify('hello.txt', ':h'))
+
+    eq_slashconvert('path/to', fnamemodify('path/to/hello.txt', ':h'))
+  end)
+
+  it('handles :t', function()
+    eq('hello.txt', fnamemodify('hello.txt', ':t'))
+    eq_slashconvert('hello.txt', fnamemodify('path/to/hello.txt', ':t'))
+  end)
+
+  it('handles :r', function()
+    eq('hello', fnamemodify('hello.txt', ':r'))
+    eq_slashconvert('path/to/hello', fnamemodify('path/to/hello.txt', ':r'))
+  end)
+
+  it('handles :e', function()
+    eq('txt', fnamemodify('hello.txt', ':e'))
+    eq_slashconvert('txt', fnamemodify('path/to/hello.txt', ':e'))
+  end)
+
+  it('handles regex replacements', function()
+    eq('content-there-here.txt', fnamemodify('content-here-here.txt', ':s/here/there/'))
+    eq('content-there-there.txt', fnamemodify('content-here-here.txt', ':gs/here/there/'))
+  end)
+
+  it('handles shell escape', function()
+    local expected
+
+    if iswin() then
+      -- we expand with double-quotes on Windows
+      expected = [["hello there! quote ' newline]] .. '\n' .. [["]]
+    else
+      expected = [['hello there! quote '\'' newline]] .. '\n' .. [[']]
+    end
+
+    eq(expected, fnamemodify("hello there! quote ' newline\n", ':S'))
+  end)
+
+  it('can combine :e and :r', function()
+    -- simple, single extension filename
+    eq('c', fnamemodify('a.c', ':e'))
+    eq('c', fnamemodify('a.c', ':e:e'))
+    eq('c', fnamemodify('a.c', ':e:e:r'))
+    eq('c', fnamemodify('a.c', ':e:e:r:r'))
+
+    -- multi extension filename
+    eq('rb', fnamemodify('a.spec.rb', ':e:r'))
+    eq('rb', fnamemodify('a.spec.rb', ':e:r:r'))
+
+    eq('spec', fnamemodify('a.spec.rb', ':e:e:r'))
+    eq('spec', fnamemodify('a.spec.rb', ':e:e:r:r'))
+
+    eq('spec', fnamemodify('a.b.spec.rb', ':e:e:r'))
+    eq('b.spec', fnamemodify('a.b.spec.rb', ':e:e:e:r'))
+    eq('b', fnamemodify('a.b.spec.rb', ':e:e:e:r:r'))
+
+    eq('spec', fnamemodify('a.b.spec.rb', ':r:e'))
+    eq('b', fnamemodify('a.b.spec.rb', ':r:r:e'))
+
+    -- extraneous :e expansions
+    eq('c', fnamemodify('a.b.c.d.e', ':r:r:e'))
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e'))
+
+    -- :e never includes the whole filename, so "a.b":e:e:e --> "b"
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e'))
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e:e'))
+  end)
 end)