vim-patch:7.4.1975

Problem:    On MS-Windows large files (> 2Gbyte) cause problems.
Solution:   Use "off_T" instead of "off_t".  Use "stat_T" instead of "struct
            stat".  Use 64 bit system functions if available.  (Ken Takata)

8767f52fbf

Only the off_T changes are relevant, since all the "struct stat" usage
is abstracted by libuv.

src/nvim/fileio.c

@@ -281,7 +281,7 @@ readfile (
   colnr_T len;
   long size = 0;
   char_u      *p = NULL;
-  off_t filesize = 0;
+  off_T filesize = 0;
   int skip_read = FALSE;
   context_sha256_T sha_ctx;
   int read_undo_file = FALSE;
@@ -777,7 +777,7 @@ retry:
     if (read_buffer) {
       read_buf_lnum = 1;
       read_buf_col = 0;
-    } else if (read_stdin || lseek(fd, (off_t)0L, SEEK_SET) != 0) {
+    } else if (read_stdin || vim_lseek(fd, (off_T)0L, SEEK_SET) != 0) {
       /* Can't rewind the file, give up. */
       error = TRUE;
       goto failed;
@@ -1626,7 +1626,7 @@ rewind_retry:
                 if (   try_unix
                        && !read_stdin
                        && (read_buffer
-                           || lseek(fd, (off_t)0L, SEEK_SET) == 0)) {
+                           || vim_lseek(fd, (off_T)0L, SEEK_SET) == 0)) {
                   fileformat = EOL_UNIX;
                   if (set_options)
                     set_fileformat(EOL_UNIX, OPT_LOCAL);
@@ -3833,7 +3833,7 @@ static bool msg_add_fileformat(int eol_type)
 /*
  * Append line and character count to IObuff.
  */
-void msg_add_lines(int insert_space, long lnum, off_t nchars)
+void msg_add_lines(int insert_space, long lnum, off_T nchars)
 {
   char_u  *p;
 

src/nvim/globals.h

@@ -97,6 +97,34 @@ typedef enum {
 EXTERN long Rows INIT(= DFLT_ROWS);     // nr of rows in the screen
 EXTERN long Columns INIT(= DFLT_COLS);  // nr of columns in the screen
 
+// We use 64-bit file functions here, if available.  E.g. ftello() returns
+// off_t instead of long, which helps if long is 32 bit and off_t is 64 bit.
+// We assume that when fseeko() is available then ftello() is too.
+// Note that Windows has different function names.
+#if (defined(_MSC_VER) && (_MSC_VER >= 1300)) || defined(__MINGW32__)
+typedef __int64 off_T;
+# ifdef __MINGW32__
+#  define vim_lseek lseek64
+#  define vim_fseek fseeko64
+#  define vim_ftell ftello64
+# else
+#  define vim_lseek _lseeki64
+#  define vim_fseek _fseeki64
+#  define vim_ftell _ftelli64
+# endif
+#else
+typedef off_t off_T;
+# ifdef HAVE_FSEEKO
+#  define vim_lseek lseek
+#  define vim_ftell ftello
+#  define vim_fseek fseeko
+# else
+#  define vim_lseek lseek
+#  define vim_ftell ftell
+#  define vim_fseek(a, b, c) fseek(a, (long)b, c)
+# endif
+#endif
+
 /*
  * The characters and attributes cached for the screen.
  */

src/nvim/memfile.c

@@ -115,18 +115,18 @@ memfile_T *mf_open(char_u *fname, int flags)
     }
   }
 
-  off_t size;
+  off_T size;
 
   // When recovering, the actual block size will be retrieved from block 0
   // in ml_recover(). The size used here may be wrong, therefore mf_blocknr_max
   // must be rounded up.
   if (mfp->mf_fd < 0
       || (flags & (O_TRUNC|O_EXCL))
-      || (size = lseek(mfp->mf_fd, (off_t)0L, SEEK_END)) <= 0) {
+      || (size = vim_lseek(mfp->mf_fd, 0L, SEEK_END)) <= 0) {
     // no file or empty file
     mfp->mf_blocknr_max = 0;
   } else {
-    assert(sizeof(off_t) <= sizeof(blocknr_T)
+    assert(sizeof(off_T) <= sizeof(blocknr_T)
            && mfp->mf_page_size > 0
            && mfp->mf_page_size - 1 <= INT64_MAX - size);
     mfp->mf_blocknr_max = (((blocknr_T)size + mfp->mf_page_size - 1)
@@ -689,9 +689,9 @@ static int mf_read(memfile_T *mfp, bhdr_T *hp)
     return FAIL;
 
   unsigned page_size = mfp->mf_page_size;
-  // TODO(elmart): Check (page_size * hp->bh_bnum) within off_t bounds.
+  // TODO(elmart): Check (page_size * hp->bh_bnum) within off_T bounds.
-  off_t offset = (off_t)(page_size * hp->bh_bnum);
+  off_T offset = (off_T)(page_size * hp->bh_bnum);
-  if (lseek(mfp->mf_fd, offset, SEEK_SET) != offset) {
+  if (vim_lseek(mfp->mf_fd, offset, SEEK_SET) != offset) {
     PERROR(_("E294: Seek error in swap file read"));
     return FAIL;
   }
@@ -716,7 +716,7 @@ static int mf_read(memfile_T *mfp, bhdr_T *hp)
 ///                - Write error in swap file.
 static int mf_write(memfile_T *mfp, bhdr_T *hp)
 {
-  off_t offset;             // offset in the file
+  off_T offset;             // offset in the file
   blocknr_T nr;             // block nr which is being written
   bhdr_T *hp2;
   unsigned page_size;       // number of bytes in a page
@@ -745,9 +745,9 @@ static int mf_write(memfile_T *mfp, bhdr_T *hp)
       hp2 = hp;
     }
 
-    // TODO(elmart): Check (page_size * nr) within off_t bounds.
+    // TODO(elmart): Check (page_size * nr) within off_T bounds.
-    offset = (off_t)(page_size * nr);
+    offset = (off_T)(page_size * nr);
-    if (lseek(mfp->mf_fd, offset, SEEK_SET) != offset) {
+    if (vim_lseek(mfp->mf_fd, offset, SEEK_SET) != offset) {
       PERROR(_("E296: Seek error in swap file write"));
       return FAIL;
     }

src/nvim/memline.c

@@ -763,7 +763,7 @@ void ml_recover(void)
   int idx;
   int top;
   int txt_start;
-  off_t size;
+  off_T size;
   int called_from_main;
   int serious_error = TRUE;
   long mtime;
@@ -914,7 +914,7 @@ void ml_recover(void)
       msg_end();
       goto theend;
     }
-    if ((size = lseek(mfp->mf_fd, (off_t)0L, SEEK_END)) <= 0)
+    if ((size = vim_lseek(mfp->mf_fd, (off_T)0L, SEEK_END)) <= 0)
       mfp->mf_blocknr_max = 0;              /* no file or empty file */
     else
       mfp->mf_blocknr_max = size / mfp->mf_page_size;

src/nvim/tag.c

@@ -105,15 +105,6 @@ static char_u *topmsg = (char_u *)N_("E556: at top of tag stack");
 
 static char_u   *tagmatchname = NULL;   /* name of last used tag */
 
-/*
- * We use ftello() here, if available.  It returns off_t instead of long,
- * which helps if long is 32 bit and off_t is 64 bit.
- * We assume that when fseeko() is available then ftello() is too.
- */
-#ifdef HAVE_FSEEKO
-# define ftell ftello
-#endif
-
 /*
  * Tag for preview window is remembered separately, to avoid messing up the
  * normal tagstack.
@@ -1091,19 +1082,19 @@ find_tags (
   int tag_file_sorted = NUL;            /* !_TAG_FILE_SORTED value */
   struct tag_search_info        /* Binary search file offsets */
   {
-    off_t low_offset;           /* offset for first char of first line that
+    off_T low_offset;           /* offset for first char of first line that
                                    could match */
-    off_t high_offset;          /* offset of char after last line that could
+    off_T high_offset;          /* offset of char after last line that could
                                    match */
-    off_t curr_offset;          /* Current file offset in search range */
+    off_T curr_offset;          /* Current file offset in search range */
-    off_t curr_offset_used;       /* curr_offset used when skipping back */
+    off_T curr_offset_used;       /* curr_offset used when skipping back */
-    off_t match_offset;         /* Where the binary search found a tag */
+    off_T match_offset;         /* Where the binary search found a tag */
     int low_char;               /* first char at low_offset */
     int high_char;              /* first char at high_offset */
   } search_info;
-  off_t filesize;
+  off_T filesize;
   int tagcmp;
-  off_t offset;
+  off_T offset;
   int round;
   enum {
     TS_START,                   /* at start of file */
@@ -1378,36 +1369,28 @@ find_tags (
         if (state == TS_BINARY || state == TS_SKIP_BACK) {
           /* Adjust the search file offset to the correct position */
           search_info.curr_offset_used = search_info.curr_offset;
-#ifdef HAVE_FSEEKO
+          vim_fseek(fp, search_info.curr_offset, SEEK_SET);
-          fseeko(fp, search_info.curr_offset, SEEK_SET);
-#else
-          fseek(fp, (long)search_info.curr_offset, SEEK_SET);
-#endif
           eof = vim_fgets(lbuf, LSIZE, fp);
           if (!eof && search_info.curr_offset != 0) {
             /* The explicit cast is to work around a bug in gcc 3.4.2
              * (repeated below). */
-            search_info.curr_offset = ftell(fp);
+            search_info.curr_offset = vim_ftell(fp);
             if (search_info.curr_offset == search_info.high_offset) {
               /* oops, gone a bit too far; try from low offset */
-#ifdef HAVE_FSEEKO
+              vim_fseek(fp, search_info.low_offset, SEEK_SET);
-              fseeko(fp, search_info.low_offset, SEEK_SET);
-#else
-              fseek(fp, (long)search_info.low_offset, SEEK_SET);
-#endif
               search_info.curr_offset = search_info.low_offset;
             }
             eof = vim_fgets(lbuf, LSIZE, fp);
           }
           /* skip empty and blank lines */
           while (!eof && vim_isblankline(lbuf)) {
-            search_info.curr_offset = ftell(fp);
+            search_info.curr_offset = vim_ftell(fp);
             eof = vim_fgets(lbuf, LSIZE, fp);
           }
           if (eof) {
             /* Hit end of file.  Skip backwards. */
             state = TS_SKIP_BACK;
-            search_info.match_offset = ftell(fp);
+            search_info.match_offset = vim_ftell(fp);
             search_info.curr_offset = search_info.curr_offset_used;
             continue;
           }
@@ -1523,10 +1506,10 @@ line_read_in:
            */
           if (state == TS_BINARY) {
             // Get the tag file size.
-            if ((filesize = lseek(fileno(fp), (off_t)0L, SEEK_END)) <= 0) {
+            if ((filesize = vim_lseek(fileno(fp), (off_T)0L, SEEK_END)) <= 0) {
               state = TS_LINEAR;
             } else {
-              lseek(fileno(fp), (off_t)0L, SEEK_SET);
+              vim_lseek(fileno(fp), (off_T)0L, SEEK_SET);
 
               /* Calculate the first read offset in the file.  Start
                * the search in the middle of the file. */
@@ -1564,11 +1547,7 @@ parse_line:
                 /* Avoid getting stuck. */
                 linear = TRUE;
                 state = TS_LINEAR;
-# ifdef HAVE_FSEEKO
+                vim_fseek(fp, search_info.low_offset, SEEK_SET);
-                fseeko(fp, search_info.low_offset, SEEK_SET);
-# else
-                fseek(fp, (long)search_info.low_offset, SEEK_SET);
-# endif
               }
               continue;
             }
@@ -1647,7 +1626,7 @@ parse_line:
               continue;
             }
             if (tagcmp < 0) {
-              search_info.curr_offset = ftell(fp);
+              search_info.curr_offset = vim_ftell(fp);
               if (search_info.curr_offset < search_info.high_offset) {
                 search_info.low_offset = search_info.curr_offset;
                 if (sortic)
@@ -1683,7 +1662,7 @@ parse_line:
           } else if (state == TS_STEP_FORWARD)   {
             assert(cmplen >= 0);
             if (mb_strnicmp(tagp.tagname, orgpat.head, (size_t)cmplen) != 0) {
-              if ((off_t)ftell(fp) > search_info.match_offset)
+              if ((off_T)vim_ftell(fp) > search_info.match_offset)
                 break;                  /* past last match */
               else
                 continue;               /* before first match */
@@ -1908,7 +1887,7 @@ parse_line:
       if (line_error) {
         EMSG2(_("E431: Format error in tags file \"%s\""), tag_fname);
         if (!use_cscope)
-          EMSGN(_("Before byte %" PRId64), ftell(fp));
+          EMSGN(_("Before byte %" PRId64), vim_ftell(fp));
         stop_searching = TRUE;
         line_error = FALSE;
       }

src/nvim/testdir/Makefile

@@ -60,6 +60,7 @@ NEW_TESTS ?= \
 	    test_quickfix.res \
 	    test_signs.res \
 	    test_smartindent.res \
+	    test_stat.res \
 	    test_substitute.res \
 	    test_syntax.res \
 	    test_tabpage.res \

src/nvim/testdir/test_largefile.vim

@@ -0,0 +1,30 @@
+" Tests for large files
+" This is only executed manually: "make test_largefile".
+" This is not run as part of "make test".
+
+func Test_largefile()
+  let fname = 'Xlarge.txt'
+
+  call delete(fname)
+  exe "e" fname
+  " Make sure that a line break is 1 byte (LF).
+  set ff=unix
+  set undolevels=-1
+  " Input 99 'A's. The line becomes 100 bytes including a line break.
+  exe "normal 99iA\<Esc>"
+  yank
+  " Put 39,999,999 times. The file becomes 4,000,000,000 bytes.
+  normal 39999999p
+  " Moving around in the file randomly.
+  normal G
+  normal 10%
+  normal 90%
+  normal 50%
+  normal gg
+  w
+  " Check if the file size is larger than 2^31 - 1 bytes.
+  " Note that getfsize() returns -2 if a Number is 32 bits.
+  let fsize=getfsize(fname)
+  call assert_true(fsize > 2147483647 || fsize == -2)
+  "call delete(fname)
+endfunc

src/nvim/testdir/test_stat.vim

@@ -0,0 +1,64 @@
+" Tests for stat functions and checktime
+
+func Test_existent_file()
+  let fname='Xtest.tmp'
+
+  let ts=localtime()
+  sleep 1
+  let fl=['Hello World!']
+  call writefile(fl, fname)
+  let tf=getftime(fname)
+  sleep 1
+  let te=localtime()
+
+  call assert_true(ts <= tf && tf <= te)
+  call assert_equal(strlen(fl[0] . "\n"), getfsize(fname))
+  call assert_equal('file', getftype(fname))
+  call assert_equal('rw-', getfperm(fname)[0:2])
+endfunc
+
+func Test_existent_directory()
+  let dname='.'
+
+  call assert_equal(0, getfsize(dname))
+  call assert_equal('dir', getftype(dname))
+  call assert_equal('rwx', getfperm(dname)[0:2])
+endfunc
+
+func Test_checktime()
+  let fname='Xtest.tmp'
+
+  let fl=['Hello World!']
+  call writefile(fl, fname)
+  set autoread
+  exec 'e' fname
+  sleep 2
+  let fl=readfile(fname)
+  let fl[0] .= ' - checktime'
+  call writefile(fl, fname)
+  checktime
+  call assert_equal(fl[0], getline(1))
+endfunc
+
+func Test_nonexistent_file()
+  let fname='Xtest.tmp'
+
+  call delete(fname)
+  call assert_equal(-1, getftime(fname))
+  call assert_equal(-1, getfsize(fname))
+  call assert_equal('', getftype(fname))
+  call assert_equal('', getfperm(fname))
+endfunc
+
+func Test_win32_symlink_dir()
+  " On Windows, non-admin users cannot create symlinks.
+  " So we use an existing symlink for this test.
+  if has('win32')
+    " Check if 'C:\Users\All Users' is a symlink to a directory.
+    let res=system('dir C:\Users /a')
+    if match(res, '\C<SYMLINKD> *All Users') >= 0
+      " Get the filetype of the symlink.
+      call assert_equal('dir', getftype('C:\Users\All Users'))
+    endif
+  endif
+endfunc

src/nvim/version.c

@@ -469,7 +469,7 @@ static const int included_patches[] = {
   // 1978,
   // 1977,
   // 1976,
-  // 1975,
+  1975,
   // 1974 NA
   1973,
   // 1972 NA