file: Use own constants, do not rely on fcntl.h

One of the reasons is that O_RDONLY is zero, which makes checking whether file
is opened read- or write-only harder. It is not guaranteed that on other system
O_WRONLY will not be zero (e.g. because file can only be opened in read-write
mode).

src/nvim/file.c

@@ -8,6 +8,7 @@
 #include <assert.h>
 #include <stddef.h>
 #include <stdbool.h>
+#include <fcntl.h>
 
 #include "auto/config.h"
 
@@ -46,15 +47,37 @@ int file_open(FileDescriptor *const ret_fp, const char *const fname,
               const int flags, const int mode)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
+  int os_open_flags = 0;
   int fd;
+  TriState wr = kNone;
+#define FLAG(flags, flag, fcntl_flags, wrval, cond) \
+  do { \
+    if (flags & flag) { \
+      os_open_flags |= fcntl_flags; \
+      assert(cond); \
+      if (wrval != kNone) { \
+        wr = wrval; \
+      } \
+    } \
+  } while (0)
+  FLAG(flags, kFileWriteOnly, O_WRONLY, kTrue, true);
+  FLAG(flags, kFileCreateOnly, O_CREAT|O_EXCL|O_WRONLY, kTrue, true);
+  FLAG(flags, kFileCreate, O_CREAT|O_WRONLY, kTrue, !(flags & kFileCreateOnly));
+  FLAG(flags, kFileTruncate, O_TRUNC|O_WRONLY, kTrue,
+       !(flags & kFileCreateOnly));
+  FLAG(flags, kFileReadOnly, O_RDONLY, kFalse, wr != kTrue);
+#ifdef O_NOFOLLOW
+  FLAG(flags, kFileNoSymlink, O_NOFOLLOW, kNone, true);
+#endif
+#undef FLAG
 
-  fd = os_open(fname, flags, mode);
+  fd = os_open(fname, os_open_flags, mode);
 
   if (fd < 0) {
     return fd;
   }
 
-  ret_fp->wr = (bool)(!!(flags & FILE_WRITE_ONLY));
+  ret_fp->wr = (wr == kTrue);
   ret_fp->fd = fd;
   ret_fp->eof = false;
   ret_fp->rv = rbuffer_new(RWBUFSIZE);

src/nvim/file.h

@@ -3,7 +3,6 @@
 
 #include <stdbool.h>
 #include <stddef.h>
-#include <fcntl.h>
 
 #include "nvim/func_attr.h"
 #include "nvim/rbuffer.h"
@@ -19,17 +18,18 @@ typedef struct {
 
 /// file_open() flags
 typedef enum {
-  FILE_READ_ONLY = O_RDONLY,  ///< Open file read-only.
-  FILE_CREATE = O_CREAT,  ///< Create file if it does not exist yet.
-  FILE_WRITE_ONLY = O_WRONLY,  ///< Open file for writing only.
-#ifdef O_NOFOLLOW
-  FILE_NOSYMLINK = O_NOFOLLOW,  ///< Do not allow symbolic links.
-#else
-  FILE_NOSYMLINK = 0,
-#endif
-  FILE_CREATE_ONLY = O_CREAT|O_EXCL,  ///< Only create the file, failing
-                                      ///< if it already exists.
-  FILE_TRUNCATE = O_TRUNC,  ///< Truncate the file if it exists.
+  kFileReadOnly = 1,  ///< Open file read-only. Default.
+  kFileCreate = 2,  ///< Create file if it does not exist yet.
+                    ///< Implies kFileWriteOnly.
+  kFileWriteOnly = 4,  ///< Open file for writing only.
+                       ///< Cannot be used with kFileReadOnly.
+  kFileNoSymlink = 8,  ///< Do not allow symbolic links.
+  kFileCreateOnly = 16,  ///< Only create the file, failing if it already
+                         ///< exists. Implies kFileWriteOnly. Cannot be used
+                         ///< with kFileCreate.
+  kFileTruncate = 32,  ///< Truncate the file if it exists.
+                       ///< Implies kFileWriteOnly. Cannot be used with
+                       ///< kFileCreateOnly.
 } FileOpenFlags;
 
 /// Check whether end of file was encountered

src/nvim/shada.c

@@ -796,7 +796,7 @@ static int open_shada_file_for_reading(const char *const fname,
     .error = NULL,
     .eof = false,
     .fpos = 0,
-    .cookie = file_open_new(&error, fname, FILE_READ_ONLY, 0),
+    .cookie = file_open_new(&error, fname, kFileReadOnly, 0),
   };
   if (sd_reader->cookie == NULL) {
     return error;
@@ -2919,8 +2919,7 @@ int shada_write_file(const char *const file, bool nomerge)
     //    reading use u=rw permissions.
 shada_write_file_open: {}
     sd_writer.cookie = file_open_new(
-        &error, tempname, FILE_CREATE_ONLY|FILE_NOSYMLINK|FILE_WRITE_ONLY,
-        perm);
+        &error, tempname, kFileCreateOnly|kFileNoSymlink, perm);
     if (sd_writer.cookie == NULL) {
       if (error == UV_EEXIST || error == UV_ELOOP) {
         // File already exists, try another name
@@ -2964,8 +2963,8 @@ shada_write_file_nomerge: {}
       *tail = tail_save;
     }
     int error;
-    sd_writer.cookie = file_open_new(
-        &error, fname, FILE_CREATE|FILE_WRITE_ONLY|FILE_TRUNCATE, 0600);
+    sd_writer.cookie = file_open_new(&error, fname, kFileCreate|kFileTruncate,
+                                     0600);
     if (sd_writer.cookie == NULL) {
       emsgf(_(SERR "System error while opening ShaDa file %s for writing: %s"),
             fname, os_strerror(error));