From 740c5b13ead2fe02691554ef9b585d7cb4ba19fa Mon Sep 17 00:00:00 2001 From: Charles Blake Date: Mon, 10 Dec 2018 11:40:08 -0500 Subject: [PATCH 1/3] Let handles be seen outside of `memfiles` module so that "updating" operations (like eg., resizing a file and re-mapping) do not need to worry about race conditions of re-opened paths, renamed parent directories and that sort of thing. Operating directly on already open handles is both safer and more efficient than relying upon the stability of filesystem paths. --- lib/pure/memfiles.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index e5345e645c..61eff3295d 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -36,11 +36,11 @@ type size*: int ## size of the memory mapped file when defined(windows): - fHandle: Handle - mapHandle: Handle - wasOpened: bool ## only close if wasOpened + fHandle*: Handle + mapHandle*: Handle + wasOpened*: bool ## only close if wasOpened else: - handle: cint + handle*: cint proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = From 369ac2dd2db036c591983839fcf4e6129a501be8 Mon Sep 17 00:00:00 2001 From: Charles Blake Date: Tue, 11 Dec 2018 10:27:27 -0500 Subject: [PATCH 2/3] Address dom96/Araq opinions in https://github.com/nim-lang/Nim/pull/9922 Updating accessors are also provided since the idea of this change is to allow "updating" operations external to the module which are by their very nature closely tied to module internals (as well as to OS interface details). --- lib/pure/memfiles.nim | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 61eff3295d..b742423708 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -36,11 +36,26 @@ type size*: int ## size of the memory mapped file when defined(windows): - fHandle*: Handle - mapHandle*: Handle - wasOpened*: bool ## only close if wasOpened + fHandle: Handle + mapHandle: Handle + wasOpened: bool ## only close if wasOpened else: - handle*: cint + handle: cint + +when defined(windows): + proc fHandle*(m: MemFile): Handle = m.fHandle + proc mapHandle*(m: MemFile): Handle = m.mapHandle + proc wasOpened*(m: MemFile): bool = m.wasOpened + proc `fHandle=`*(m: var MemFile, fHandle: Handle) = + m.fHandle = fHandle + proc `mapHandle=`*(m: var MemFile, mapHandle: Handle) = + m.mapHandle = mapHandle + proc `wasOpened=`*(m: var MemFile, wasOpened: bool) = + m.wasOpened = wasOpened +else: + proc handle*(m: MemFile): cint = m.handle + proc `handle=`*(m: var MemFile, handle: cint) = + m.handle = handle proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = From b92594572e485fabe4bacec500ba4f9409e234c8 Mon Sep 17 00:00:00 2001 From: Charles Blake Date: Tue, 11 Dec 2018 14:57:28 -0500 Subject: [PATCH 3/3] For now just implement `resize` per https://github.com/nim-lang/Nim/pull/9922 discussion (with special mremap usage on Linux, but ordinary munmap, mmap on other POSIX). Someone needs to do the when windows branch. --- lib/pure/memfiles.nim | 44 ++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index b742423708..d2beb629a2 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -42,21 +42,6 @@ type else: handle: cint -when defined(windows): - proc fHandle*(m: MemFile): Handle = m.fHandle - proc mapHandle*(m: MemFile): Handle = m.mapHandle - proc wasOpened*(m: MemFile): bool = m.wasOpened - proc `fHandle=`*(m: var MemFile, fHandle: Handle) = - m.fHandle = fHandle - proc `mapHandle=`*(m: var MemFile, mapHandle: Handle) = - m.mapHandle = mapHandle - proc `wasOpened=`*(m: var MemFile, wasOpened: bool) = - m.wasOpened = wasOpened -else: - proc handle*(m: MemFile): cint = m.handle - proc `handle=`*(m: var MemFile, handle: cint) = - m.handle = handle - proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = ## returns a pointer to a mapped portion of MemFile `m` @@ -296,6 +281,35 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") +proc resize*(f: var MemFile, newFileSize: int) = + ## resize and re-map the file underlying an ``allowRemap MemFile``. NOTE: + ## this assumes the entire file is mapped read-write at offset zero. Also, + ## the value of ``.mem`` will probably change. + when defined(windows): + discard #TODO This needs to be implemented. + else: + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if ftruncate(f.handle, newFileSize) == -1: + raiseOSError(osLastError()) + when defined(linux): #Maybe NetBSD, too? + #On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize,newSize: csize; flags: cint): pointer {. + importc: "mremap", header: "" .} + let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1)) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + MAP_SHARED or MAP_POPULATE, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize + proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the ## file system, if `f` was opened with write access.