From 272fd42c970d48d526f0d5ab8dc4fef4d8f177e7 Mon Sep 17 00:00:00 2001 From: Nick Greenfield Date: Wed, 23 Jul 2014 11:25:21 -0700 Subject: [PATCH 1/4] Add automatic MAP_POPULATE flag for opening read_only (MAP_PRIVATE) and shared (MAP_SHARED) mmap files. --- lib/posix/posix.nim | 2 ++ lib/pure/memfiles.nim | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 8e66336c2e..17bcc420ed 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1375,6 +1375,8 @@ var ## Share changes. MAP_PRIVATE* {.importc, header: "".}: cint ## Changes are private. + MAP_POPULATE* {.importc, header: "".}: cint + ## Populate (prefault) page tables for a mapping. MAP_FIXED* {.importc, header: "".}: cint ## Interpret addr exactly. MS_ASYNC* {.importc, header: "".}: cint diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 31fefc6c82..ffeb0beffe 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -54,7 +54,7 @@ proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead, nil, mappedSize, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: MAP_PRIVATE else: MAP_SHARED, + if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), m.handle, offset) if result == cast[pointer](MAP_FAILED): osError(osLastError()) @@ -207,7 +207,7 @@ proc open*(filename: string, mode: TFileMode = fmRead, nil, result.size, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: MAP_PRIVATE else: MAP_SHARED, + if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), result.handle, offset) From cba75db4e36b3af0d39b5717141a971e6d558554 Mon Sep 17 00:00:00 2001 From: Nick Greenfield Date: Wed, 23 Jul 2014 11:31:03 -0700 Subject: [PATCH 2/4] Do not automatically use MAP_POPULATE for opening mmap files. Adds use_map_populate keyword to memfiles.open and memfiles.mapMem to govern MAP_POPULATE use. This is set to false by default. --- lib/pure/memfiles.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index ffeb0beffe..06f2c215e4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -30,14 +30,17 @@ type when defined(windows): fHandle: int - mapHandle: int + mapHandle: int else: handle: cint proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead, - mappedSize = -1, offset = 0): pointer = + mappedSize = -1, offset = 0, + use_map_populate = false): pointer = var readonly = mode == fmRead + if not use_map_populate: + MAP_POPULATE = 0 when defined(windows): result = mapViewOfFileEx( m.mapHandle, @@ -72,7 +75,8 @@ proc unmapMem*(f: var TMemFile, p: pointer, size: int) = proc open*(filename: string, mode: TFileMode = fmRead, - mappedSize = -1, offset = 0, newFileSize = -1): TMemFile = + mappedSize = -1, offset = 0, newFileSize = -1, + use_map_populate = false): TMemFile = ## opens a memory mapped file. If this fails, ``EOS`` is raised. ## `newFileSize` can only be set if the file does not exist and is opened ## with write access (e.g., with fmReadWrite). `mappedSize` and `offset` @@ -94,6 +98,8 @@ proc open*(filename: string, mode: TFileMode = fmRead, # The file can be resized only when write mode is used: assert newFileSize == -1 or mode != fmRead var readonly = mode == fmRead + if not use_map_populate: + MAP_POPULATE = 0 template rollback = result.mem = nil From 3a57052e57a6e6d2a7b4b6fa8babdd750620d687 Mon Sep 17 00:00:00 2001 From: Nick Greenfield Date: Wed, 23 Jul 2014 11:36:35 -0700 Subject: [PATCH 3/4] Revert "Do not automatically use MAP_POPULATE for opening mmap files." This reverts commit cba75db4e36b3af0d39b5717141a971e6d558554. The prior commit yields a compiler error 'error: lvalue required as left operand of assignment; MAP_POPULATE = ((int) 0);'. So for now this branch *always* adds MAP_POPULATE when using the open and mapMem procs. --- lib/pure/memfiles.nim | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 06f2c215e4..ffeb0beffe 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -30,17 +30,14 @@ type when defined(windows): fHandle: int - mapHandle: int + mapHandle: int else: handle: cint proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead, - mappedSize = -1, offset = 0, - use_map_populate = false): pointer = + mappedSize = -1, offset = 0): pointer = var readonly = mode == fmRead - if not use_map_populate: - MAP_POPULATE = 0 when defined(windows): result = mapViewOfFileEx( m.mapHandle, @@ -75,8 +72,7 @@ proc unmapMem*(f: var TMemFile, p: pointer, size: int) = proc open*(filename: string, mode: TFileMode = fmRead, - mappedSize = -1, offset = 0, newFileSize = -1, - use_map_populate = false): TMemFile = + mappedSize = -1, offset = 0, newFileSize = -1): TMemFile = ## opens a memory mapped file. If this fails, ``EOS`` is raised. ## `newFileSize` can only be set if the file does not exist and is opened ## with write access (e.g., with fmReadWrite). `mappedSize` and `offset` @@ -98,8 +94,6 @@ proc open*(filename: string, mode: TFileMode = fmRead, # The file can be resized only when write mode is used: assert newFileSize == -1 or mode != fmRead var readonly = mode == fmRead - if not use_map_populate: - MAP_POPULATE = 0 template rollback = result.mem = nil From a0df72fcdc8c180b14d4e030c083a6ef1b5320b3 Mon Sep 17 00:00:00 2001 From: Nick Greenfield Date: Wed, 23 Jul 2014 12:06:13 -0700 Subject: [PATCH 4/4] Only try to import MAP_POPULATE on Linux, define flag as 0 otherwise. --- lib/posix/posix.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 17bcc420ed..5bd43fe9cf 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1375,8 +1375,6 @@ var ## Share changes. MAP_PRIVATE* {.importc, header: "".}: cint ## Changes are private. - MAP_POPULATE* {.importc, header: "".}: cint - ## Populate (prefault) page tables for a mapping. MAP_FIXED* {.importc, header: "".}: cint ## Interpret addr exactly. MS_ASYNC* {.importc, header: "".}: cint @@ -1581,6 +1579,16 @@ var MSG_OOB* {.importc, header: "".}: cint ## Out-of-band data. + +when defined(linux): + var + MAP_POPULATE* {.importc, header: "".}: cint + ## Populate (prefault) page tables for a mapping. +else: + var + MAP_POPULATE*: cint = 0 + + when defined(macosx): var MSG_HAVEMORE* {.importc, header: "".}: cint