mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-01-10 06:54:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add MemMapFileStream. Fixes in memFiles. (#7944)

Browse Source 
* Add MemMapFileStream

* Added tests

* Fixed bug in memfiles (zero index for string)

* Added flush to changelog

* Attempt to fix Win's nuances

* Fix attempt to fix

* Continue...

* And again...

* Reworked tests (all for win on Win)

* Fixes in flush (Win)

* Replace fn vars to consts

* Added the attempts parameter to the flush

* Replace while to for

* Move to memfiles

* Use Natural instead of uint

* Better error messages for append mode. Handle specific cases.
This commit is contained in:
Dmitry Atamanov
2018-06-14 19:34:26 +03:00
committed by Dominik Picheta
parent f1d5e9090e
commit bf5d619a52
5 changed files with 195 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
changelog.md
View File

@@ -78,6 +78,8 @@
- Added the proc ``math.prod`` for product of elements in openArray.
- Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
- ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``
- Added the proc ``flush`` for memory mapped files.
- Added the ``MemMapFileStream``.
### Library changes

118
lib/pure/memfiles.nim
View File

@@ -22,7 +22,11 @@ elif defined(posix):
else:
{.error: "the memfiles module is not supported on your operating system!".}
import os
import os, streams
proc newEIO(msg: string): ref IOError =
new(result)
result.msg = msg
type
MemFile* = object ## represents a memory mapped file
@@ -44,11 +48,14 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
##
## ``mappedSize`` of ``-1`` maps to the whole file, and
## ``offset`` must be multiples of the PAGE SIZE of your OS
if mode == fmAppend:
raise newEIO("The append mode is not supported.")
var readonly = mode == fmRead
when defined(windows):
result = mapViewOfFileEx(
m.mapHandle,
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
int32(offset shr 32),
int32(offset and 0xffffffff),
if mappedSize == -1: 0 else: mappedSize,
@@ -113,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead,
## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
# The file can be resized only when write mode is used:
if mode == fmAppend:
raise newEIO("The append mode is not supported.")
assert newFileSize == -1 or mode != fmRead
var readonly = mode == fmRead
@@ -121,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead,
result.size = 0
when defined(windows):
let desiredAccess = GENERIC_READ
let shareMode = FILE_SHARE_READ
let flags = FILE_FLAG_RANDOM_ACCESS
template fail(errCode: OSErrorCode, msg: untyped) =
rollback()
if result.fHandle != 0: discard closeHandle(result.fHandle)
@@ -133,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead,
winApiProc(
filename,
# GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE,
FILE_SHARE_READ,
if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE,
if readonly: shareMode else: shareMode or FILE_SHARE_WRITE,
nil,
if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags,
0)
when useWinUnicode:
@@ -172,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
result.mem = mapViewOfFileEx(
result.mapHandle,
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
int32(offset shr 32),
int32(offset and 0xffffffff),
if mappedSize == -1: 0 else: mappedSize,
@@ -245,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead,
if close(result.handle) == 0:
result.handle = -1
proc flush*(f: var MemFile; attempts: Natural = 3) =
## Flushes `f`'s buffer for the number of attempts equal to `attempts`.
## If were errors an exception `OSError` will be raised.
var res = false
var lastErr: OSErrorCode
when defined(windows):
for i in 1..attempts:
res = flushViewOfFile(f.mem, 0) != 0
if res:
break
lastErr = osLastError()
if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
raiseOSError(lastErr)
else:
for i in 1..attempts:
res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
if res:
break
lastErr = osLastError()
if lastErr != EBUSY.OSErrorCode:
raiseOSError(lastErr, "error flushing mapping")
proc close*(f: var MemFile) =
## closes the memory mapped file `f`. All changes are written back to the
## file system, if `f` was opened with write access.
@@ -362,9 +398,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
## echo line
for ms in memSlices(mfile, delim, eat):
buf.setLen(ms.size)
copyMem(addr(buf[0]), ms.data, ms.size)
buf[ms.size] = '\0'
setLen(buf.string, ms.size)
copyMem(buf.cstring, ms.data, ms.size)
yield buf
iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
@@ -382,3 +417,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
var buf = TaintedString(newStringOfCap(80))
for line in lines(mfile, buf, delim, eat):
yield buf
type
MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile`
MemMapFileStreamObj* = object of Stream
mf: MemFile
mode: FileMode
pos: ByteAddress
proc mmsClose(s: Stream) =
MemMapFileStream(s).pos = -1
close(MemMapFileStream(s).mf)
proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf)
proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or
(MemMapFileStream(s).pos < 0)
proc mmsSetPosition(s: Stream, pos: int) =
if pos > MemMapFileStream(s).mf.size or pos < 0:
raise newEIO("cannot set pos in stream")
MemMapFileStream(s).pos = pos
proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos
proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem)
let p = cast[ByteAddress](MemMapFileStream(s).pos)
let l = min(bufLen, MemMapFileStream(s).mf.size - p)
moveMem(buffer, cast[pointer](startAddress + p), l)
result = l
proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int =
result = mmsPeekData(s, buffer, bufLen)
inc(MemMapFileStream(s).pos, result)
proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) =
if MemMapFileStream(s).mode == fmRead:
raise newEIO("cannot write to read-only stream")
let size = MemMapFileStream(s).mf.size
if MemMapFileStream(s).pos + bufLen > size:
raise newEIO("cannot write to stream")
let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) +
cast[ByteAddress](MemMapFileStream(s).pos)
moveMem(cast[pointer](p), buffer, bufLen)
inc(MemMapFileStream(s).pos, bufLen)
proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1):
MemMapFileStream =
## creates a new stream from the file named `filename` with the mode `mode`.
## Raises ## `EOS` if the file cannot be opened. See the `system
## <system.html>`_ module for a list of available FileMode enums.
## ``fileSize`` can only be set if the file does not exist and is opened
## with write access (e.g., with fmReadWrite).
var mf: MemFile = open(filename, mode, newFileSize = fileSize)
new(result)
result.mode = mode
result.mf = mf
result.closeImpl = mmsClose
result.atEndImpl = mmsAtEnd
result.setPositionImpl = mmsSetPosition
result.getPositionImpl = mmsGetPosition
result.readDataImpl = mmsReadData
result.peekDataImpl = mmsPeekData
result.writeDataImpl = mmsWriteData
result.flushImpl = mmsFlush

42
lib/windows/winlean.nim
View File

@@ -129,7 +129,6 @@ const
PIPE_ACCESS_OUTBOUND* = 2'i32
PIPE_NOWAIT* = 0x00000001'i32
SYNCHRONIZE* = 0x00100000'i32
FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
CREATE_NO_WINDOW* = 0x08000000'i32
@@ -281,15 +280,31 @@ else:
importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.}
const
FILE_ATTRIBUTE_ARCHIVE* = 32'i32
FILE_ATTRIBUTE_COMPRESSED* = 2048'i32
FILE_ATTRIBUTE_NORMAL* = 128'i32
FILE_ATTRIBUTE_DIRECTORY* = 16'i32
FILE_ATTRIBUTE_HIDDEN* = 2'i32
FILE_ATTRIBUTE_READONLY* = 1'i32
FILE_ATTRIBUTE_REPARSE_POINT* = 1024'i32
FILE_ATTRIBUTE_SYSTEM* = 4'i32
FILE_ATTRIBUTE_TEMPORARY* = 256'i32
FILE_ATTRIBUTE_READONLY* = 0x00000001'i32
FILE_ATTRIBUTE_HIDDEN* = 0x00000002'i32
FILE_ATTRIBUTE_SYSTEM* = 0x00000004'i32
FILE_ATTRIBUTE_DIRECTORY* = 0x00000010'i32
FILE_ATTRIBUTE_ARCHIVE* = 0x00000020'i32
FILE_ATTRIBUTE_DEVICE* = 0x00000040'i32
FILE_ATTRIBUTE_NORMAL* = 0x00000080'i32
FILE_ATTRIBUTE_TEMPORARY* = 0x00000100'i32
FILE_ATTRIBUTE_SPARSE_FILE* = 0x00000200'i32
FILE_ATTRIBUTE_REPARSE_POINT* = 0x00000400'i32
FILE_ATTRIBUTE_COMPRESSED* = 0x00000800'i32
FILE_ATTRIBUTE_OFFLINE* = 0x00001000'i32
FILE_ATTRIBUTE_NOT_CONTENT_INDEXED* = 0x00002000'i32
FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'i32
FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'i32
FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
FILE_FLAG_POSIX_SEMANTICS* = 0x01000000'i32
FILE_FLAG_BACKUP_SEMANTICS* = 0x02000000'i32
FILE_FLAG_DELETE_ON_CLOSE* = 0x04000000'i32
FILE_FLAG_SEQUENTIAL_SCAN* = 0x08000000'i32
FILE_FLAG_RANDOM_ACCESS* = 0x10000000'i32
FILE_FLAG_NO_BUFFERING* = 0x20000000'i32
FILE_FLAG_OVERLAPPED* = 0x40000000'i32
FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
MAX_PATH* = 260
@@ -683,8 +698,6 @@ const
FILE_MAP_WRITE* = 2'i32
INVALID_FILE_SIZE* = -1'i32
FILE_FLAG_BACKUP_SEMANTICS* = 33554432'i32
FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
DUPLICATE_SAME_ACCESS* = 2
FILE_READ_DATA* = 0x00000001 # file & pipe
FILE_WRITE_DATA* = 0x00000002 # file & pipe
@@ -695,6 +708,7 @@ const
ERROR_PATH_NOT_FOUND* = 3
ERROR_ACCESS_DENIED* = 5
ERROR_NO_MORE_FILES* = 18
ERROR_LOCK_VIOLATION* = 33
ERROR_HANDLE_EOF* = 38
ERROR_BAD_ARGUMENTS* = 165
@@ -763,6 +777,9 @@ when not useWinUnicode:
proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
dynlib: "kernel32", importc: "UnmapViewOfFile".}
proc flushViewOfFile*(lpBaseAddress: pointer, dwNumberOfBytesToFlush: DWORD): WINBOOL {.
stdcall, dynlib: "kernel32", importc: "FlushViewOfFile".}
type
OVERLAPPED* {.pure, inheritable.} = object
internal*: PULONG
@@ -785,7 +802,6 @@ type
const
ERROR_IO_PENDING* = 997 # a.k.a WSA_IO_PENDING
FILE_FLAG_OVERLAPPED* = 1073741824
WSAECONNABORTED* = 10053
WSAEADDRINUSE* = 10048
WSAECONNRESET* = 10054

3
tests/stdlib/tmemfiles2.nim
View File

@@ -4,9 +4,10 @@ discard """
Half read size: 10 Data: Hello'''
"""
import memfiles, os
const
fn = "test.mmap"
var
mm, mm_full, mm_half: MemFile
fn = "test.mmap"
p: pointer
if fileExists(fn): removeFile(fn)

53
tests/stdlib/tmemmapstreams.nim Normal file
View File

@@ -0,0 +1,53 @@
discard """
file: "tmemmapstreams.nim"
output: '''Created size: 10
Position after writing: 5
Position after writing one char: 6
Peeked data: Hello
Position after peeking: 0
Readed data: Hello!
Position after reading line: 7
Position after setting position: 6
Readed line: Hello!
Position after reading line: 7'''
"""
import os, streams, memfiles
const
fn = "test.mmapstream"
var
mms: MemMapFileStream
if fileExists(fn): removeFile(fn)
# Create a new memory mapped file, data all zeros
mms = newMemMapFileStream(fn, mode = fmReadWrite, fileSize = 10)
mms.close()
if fileExists(fn): echo "Created size: ", getFileSize(fn)
# write, flush, peek, read
mms = newMemMapFileStream(fn, mode = fmReadWrite)
let s = "Hello"
mms.write(s)
mms.flush
echo "Position after writing: ", mms.getPosition()
mms.write('!')
mms.flush
echo "Position after writing one char: ", mms.getPosition()
mms.close()
mms = newMemMapFileStream(fn, mode = fmRead)
echo "Peeked data: ", mms.peekStr(s.len)
echo "Position after peeking: ", mms.getPosition()
echo "Readed data: ", mms.readLine
echo "Position after reading line: ", mms.getPosition()
mms.setPosition(mms.getPosition() - 1)
echo "Position after setting position: ", mms.getPosition()
mms.setPosition(0)
echo "Readed line: ", mms.readLine
echo "Position after reading line: ", mms.getPosition()
mms.close()
if fileExists(fn): removeFile(fn)