From dc08a033d910aea19fde0f59c9d1a5e70d7efc15 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 5 Nov 2011 12:16:49 +0100 Subject: [PATCH] memfiles now uses winlean; changed the interface to raise EOS --- compiler/rodread.nim | 5 +- doc/lib.txt | 4 ++ lib/pure/collections/sequtils.nim | 3 + lib/pure/memfiles.nim | 113 +++++++++++++----------------- lib/pure/unittest.nim | 5 +- lib/windows/winlean.nim | 60 +++++++++++++++- web/news.txt | 1 + web/nimrod.ini | 2 +- 8 files changed, 123 insertions(+), 70 deletions(-) diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 0cc91140d2..900361ed7c 100755 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -632,7 +632,10 @@ proc newRodReader(modfilename: string, crc: TCrc32, r.readerIndex = readerIndex r.filename = modfilename InitIdTable(r.syms) - if not open(r.memFile, modfilename): return nil + try: + r.memFile = memfiles.open(modfilename) + except EOS: + return nil # we terminate the file explicitely with ``\0``, so the cast to `cstring` # is save: r.s = cast[cstring](r.memFile.mem) diff --git a/doc/lib.txt b/doc/lib.txt index 556559e3b8..775223fc16 100755 --- a/doc/lib.txt +++ b/doc/lib.txt @@ -141,6 +141,10 @@ Generic Operating System Services (also called *console*). The implementation simply uses ANSI escape sequences and does not depend on any other module. +* `memfiles `_ + This module provides support for memory mapped files (Posix's ``mmap``) + on the different operating systems. + Math libraries -------------- diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 59d0d2658a..b7a57ac0c4 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -11,6 +11,9 @@ ## ## This module implements operations for the built-in `seq`:idx: type ## which were inspired by functional programming languages. +## +## **Note**: This interface will change as soon as the compiler supports +## closures and proper coroutines. proc concat*[T](seqs: openarray[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside of one sequence. diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 7fb73186a5..9dd93fc5fd 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -1,39 +1,32 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2011 Andreas Rumpf +# (c) Copyright 2011 Nimrod Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # +## :Authors: Zahary Karadjov, Andreas Rumpf +## ## This module provides support for `memory mapped files`:idx: ## (Posix's `mmap`:idx:) on the different operating systems. when defined(windows): - import windows + import winlean elif defined(posix): import posix else: - {.error: "the memfiles module is not supported yet on your operating system!".} + {.error: "the memfiles module is not supported on your operating system!".} -## mem -## a pointer to the memory mapped file `f`. The pointer can be -## used directly to change the contents of the file, if `f` was opened -## with write access. - -## size -## size of the memory mapped file `f`. - -when defined(windows): - type Tsize = int64 -else: - type Tsize = int +import os type TMemFile* = object {.pure.} ## represents a memory mapped file - mem*: pointer # XXX: The compiler won't let me add comments on the next line - size*: Tsize + mem*: pointer ## a pointer to the memory mapped file. The pointer + ## can be used directly to change the contents of the + ## file, if it was opened with write access. + size*: int ## size of the memory mapped file when defined(windows): fHandle: int @@ -41,32 +34,31 @@ type else: handle: cint -proc open*( - f : var TMemFile, - filename : string, - mode : TFileMode = fmRead, - mappedSize : int = -1, - offset : Tsize = 0, - newFileSize : Tsize = -1 ): bool = - ## open a memory mapped file `f`. Returns true for success. +proc open*(filename: string, mode: TFileMode = fmRead, + mappedSize = -1, offset = 0, newFileSize = -1): TMemFile = + ## opens a memory mapped file. If this fails, ``EOS`` is raised. + ## `newFileSize` can only be set if the file is not opened with ``fmRead`` + ## access. `mappedSize` and `offset` can be used to map only a slice of + ## the file. - # The file can be resized only when write mode is used + # The file can be resized only when write mode is used: assert newFileSize == -1 or mode != fmRead var readonly = mode == fmRead template rollback = - f.mem = nil - f.size = 0 + result.mem = nil + result.size = 0 when defined(windows): template fail(msg: expr) = rollback() - if f.fHandle != 0: discard CloseHandle(f.fHandle) - if f.mapHandle != 0: discard CloseHandle(f.mapHandle) + if result.fHandle != 0: discard CloseHandle(result.fHandle) + if result.mapHandle != 0: discard CloseHandle(result.mapHandle) + OSError() # return false - raise newException(EIO, msg) + #raise newException(EIO, msg) - f.fHandle = CreateFileA( + result.fHandle = CreateFileA( filename, if readonly: GENERIC_READ else: GENERIC_ALL, FILE_SHARE_READ, @@ -75,7 +67,7 @@ proc open*( if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY, 0) - if f.fHandle == INVALID_HANDLE_VALUE: + if result.fHandle == INVALID_HANDLE_VALUE: fail "error opening file" if newFileSize != -1: @@ -83,87 +75,83 @@ proc open*( sizeHigh = int32(newFileSize shr 32) sizeLow = int32(newFileSize and 0xffffffff) - var status = SetFilePointer(f.fHandle, sizeLow, addr(sizeHigh), FILE_BEGIN) + var status = SetFilePointer(result.fHandle, sizeLow, addr(sizeHigh), + FILE_BEGIN) if (status == INVALID_SET_FILE_POINTER and GetLastError() != NO_ERROR) or - (SetEndOfFile(f.fHandle) == 0): + (SetEndOfFile(result.fHandle) == 0): fail "error setting file size" - f.mapHandle = CreateFileMapping( - f.fHandle, nil, + result.mapHandle = CreateFileMapping( + result.fHandle, nil, if readonly: PAGE_READONLY else: PAGE_READWRITE, 0, 0, nil) - if f.mapHandle == 0: + if result.mapHandle == 0: fail "error creating mapping" - f.mem = MapViewOfFileEx( - f.mapHandle, + result.mem = MapViewOfFileEx( + result.mapHandle, if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, int32(offset shr 32), int32(offset and 0xffffffff), if mappedSize == -1: 0 else: mappedSize, nil) - if f.mem == nil: + if result.mem == nil: fail "error mapping view" var hi, low: int32 - low = GetFileSize(f.fHandle, addr(hi)) + low = GetFileSize(result.fHandle, addr(hi)) if low == INVALID_FILE_SIZE: fail "error getting file size" else: var fileSize = (int64(hi) shr 32) or low - f.size = if mappedSize != -1: min(fileSize, mappedSize) else: fileSize - - result = true + if mappedSize != -1: result.size = min(fileSize, mappedSize).int + else: result.size = fileSize.int else: template fail(msg: expr) = rollback() - if f.handle != 0: - discard close(f.handle) - # return false - raise newException(system.EIO, msg) + if result.handle != 0: discard close(result.handle) + OSError() var flags = if readonly: O_RDONLY else: O_RDWR if newFileSize != -1: flags = flags or O_CREAT or O_TRUNC - f.handle = open(filename, flags) - if f.handle == -1: + result.handle = open(filename, flags) + if result.handle == -1: # XXX: errno is supposed to be set here # Is there an exception that wraps it? fail "error opening file" if newFileSize != -1: - if ftruncate(f.handle, newFileSize) == -1: + if ftruncate(result.handle, newFileSize) == -1: fail "error setting file size" if mappedSize != -1: - f.size = mappedSize + result.size = mappedSize else: var stat: Tstat - if fstat(f.handle, stat) != -1: + if fstat(result.handle, stat) != -1: # XXX: Hmm, this could be unsafe # Why is mmap taking int anyway? - f.size = int(stat.st_size) + result.size = int(stat.st_size) else: fail "error getting file size" - f.mem = mmap( + result.mem = mmap( nil, - f.size, + result.size, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, if readonly: MAP_PRIVATE else: MAP_SHARED, - f.handle, + result.handle, offset) - if f.mem == cast[pointer](MAP_FAILED): + if result.mem == cast[pointer](MAP_FAILED): fail "file mapping failed" - result = true - proc close*(f: var TMemFile) = ## closes the memory mapped file `f`. All changes are written back to the ## file system, if `f` was opened with write access. @@ -189,6 +177,5 @@ proc close*(f: var TMemFile) = else: f.handle = 0 - if error: - raise newException(system.EIO, "error closing file") + if error: OSError() diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 7f70811b02..87c2b6ed77 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -7,15 +7,14 @@ # distribution, for details about the copyright. # +## :Author: Zahary Karadjov (zah@github) +## ## This module implements the standard unit testing facilities such as ## suites, fixtures and test cases as well as facilities for combinatorial ## and randomzied test case generation (not yet available) ## and object mocking (not yet available) ## ## It is loosely based on C++'s boost.test and Haskell's QuickTest -## -## Maintainer: Zahary Karadjov (zah@github) -## import macros, terminal diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index a997b0e1cc..18e287bfa1 100755 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -12,6 +12,7 @@ type THandle* = int + LONG* = int WINBOOL* = int32 DWORD* = int32 @@ -65,6 +66,7 @@ const DETACHED_PROCESS* = 8'i32 SW_SHOWNORMAL* = 1'i32 + INVALID_HANDLE_VALUE* = THANDLE(-1) proc CloseHandle*(hObject: THANDLE): WINBOOL {.stdcall, dynlib: "kernel32", importc: "CloseHandle".} @@ -143,6 +145,7 @@ const FILE_ATTRIBUTE_HIDDEN* = 2'i32 FILE_ATTRIBUTE_READONLY* = 1'i32 FILE_ATTRIBUTE_SYSTEM* = 4'i32 + FILE_ATTRIBUTE_TEMPORARY* = 256'i32 MAX_PATH* = 260 type @@ -239,7 +242,8 @@ type TInAddr* {.pure, final, importc: "IN_ADDR", header: "Winsock2.h".} = object s_addr*: int32 # IP address - Tsockaddr_in* {.pure, final, importc: "SOCKADDR_IN", header: "Winsock2.h".} = object + Tsockaddr_in* {.pure, final, importc: "SOCKADDR_IN", + header: "Winsock2.h".} = object sin_family*: int16 sin_port*: int16 # unsigned sin_addr*: TInAddr @@ -248,7 +252,8 @@ type Tin6_addr* {.pure, final, importc: "IN6_ADDR", header: "Winsock2.h".} = object bytes*: array[0..15, char] - Tsockaddr_in6* {.pure, final, importc: "SOCKADDR_IN6", header: "Winsock2.h".} = object + Tsockaddr_in6* {.pure, final, importc: "SOCKADDR_IN6", + header: "Winsock2.h".} = object sin6_family*: int16 sin6_port*: int16 # unsigned sin6_flowinfo*: int32 # unsigned @@ -397,3 +402,54 @@ type proc WaitForMultipleObjects*(nCount: DWORD, lpHandles: PWOHandleArray, bWaitAll: WINBOOL, dwMilliseconds: DWORD): DWORD{. stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + +# for memfiles.nim: + +const + GENERIC_READ* = 0x80000000'i32 + GENERIC_ALL* = 0x10000000'i32 + FILE_SHARE_READ* = 1'i32 + CREATE_ALWAYS* = 2'i32 + OPEN_EXISTING* = 3'i32 + FILE_BEGIN* = 0'i32 + INVALID_SET_FILE_POINTER* = -1'i32 + NO_ERROR* = 0'i32 + PAGE_READONLY* = 2'i32 + PAGE_READWRITE* = 4'i32 + FILE_MAP_READ* = 4'i32 + FILE_MAP_WRITE* = 2'i32 + INVALID_FILE_SIZE* = -1'i32 + +proc CreateFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, + lpSecurityAttributes: pointer, + dwCreationDisposition, dwFlagsAndAttributes: DWORD, + hTemplateFile: THANDLE): THANDLE {. + stdcall, dynlib: "kernel32", importc: "CreateFileA".} + +proc SetEndOfFile*(hFile: THANDLE): WINBOOL {.stdcall, dynlib: "kernel32", + importc: "SetEndOfFile".} + +proc SetFilePointer*(hFile: THANDLE, lDistanceToMove: LONG, + lpDistanceToMoveHigh: ptr LONG, + dwMoveMethod: DWORD): DWORD {. + stdcall, dynlib: "kernel32", importc: "SetFilePointer".} + +proc GetFileSize*(hFile: THANDLE, lpFileSizeHigh: ptr DWORD): DWORD{.stdcall, + dynlib: "kernel32", importc: "GetFileSize".} + +proc MapViewOfFileEx*(hFileMappingObject: THANDLE, dwDesiredAccess: DWORD, + dwFileOffsetHigh, dwFileOffsetLow: DWORD, + dwNumberOfBytesToMap: DWORD, + lpBaseAddress: pointer): pointer{. + stdcall, dynlib: "kernel32", importc: "MapViewOfFileEx".} + +proc CreateFileMapping*(hFile: THANDLE, + lpFileMappingAttributes: pointer, + flProtect, dwMaximumSizeHigh: DWORD, + dwMaximumSizeLow: DWORD, lpName: cstring): THANDLE {. + stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} + +proc UnmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, + dynlib: "kernel32", importc: "UnmapViewOfFile".} + diff --git a/web/news.txt b/web/news.txt index 46c930fdcf..7dc272d98c 100755 --- a/web/news.txt +++ b/web/news.txt @@ -106,6 +106,7 @@ Library Additions - Added ``locks`` core module for more flexible locking support. - Added ``irc`` module. - Added ``ftpclient`` module. +- Added ``memfiles`` module. 2011-07-10 Version 0.8.12 released diff --git a/web/nimrod.ini b/web/nimrod.ini index d1acd708be..25d1e2e6a2 100755 --- a/web/nimrod.ini +++ b/web/nimrod.ini @@ -42,7 +42,7 @@ srcdoc: "impure/rdstdin;wrappers/zmq;wrappers/sphinx" srcdoc: "pure/collections/tables;pure/collections/sets;pure/collections/lists" srcdoc: "pure/collections/intsets;pure/collections/queues;pure/encodings" srcdoc: "pure/events;pure/collections/sequtils;pure/irc;ecmas/dom" -srcdoc: "pure/ftpclient" +srcdoc: "pure/ftpclient;pure/memfiles" webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup" webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc"