Merge branch 'devel' of github.com:nim-lang/Nim into devel

This commit is contained in:
Araq
2018-02-09 16:38:53 +01:00
64 changed files with 853 additions and 2036 deletions

View File

@@ -265,9 +265,15 @@ when defined(windows) or defined(nimdoc):
setGlobalDispatcher(newDispatcher())
result = gDisp
proc getIoHandler*(disp: PDispatcher): Handle =
## Returns the underlying IO Completion Port handle (Windows) or selector
## (Unix) for the specified dispatcher.
return disp.ioPort
proc register*(fd: AsyncFD) =
## Registers ``fd`` with the dispatcher.
let p = getGlobalDispatcher()
if createIoCompletionPort(fd.Handle, p.ioPort,
cast[CompletionKey](fd), 1) == 0:
raiseOSError(osLastError())
@@ -757,6 +763,9 @@ when defined(windows) or defined(nimdoc):
## Unregisters ``fd``.
getGlobalDispatcher().handles.excl(fd)
proc contains*(disp: PDispatcher, fd: AsyncFD): bool =
return fd in disp.handles
{.push stackTrace:off.}
proc waitableCallback(param: pointer,
timerOrWaitFired: WINBOOL): void {.stdcall.} =
@@ -977,7 +986,7 @@ when defined(windows) or defined(nimdoc):
proc newAsyncEvent*(): AsyncEvent =
## Creates a new thread-safe ``AsyncEvent`` object.
##
## New ``AsyncEvent`` object is not automatically registered with # TODO: Why? -- DP
## New ``AsyncEvent`` object is not automatically registered with
## dispatcher like ``AsyncSocket``.
var sa = SECURITY_ATTRIBUTES(
nLength: sizeof(SECURITY_ATTRIBUTES).cint,
@@ -1095,6 +1104,9 @@ else:
setGlobalDispatcher(newDispatcher())
result = gDisp
proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] =
return disp.selector
proc register*(fd: AsyncFD) =
let p = getGlobalDispatcher()
var data = newAsyncData()
@@ -1110,6 +1122,9 @@ else:
proc unregister*(ev: AsyncEvent) =
getGlobalDispatcher().selector.unregister(SelectEvent(ev))
proc contains*(disp: PDispatcher, fd: AsyncFd): bool =
return fd.SocketHandle in disp.selector
proc addRead*(fd: AsyncFD, cb: Callback) =
let p = getGlobalDispatcher()

View File

@@ -85,7 +85,7 @@ proc newAsyncFile*(fd: AsyncFd): AsyncFile =
## Creates `AsyncFile` with a previously opened file descriptor `fd`.
new result
result.fd = fd
register(result.fd)
register(fd)
proc openAsync*(filename: string, mode = fmRead): AsyncFile =
## Opens a file specified by the path in ``filename`` using
@@ -97,16 +97,16 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile =
when useWinUnicode:
let fd = createFileW(newWideCString(filename), desiredAccess,
FILE_SHARE_READ,
nil, creationDisposition, flags, 0).AsyncFd
nil, creationDisposition, flags, 0)
else:
let fd = createFileA(filename, desiredAccess,
FILE_SHARE_READ,
nil, creationDisposition, flags, 0).AsyncFd
nil, creationDisposition, flags, 0)
if fd.Handle == INVALID_HANDLE_VALUE:
if fd == INVALID_HANDLE_VALUE:
raiseOSError(osLastError())
result = newAsyncFile(fd)
result = newAsyncFile(fd.AsyncFd)
if mode == fmAppend:
result.offset = getFileSize(result)
@@ -115,11 +115,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile =
let flags = getPosixFlags(mode)
# RW (Owner), RW (Group), R (Other)
let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH
let fd = open(filename, flags, perm).AsyncFD
if fd.cint == -1:
let fd = open(filename, flags, perm)
if fd == -1:
raiseOSError(osLastError())
result = newAsyncFile(fd)
result = newAsyncFile(fd.AsyncFd)
proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
## Read ``size`` bytes from the specified file asynchronously starting at

View File

@@ -342,6 +342,7 @@ proc asyncCheck*[T](future: Future[T]) =
## finished with an error.
##
## This should be used instead of ``discard`` to discard void futures.
assert(not future.isNil, "Future is nil")
future.callback =
proc () =
if future.failed:

View File

@@ -140,9 +140,16 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET,
sockType: SockType = SOCK_STREAM,
protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket =
## Creates a new ``AsyncSocket`` based on the supplied params.
##
## The supplied ``fd``'s non-blocking state will be enabled implicitly.
##
## **Note**: This procedure will **NOT** register ``fd`` with the global
## async dispatcher. You need to do this manually. If you have used
## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered.
assert fd != osInvalidSocket.AsyncFD
new(result)
result.fd = fd.SocketHandle
fd.SocketHandle.setBlocking(false)
result.isBuffered = buffered
result.domain = domain
result.sockType = sockType

View File

@@ -141,7 +141,7 @@ template checkFd(s, f) =
if f >= s.maxFD:
raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event], data: T) =
let fdi = int(fd)
s.checkFd(fdi)
@@ -156,7 +156,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
raiseIOSelectorsError(osLastError())
inc(s.count)
proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) =
proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) =
let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode,
Event.User, Event.Oneshot, Event.Error}
let fdi = int(fd)
@@ -392,9 +392,19 @@ proc selectInto*[T](s: Selector[T], timeout: int,
let pevents = resTable[i].events
var pkey = addr(s.fds[fdi])
doAssert(pkey.ident != 0)
var rkey = ReadyKey(fd: int(fdi), events: {})
var rkey = ReadyKey(fd: fdi, events: {})
if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0:
if (pevents and EPOLLHUP) != 0:
rkey.errorCode = ECONNRESET.OSErrorCode
else:
# Try reading SO_ERROR from fd.
var error: cint
var size = sizeof(error).SockLen
if getsockopt(fdi.SocketHandle, SOL_SOCKET, SO_ERROR, addr(error),
addr(size)) == 0'i32:
rkey.errorCode = error.OSErrorCode
rkey.events.incl(Event.Error)
if (pevents and EPOLLOUT) != 0:
rkey.events.incl(Event.Write)
@@ -482,7 +492,7 @@ template isEmpty*[T](s: Selector[T]): bool =
(s.count == 0)
proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} =
return s.fds[fd].ident != 0
return s.fds[fd.int].ident != 0
proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T =
let fdi = int(fd)
@@ -516,3 +526,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1,
body1
else:
body2
proc getFd*[T](s: Selector[T]): int =
return s.epollFd.int

View File

@@ -217,7 +217,7 @@ else:
raiseIOSelectorsError(osLastError())
s.changes.setLen(0)
proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event], data: T) =
let fdi = int(fd)
s.checkFd(fdi)
@@ -235,7 +235,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
when not declared(CACHE_EVENTS):
flushKQueue(s)
proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event]) =
let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode,
Event.User, Event.Oneshot, Event.Error}
@@ -503,6 +503,7 @@ proc selectInto*[T](s: Selector[T], timeout: int,
if (kevent.flags and EV_ERROR) != 0:
rkey.events = {Event.Error}
rkey.errorCode = kevent.data.OSErrorCode
case kevent.filter:
of EVFILT_READ:
@@ -569,6 +570,13 @@ proc selectInto*[T](s: Selector[T], timeout: int,
doAssert(true, "Unsupported kqueue filter in the queue!")
if (kevent.flags and EV_EOF) != 0:
if kevent.fflags != 0:
rkey.errorCode = kevent.fflags.OSErrorCode
else:
# This assumes we are dealing with sockets.
# TODO: For future-proofing it might be a good idea to give the
# user access to the raw `kevent`.
rkey.errorCode = ECONNRESET.OSErrorCode
rkey.events.incl(Event.Error)
results[k] = rkey
@@ -585,7 +593,7 @@ template isEmpty*[T](s: Selector[T]): bool =
(s.count == 0)
proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} =
return s.fds[fd].ident != 0
return s.fds[fd.int].ident != 0
proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T =
let fdi = int(fd)
@@ -619,3 +627,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1,
body1
else:
body2
proc getFd*[T](s: Selector[T]): int =
return s.kqFD.int

View File

@@ -141,7 +141,7 @@ template checkFd(s, f) =
if f >= s.maxFD:
raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event], data: T) =
var fdi = int(fd)
s.checkFd(fdi)
@@ -149,7 +149,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
setKey(s, fdi, events, 0, data)
if events != {}: s.pollAdd(fdi.cint, events)
proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event]) =
let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode,
Event.User, Event.Oneshot, Event.Error}
@@ -280,7 +280,7 @@ template isEmpty*[T](s: Selector[T]): bool =
(s.count == 0)
proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} =
return s.fds[fd].ident != 0
return s.fds[fd.int].ident != 0
proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T =
let fdi = int(fd)
@@ -314,3 +314,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1,
body1
else:
body2
proc getFd*[T](s: Selector[T]): int =
return -1

View File

@@ -229,7 +229,7 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) =
doAssert(i < FD_SETSIZE,
"Descriptor [" & $int(fd) & "] is not registered in the queue!")
proc registerHandle*[T](s: Selector[T], fd: SocketHandle,
proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event], data: T) =
when not defined(windows):
let fdi = int(fd)
@@ -255,7 +255,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) =
IOFD_SET(ev.rsock, addr s.rSet)
inc(s.count)
proc updateHandle*[T](s: Selector[T], fd: SocketHandle,
proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event]) =
let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode,
Event.User, Event.Oneshot, Event.Error}
@@ -453,3 +453,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value,
else:
body2
proc getFd*[T](s: Selector[T]): int =
return -1

View File

@@ -187,12 +187,12 @@ proc toSockType*(protocol: Protocol): SockType =
proc newNativeSocket*(domain: Domain = AF_INET,
sockType: SockType = SOCK_STREAM,
protocol: Protocol = IPPROTO_TCP): SocketHandle =
## Creates a new socket; returns `InvalidSocket` if an error occurs.
## Creates a new socket; returns `osInvalidSocket` if an error occurs.
socket(toInt(domain), toInt(sockType), toInt(protocol))
proc newNativeSocket*(domain: cint, sockType: cint,
protocol: cint): SocketHandle =
## Creates a new socket; returns `InvalidSocket` if an error occurs.
## Creates a new socket; returns `osInvalidSocket` if an error occurs.
##
## Use this overload if one of the enums specified above does
## not contain what you need.
@@ -666,6 +666,19 @@ proc selectWrite*(writefds: var seq[SocketHandle],
pruneSocketSet(writefds, (wr))
proc accept*(fd: SocketHandle): (SocketHandle, string) =
## Accepts a new client connection.
##
## Returns (osInvalidSocket, "") if an error occurred.
var sockAddress: Sockaddr_in
var addrLen = sizeof(sockAddress).SockLen
var sock = accept(fd, cast[ptr SockAddr](addr(sockAddress)),
addr(addrLen))
if sock == osInvalidSocket:
return (osInvalidSocket, "")
else:
return (sock, $inet_ntoa(sockAddress.sin_addr))
when defined(Windows):
var wsa: WSAData
if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError())

View File

@@ -753,10 +753,8 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string,
## flag is specified then this error will not be raised and instead
## accept will be called again.
assert(client != nil)
var sockAddress: Sockaddr_in
var addrLen = sizeof(sockAddress).SockLen
var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)),
addr(addrLen))
let ret = accept(server.fd)
let sock = ret[0]
if sock == osInvalidSocket:
let err = osLastError()
@@ -764,6 +762,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string,
acceptAddr(server, client, address, flags)
raiseOSError(err)
else:
address = ret[1]
client.fd = sock
client.isBuffered = server.isBuffered
@@ -776,9 +775,6 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string,
let ret = SSLAccept(client.sslHandle)
socketError(client, ret, false)
# Client socket is set above.
address = $inet_ntoa(sockAddress.sin_addr)
when false: #defineSsl:
proc acceptAddrSSL*(server: Socket, client: var Socket,
address: var string): SSLAcceptResult {.

View File

@@ -139,10 +139,15 @@ proc findExe*(exe: string, followSymlinks: bool = true;
## is added the `ExeExts <#ExeExts>`_ file extensions if it has none.
## If the system supports symlinks it also resolves them until it
## meets the actual file. This behavior can be disabled if desired.
for ext in extensions:
result = addFileExt(exe, ext)
if existsFile(result): return
var path = string(getEnv("PATH"))
template checkCurrentDir() =
for ext in extensions:
result = addFileExt(exe, ext)
if existsFile(result): return
when defined(posix):
if '/' in exe: checkCurrentDir()
else:
checkCurrentDir()
let path = string(getEnv("PATH"))
for candidate in split(path, PathSep):
when defined(windows):
var x = (if candidate[0] == '"' and candidate[^1] == '"':
@@ -824,7 +829,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
iterator walkDirRec*(dir: string, yieldFilter = {pcFile},
followFilter = {pcDir}): string {.tags: [ReadDirEffect].} =
## Recursively walks over the directory `dir` and yields for each file
## Recursively walks over the directory `dir` and yields for each file
## or directory in `dir`.
## The full path for each file or directory is returned.
## **Warning**:

View File

@@ -1,195 +1,6 @@
#
#
# The Nim Compiler
# (c) Copyright 2015 Nim Contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
import strutils
const Sha1DigestSize = 20
type
Sha1Digest = array[0 .. Sha1DigestSize-1, uint8]
SecureHash* = distinct Sha1Digest
# Copyright (c) 2011, Micael Hildenborg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Micael Hildenborg nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Ported to Nim by Erik O'Leary
type
Sha1State* = array[0 .. 5-1, uint32]
Sha1Buffer = array[0 .. 80-1, uint32]
template clearBuffer(w: Sha1Buffer, len = 16) =
zeroMem(addr(w), len * sizeof(uint32))
proc init*(result: var Sha1State) =
result[0] = 0x67452301'u32
result[1] = 0xefcdab89'u32
result[2] = 0x98badcfe'u32
result[3] = 0x10325476'u32
result[4] = 0xc3d2e1f0'u32
proc innerHash(state: var Sha1State, w: var Sha1Buffer) =
var
a = state[0]
b = state[1]
c = state[2]
d = state[3]
e = state[4]
var round = 0
template rot(value, bits: uint32): uint32 =
(value shl bits) or (value shr (32 - bits))
template sha1(fun, val: uint32) =
let t = rot(a, 5) + fun + e + val + w[round]
e = d
d = c
c = rot(b, 30)
b = a
a = t
template process(body: untyped) =
w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1)
body
inc(round)
template wrap(dest, value: untyped) =
let v = dest + value
dest = v
while round < 16:
sha1((b and c) or (not b and d), 0x5a827999'u32)
inc(round)
while round < 20:
process:
sha1((b and c) or (not b and d), 0x5a827999'u32)
while round < 40:
process:
sha1(b xor c xor d, 0x6ed9eba1'u32)
while round < 60:
process:
sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32)
while round < 80:
process:
sha1(b xor c xor d, 0xca62c1d6'u32)
wrap state[0], a
wrap state[1], b
wrap state[2], c
wrap state[3], d
wrap state[4], e
proc sha1(src: cstring; len: int): Sha1Digest =
#Initialize state
var state: Sha1State
init(state)
#Create w buffer
var w: Sha1Buffer
#Loop through all complete 64byte blocks.
let byteLen = len
let endOfFullBlocks = byteLen - 64
var endCurrentBlock = 0
var currentBlock = 0
while currentBlock <= endOfFullBlocks:
endCurrentBlock = currentBlock + 64
var i = 0
while currentBlock < endCurrentBlock:
w[i] = uint32(src[currentBlock+3]) or
uint32(src[currentBlock+2]) shl 8'u32 or
uint32(src[currentBlock+1]) shl 16'u32 or
uint32(src[currentBlock]) shl 24'u32
currentBlock += 4
inc(i)
innerHash(state, w)
#Handle last and not full 64 byte block if existing
endCurrentBlock = byteLen - currentBlock
clearBuffer(w)
var lastBlockBytes = 0
while lastBlockBytes < endCurrentBlock:
var value = uint32(src[lastBlockBytes + currentBlock]) shl
((3'u32 - uint32(lastBlockBytes and 3)) shl 3)
w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value
inc(lastBlockBytes)
w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or (
0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3)
)
if endCurrentBlock >= 56:
innerHash(state, w)
clearBuffer(w)
w[15] = uint32(byteLen) shl 3
innerHash(state, w)
# Store hash in result pointer, and make sure we get in in the correct order
# on both endian models.
for i in 0 .. Sha1DigestSize-1:
result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255)
proc sha1(src: string): Sha1Digest =
## Calculate SHA1 from input string
sha1(src, src.len)
proc secureHash*(str: string): SecureHash = SecureHash(sha1(str))
proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename))
proc `$`*(self: SecureHash): string =
result = ""
for v in Sha1Digest(self):
result.add(toHex(int(v), 2))
proc parseSecureHash*(hash: string): SecureHash =
for i in 0 ..< Sha1DigestSize:
Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1]))
proc `==`*(a, b: SecureHash): bool =
# Not a constant-time comparison, but that's acceptable in this context
Sha1Digest(a) == Sha1Digest(b)
when isMainModule:
let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]")
doAssert hash1 == hash1
doAssert parseSecureHash($hash1) == hash1
## This module is a deprecated alias for the ``sha1`` module.
{.deprecated.}
include "../std/sha1"

View File

@@ -54,9 +54,9 @@ when defined(nimdoc):
Timer, ## Timer descriptor is completed
Signal, ## Signal is raised
Process, ## Process is finished
Vnode, ## BSD specific file change happens
Vnode, ## BSD specific file change
User, ## User event is raised
Error, ## Error happens while waiting, for descriptor
Error, ## Error occurred while waiting for descriptor
VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred)
VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred)
VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended)
@@ -69,6 +69,8 @@ when defined(nimdoc):
## An object which holds result for descriptor
fd* : int ## file/socket descriptor
events*: set[Event] ## set of events
errorCode*: OSErrorCode ## additional error code information for
## Error events
SelectEvent* = object
## An object which holds user defined event
@@ -79,13 +81,14 @@ when defined(nimdoc):
proc close*[T](s: Selector[T]) =
## Closes the selector.
proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event],
data: T) =
proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event], data: T) =
## Registers file/socket descriptor ``fd`` to selector ``s``
## with events set in ``events``. The ``data`` is application-defined
## data, which will be passed when an event is triggered.
proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) =
proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
events: set[Event]) =
## Update file/socket descriptor ``fd``, registered in selector
## ``s`` with new events set ``event``.
@@ -221,11 +224,15 @@ when defined(nimdoc):
proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} =
## Determines whether selector contains a file descriptor.
proc getFd*[T](s: Selector[T]): int =
## Retrieves the underlying selector's file descriptor.
##
## For *poll* and *select* selectors ``-1`` is returned.
else:
when hasThreadSupport:
import locks
type
SharedArray[T] = UncheckedArray[T]
@@ -234,7 +241,6 @@ else:
proc deallocSharedArray[T](sa: ptr SharedArray[T]) =
deallocShared(cast[pointer](sa))
type
Event* {.pure.} = enum
Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot,
@@ -247,6 +253,7 @@ else:
ReadyKey* = object
fd* : int
events*: set[Event]
errorCode*: OSErrorCode
SelectorKey[T] = object
ident: int
@@ -264,7 +271,7 @@ else:
msg.add("Internal Error\n")
var err = newException(IOSelectorsException, msg)
raise err
proc setNonBlocking(fd: cint) {.inline.} =
setBlocking(fd.SocketHandle, false)

View File

@@ -178,9 +178,8 @@ proc parseUri*(uri: string, result: var Uri) =
i.inc(2) # Skip //
var authority = ""
i.inc parseUntil(uri, authority, {'/', '?', '#'}, i)
if authority == "":
raise newException(ValueError, "Expected authority got nothing.")
parseAuthority(authority, result)
if authority.len > 0:
parseAuthority(authority, result)
else:
result.opaque = true
@@ -465,6 +464,15 @@ when isMainModule:
doAssert test.hostname == "github.com"
doAssert test.port == "dom96"
doAssert test.path == "/packages"
block:
let str = "file:///foo/bar/baz.txt"
let test = parseUri(str)
doAssert test.scheme == "file"
doAssert test.username == ""
doAssert test.hostname == ""
doAssert test.port == ""
doAssert test.path == "/foo/bar/baz.txt"
# Remove dot segments tests
block:

195
lib/std/sha1.nim Normal file
View File

@@ -0,0 +1,195 @@
#
#
# The Nim Compiler
# (c) Copyright 2015 Nim Contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
import strutils
const Sha1DigestSize = 20
type
Sha1Digest = array[0 .. Sha1DigestSize-1, uint8]
SecureHash* = distinct Sha1Digest
# Copyright (c) 2011, Micael Hildenborg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Micael Hildenborg nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Ported to Nim by Erik O'Leary
type
Sha1State* = array[0 .. 5-1, uint32]
Sha1Buffer = array[0 .. 80-1, uint32]
template clearBuffer(w: Sha1Buffer, len = 16) =
zeroMem(addr(w), len * sizeof(uint32))
proc init*(result: var Sha1State) =
result[0] = 0x67452301'u32
result[1] = 0xefcdab89'u32
result[2] = 0x98badcfe'u32
result[3] = 0x10325476'u32
result[4] = 0xc3d2e1f0'u32
proc innerHash(state: var Sha1State, w: var Sha1Buffer) =
var
a = state[0]
b = state[1]
c = state[2]
d = state[3]
e = state[4]
var round = 0
template rot(value, bits: uint32): uint32 =
(value shl bits) or (value shr (32u32 - bits))
template sha1(fun, val: uint32) =
let t = rot(a, 5) + fun + e + val + w[round]
e = d
d = c
c = rot(b, 30)
b = a
a = t
template process(body: untyped) =
w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1)
body
inc(round)
template wrap(dest, value: untyped) =
let v = dest + value
dest = v
while round < 16:
sha1((b and c) or (not b and d), 0x5a827999'u32)
inc(round)
while round < 20:
process:
sha1((b and c) or (not b and d), 0x5a827999'u32)
while round < 40:
process:
sha1(b xor c xor d, 0x6ed9eba1'u32)
while round < 60:
process:
sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32)
while round < 80:
process:
sha1(b xor c xor d, 0xca62c1d6'u32)
wrap state[0], a
wrap state[1], b
wrap state[2], c
wrap state[3], d
wrap state[4], e
proc sha1(src: cstring; len: int): Sha1Digest =
#Initialize state
var state: Sha1State
init(state)
#Create w buffer
var w: Sha1Buffer
#Loop through all complete 64byte blocks.
let byteLen = len
let endOfFullBlocks = byteLen - 64
var endCurrentBlock = 0
var currentBlock = 0
while currentBlock <= endOfFullBlocks:
endCurrentBlock = currentBlock + 64
var i = 0
while currentBlock < endCurrentBlock:
w[i] = uint32(src[currentBlock+3]) or
uint32(src[currentBlock+2]) shl 8'u32 or
uint32(src[currentBlock+1]) shl 16'u32 or
uint32(src[currentBlock]) shl 24'u32
currentBlock += 4
inc(i)
innerHash(state, w)
#Handle last and not full 64 byte block if existing
endCurrentBlock = byteLen - currentBlock
clearBuffer(w)
var lastBlockBytes = 0
while lastBlockBytes < endCurrentBlock:
var value = uint32(src[lastBlockBytes + currentBlock]) shl
((3'u32 - uint32(lastBlockBytes and 3)) shl 3)
w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value
inc(lastBlockBytes)
w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or (
0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3)
)
if endCurrentBlock >= 56:
innerHash(state, w)
clearBuffer(w)
w[15] = uint32(byteLen) shl 3
innerHash(state, w)
# Store hash in result pointer, and make sure we get in in the correct order
# on both endian models.
for i in 0 .. Sha1DigestSize-1:
result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255)
proc sha1(src: string): Sha1Digest =
## Calculate SHA1 from input string
sha1(src, src.len)
proc secureHash*(str: string): SecureHash = SecureHash(sha1(str))
proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename))
proc `$`*(self: SecureHash): string =
result = ""
for v in Sha1Digest(self):
result.add(toHex(int(v), 2))
proc parseSecureHash*(hash: string): SecureHash =
for i in 0 ..< Sha1DigestSize:
Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1]))
proc `==`*(a, b: SecureHash): bool =
# Not a constant-time comparison, but that's acceptable in this context
Sha1Digest(a) == Sha1Digest(b)
when isMainModule:
let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]")
doAssert hash1 == hash1
doAssert parseSecureHash($hash1) == hash1

View File

@@ -1955,13 +1955,13 @@ const
## that you cannot compare a floating point value to this value
## and expect a reasonable result - use the `classify` procedure
## in the module ``math`` for checking for NaN.
NimMajor*: int = 0
NimMajor* {.intdefine.}: int = 0
## is the major number of Nim's version.
NimMinor*: int = 17
NimMinor* {.intdefine.}: int = 17
## is the minor number of Nim's version.
NimPatch*: int = 3
NimPatch* {.intdefine.}: int = 3
## is the patch number of Nim's version.
NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch
@@ -3025,9 +3025,9 @@ when not defined(JS): #and not defined(nimscript):
proc endOfFile*(f: File): bool {.tags: [], benign.}
## Returns true iff `f` is at the end.
proc readChar*(f: File): char {.tags: [ReadIOEffect], deprecated.}
## Reads a single character from the stream `f`. **Deprecated** since
## version 0.16.2. Use some variant of ``readBuffer`` instead.
proc readChar*(f: File): char {.tags: [ReadIOEffect].}
## Reads a single character from the stream `f`. Should not be used in
## performance sensitive code.
proc flushFile*(f: File) {.tags: [WriteIOEffect].}
## Flushes `f`'s buffer.
@@ -3769,7 +3769,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} =
# by ``assert``.
type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect,
tags: [].}
{.deprecated: [THide: Hide].}
Hide(raiseAssert)(msg)
template assert*(cond: bool, msg = "") =

View File

@@ -104,7 +104,7 @@ type
slBitmap: array[RealFli, uint32]
matrix: array[RealFli, array[MaxSli, PBigChunk]]
llmem: PLLChunk
currMem, maxMem, freeMem: int # memory sizes (allocated from OS)
currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS)
lastSize: int # needed for the case that OS gives us pages linearly
chunkStarts: IntSet
root, deleted, last, freeAvlNodes: PAvlNode
@@ -421,7 +421,7 @@ const nimMaxHeap {.intdefine.} = 0
proc requestOsChunks(a: var MemRegion, size: int): PBigChunk =
when not defined(emscripten):
if not a.blockChunkSizeIncrease:
let usedMem = a.currMem # - a.freeMem
let usedMem = a.occ #a.currMem # - a.freeMem
when nimMaxHeap != 0:
if usedMem > nimMaxHeap * 1024 * 1024:
raiseOutOfMem()
@@ -567,7 +567,6 @@ proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) =
addChunkToMatrix(a, rest)
proc getBigChunk(a: var MemRegion, size: int): PBigChunk =
# use first fit for now:
sysAssert(size > 0, "getBigChunk 2")
var size = size # roundup(size, PageSize)
var fl, sl: int
@@ -627,6 +626,85 @@ else:
c = c.next
result = true
when false:
var
rsizes: array[50_000, int]
rsizesLen: int
proc trackSize(size: int) =
rsizes[rsizesLen] = size
inc rsizesLen
proc untrackSize(size: int) =
for i in 0 .. rsizesLen-1:
if rsizes[i] == size:
rsizes[i] = rsizes[rsizesLen-1]
dec rsizesLen
return
c_fprintf(stdout, "%ld\n", size)
sysAssert(false, "untracked size!")
else:
template trackSize(x) = discard
template untrackSize(x) = discard
when false:
# not yet used by the GCs
proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer =
sysAssert(allocInv(a), "rawAlloc: begin")
sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken")
sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small")
var size = roundup(requestedSize, MemAlign)
inc a.occ, size
trackSize(size)
sysAssert(size >= requestedSize, "insufficient allocated size!")
#c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size)
if size <= SmallChunkSize-smallChunkOverhead():
# allocate a small block: for small chunks, we use only its next pointer
var s = size div MemAlign
var c = a.freeSmallChunks[s]
if c == nil:
result = nil
else:
sysAssert c.size == size, "rawAlloc 6"
if c.freeList == nil:
sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize,
"rawAlloc 7")
result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc)
inc(c.acc, size)
else:
result = c.freeList
sysAssert(c.freeList.zeroField == 0, "rawAlloc 8")
c.freeList = c.freeList.next
dec(c.free, size)
sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9")
if c.free < size:
listRemove(a.freeSmallChunks[s], c)
sysAssert(allocInv(a), "rawAlloc: end listRemove test")
sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %%
size == 0, "rawAlloc 21")
sysAssert(allocInv(a), "rawAlloc: end small size")
else:
inc size, bigChunkOverhead()
var fl, sl: int
mappingSearch(size, fl, sl)
sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk")
let c = findSuitableBlock(a, fl, sl)
if c != nil:
removeChunkFromMatrix2(a, c, fl, sl)
if c.size >= size + PageSize:
splitChunk(a, c, size)
# set 'used' to to true:
c.prevSize = 1
incl(a, a.chunkStarts, pageIndex(c))
dec(a.freeMem, size)
result = addr(c.data)
sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13")
sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary")
if a.root == nil: a.root = getBottom(a)
add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size)
else:
result = nil
proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
sysAssert(allocInv(a), "rawAlloc: begin")
sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken")
@@ -676,6 +754,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %%
size == 0, "rawAlloc 21")
sysAssert(allocInv(a), "rawAlloc: end small size")
inc a.occ, size
trackSize(c.size)
else:
size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize)
# allocate a large block
@@ -687,6 +767,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary")
if a.root == nil: a.root = getBottom(a)
add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size)
inc a.occ, c.size
trackSize(c.size)
sysAssert(isAccessible(a, result), "rawAlloc 14")
sysAssert(allocInv(a), "rawAlloc: end")
when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize)
@@ -703,6 +785,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
# `p` is within a small chunk:
var c = cast[PSmallChunk](c)
var s = c.size
dec a.occ, s
untrackSize(s)
sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)"
sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %%
s == 0, "rawDealloc 3")
var f = cast[ptr FreeCell](p)
@@ -733,6 +818,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead())
# free big chunk
var c = cast[PBigChunk](c)
dec a.occ, c.size
untrackSize(c.size)
sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)"
a.deleted = getBottom(a)
del(a, a.root, cast[int](addr(c.data)))
freeBigChunk(a, c)
@@ -851,7 +939,8 @@ proc deallocOsPages(a: var MemRegion) =
proc getFreeMem(a: MemRegion): int {.inline.} = result = a.freeMem
proc getTotalMem(a: MemRegion): int {.inline.} = result = a.currMem
proc getOccupiedMem(a: MemRegion): int {.inline.} =
result = a.currMem - a.freeMem
result = a.occ
# a.currMem - a.freeMem
# ---------------------- thread memory region -------------------------------
@@ -893,7 +982,7 @@ template instantiateForRegion(allocator: untyped) =
#sysAssert(result == countFreeMem())
proc getTotalMem(): int = return allocator.currMem
proc getOccupiedMem(): int = return getTotalMem() - getFreeMem()
proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem()
proc getMaxMem*(): int = return getMaxMem(allocator)
# -------------------- shared heap region ----------------------------------
@@ -944,7 +1033,8 @@ template instantiateForRegion(allocator: untyped) =
sharedMemStatsShared(sharedHeap.currMem)
proc getOccupiedSharedMem(): int =
sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem)
sharedMemStatsShared(sharedHeap.occ)
#sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem)
{.pop.}
{.pop.}

View File

@@ -125,7 +125,7 @@ when BitsPerPage mod (sizeof(int)*8) != 0:
{.error: "(BitsPerPage mod BitsPerUnit) should be zero!".}
# forward declarations:
proc collectCT(gch: var GcHeap) {.benign.}
proc collectCT(gch: var GcHeap; size: int) {.benign.}
proc forAllChildren(cell: PCell, op: WalkOp) {.benign.}
proc doOperation(p: pointer, op: WalkOp) {.benign.}
proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.}
@@ -277,7 +277,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
incTypeSize typ, size
acquire(gch)
gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1")
collectCT(gch)
collectCT(gch, size + sizeof(Cell))
var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell)))
gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
# now it is buffered in the ZCT
@@ -332,7 +332,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
acquire(gch)
collectCT(gch)
collectCT(gch, newsize + sizeof(Cell))
var ol = usrToCell(old)
sysAssert(ol.typ != nil, "growObj: 1")
gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2")
@@ -494,8 +494,9 @@ proc collectCTBody(gch: var GcHeap) =
gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold)
sysAssert(allocInv(gch.region), "collectCT: end")
proc collectCT(gch: var GcHeap) =
if getOccupiedMem(gch.region) >= gch.cycleThreshold and gch.recGcLock == 0:
proc collectCT(gch: var GcHeap; size: int) =
if (getOccupiedMem(gch.region) >= gch.cycleThreshold or
size > getFreeMem(gch.region)) and gch.recGcLock == 0:
collectCTBody(gch)
when not defined(useNimRtl):
@@ -530,7 +531,7 @@ when not defined(useNimRtl):
acquire(gch)
var oldThreshold = gch.cycleThreshold
gch.cycleThreshold = 0 # forces cycle collection
collectCT(gch)
collectCT(gch, 0)
gch.cycleThreshold = oldThreshold
release(gch)

View File

@@ -47,10 +47,22 @@ when not declared(c_fwrite):
# C routine that is used here:
proc c_fread(buf: pointer, size, n: csize, f: File): csize {.
importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].}
proc c_fseek(f: File, offset: clong, whence: cint): cint {.
importc: "fseek", header: "<stdio.h>", tags: [].}
proc c_ftell(f: File): clong {.
importc: "ftell", header: "<stdio.h>", tags: [].}
when defined(windows):
when not defined(amd64):
proc c_fseek(f: File, offset: int64, whence: cint): cint {.
importc: "fseek", header: "<stdio.h>", tags: [].}
proc c_ftell(f: File): int64 {.
importc: "ftell", header: "<stdio.h>", tags: [].}
else:
proc c_fseek(f: File, offset: int64, whence: cint): cint {.
importc: "_fseeki64", header: "<stdio.h>", tags: [].}
proc c_ftell(f: File): int64 {.
importc: "_ftelli64", header: "<stdio.h>", tags: [].}
else:
proc c_fseek(f: File, offset: int64, whence: cint): cint {.
importc: "fseeko", header: "<stdio.h>", tags: [].}
proc c_ftell(f: File): int64 {.
importc: "ftello", header: "<stdio.h>", tags: [].}
proc c_ferror(f: File): cint {.
importc: "ferror", header: "<stdio.h>", tags: [].}
proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {.
@@ -210,12 +222,12 @@ proc readAllBuffer(file: File): string =
result.add(buffer)
break
proc rawFileSize(file: File): int =
proc rawFileSize(file: File): int64 =
# this does not raise an error opposed to `getFileSize`
var oldPos = c_ftell(file)
discard c_fseek(file, 0, 2) # seek the end of the file
result = c_ftell(file)
discard c_fseek(file, clong(oldPos), 0)
discard c_fseek(file, oldPos, 0)
proc endOfFile(f: File): bool =
var c = c_fgetc(f)
@@ -223,7 +235,7 @@ proc endOfFile(f: File): bool =
return c < 0'i32
#result = c_feof(f) != 0
proc readAllFile(file: File, len: int): string =
proc readAllFile(file: File, len: int64): string =
# We acquire the filesize beforehand and hope it doesn't change.
# Speeds things up.
result = newString(len)
@@ -363,7 +375,7 @@ proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool =
result = f != nil
proc setFilePos(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) =
if c_fseek(f, clong(pos), cint(relativeTo)) != 0:
if c_fseek(f, pos, cint(relativeTo)) != 0:
raiseEIO("cannot set file position")
proc getFilePos(f: File): int64 =

File diff suppressed because it is too large Load Diff