From a0df21edc04f9853ae78788e51b65b3e25a4ba45 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 6 May 2015 09:20:50 +0200 Subject: [PATCH 1/9] development version is 0.11.3 --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index ca4c814110..897faf54c1 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1535,7 +1535,7 @@ const NimMinor*: int = 11 ## is the minor number of Nim's version. - NimPatch*: int = 2 + NimPatch*: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch From f9be37e7cb9d919ba423fde5d05aa87962da0259 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 12 May 2015 14:45:50 +0200 Subject: [PATCH 2/9] prepare for unidecode Nimble package --- doc/lib.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/lib.txt b/doc/lib.txt index 1c02780683..15e99ca05e 100644 --- a/doc/lib.txt +++ b/doc/lib.txt @@ -123,11 +123,6 @@ String handling Ropes can represent very long strings efficiently; especially concatenation is done in O(1) instead of O(n). -* `unidecode `_ - This module provides Unicode to ASCII transliterations: - It finds the sequence of ASCII characters that is the closest approximation - to the Unicode string. - * `matchers `_ This module contains various string matchers for email addresses, etc. From bd4d5db18e0523819d80182360433095a97deec9 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 13 May 2015 12:17:11 +0200 Subject: [PATCH 3/9] terminal: doc improvments --- lib/pure/terminal.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 29f700db5c..7b4d548feb 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -364,7 +364,13 @@ macro styledEcho*(m: varargs[expr]): stmt = result.add(newCall(bindSym"write", bindSym"stdout", newStrLitNode("\n"))) result.add(newCall(bindSym"resetAttributes")) -when not defined(windows): +when defined(nimdoc): + proc getch*(): char = + ## Read a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. This is not available for + ## Windows. + discard +elif not defined(windows): proc getch*(): char = ## Read a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. This is not available for From 75ccdebd1e8ea9ffd81815716c2593cfbc9820c0 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 13 May 2015 12:19:08 +0200 Subject: [PATCH 4/9] get rid of deprecated effects --- lib/core/locks.nim | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 8a809fc84b..92ae24a377 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -15,18 +15,6 @@ type TLock* = TSysLock ## Nim lock; whether this is re-entrant ## or not is unspecified! TCond* = TSysCond ## Nim condition variable - - LockEffect* {.deprecated.} = object of RootEffect ## \ - ## effect that denotes that some lock operation - ## is performed. Deprecated, do not use anymore! - AquireEffect* {.deprecated.} = object of LockEffect ## \ - ## effect that denotes that some lock is - ## acquired. Deprecated, do not use anymore! - ReleaseEffect* {.deprecated.} = object of LockEffect ## \ - ## effect that denotes that some lock is - ## released. Deprecated, do not use anymore! -{.deprecated: [FLock: LockEffect, FAquireLock: AquireEffect, - FReleaseLock: ReleaseEffect].} proc initLock*(lock: var TLock) {.inline.} = ## Initializes the given lock. @@ -36,14 +24,14 @@ proc deinitLock*(lock: var TLock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var TLock): bool = +proc tryAcquire*(lock: var TLock): bool = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) proc acquire*(lock: var TLock) = ## Acquires the given lock. acquireSys(lock) - + proc release*(lock: var TLock) = ## Releases the given lock. releaseSys(lock) @@ -58,10 +46,10 @@ proc deinitCond*(cond: var TCond) {.inline.} = deinitSysCond(cond) proc wait*(cond: var TCond, lock: var TLock) {.inline.} = - ## waits on the condition variable `cond`. + ## waits on the condition variable `cond`. waitSysCond(cond, lock) - + proc signal*(cond: var TCond) {.inline.} = - ## sends a signal to the condition variable `cond`. + ## sends a signal to the condition variable `cond`. signalSysCond(cond) From 94f54700c4bfcc04966e0ea011960c5432b29092 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 28 May 2015 12:42:04 +0200 Subject: [PATCH 5/9] first implementation of pinnedSpawn --- compiler/lowerings.nim | 13 ++++--- compiler/semexprs.nim | 16 +++++---- compiler/sempass2.nim | 5 +-- lib/pure/concurrency/threadpool.nim | 56 ++++++++++++++++++++++++++--- 4 files changed, 73 insertions(+), 17 deletions(-) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index b6b01d5585..842bb01f6b 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -167,7 +167,7 @@ proc genDeref*(n: PNode): PNode = result.add n proc callCodegenProc*(name: string, arg1: PNode; - arg2, arg3: PNode = nil): PNode = + arg2, arg3, optionalArgs: PNode = nil): PNode = result = newNodeI(nkCall, arg1.info) let sym = magicsys.getCompilerProc(name) if sym == nil: @@ -177,6 +177,9 @@ proc callCodegenProc*(name: string, arg1: PNode; result.add arg1 if arg2 != nil: result.add arg2 if arg3 != nil: result.add arg3 + if optionalArgs != nil: + for i in 1..optionalArgs.len-3: + result.add optionalArgs[i] result.typ = sym.typ.sons[0] proc callProc(a: PNode): PNode = @@ -483,7 +486,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; barrier, dest: PNode = nil): PNode = # if 'barrier' != nil, then it is in a 'parallel' section and we # generate quite different code - let n = spawnExpr[1] + let n = spawnExpr[^2] let spawnKind = spawnResult(retType, barrier!=nil) case spawnKind of srVoid: @@ -569,7 +572,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; fvField = newDotExpr(scratchObj, field) fvAsExpr = indirectAccess(castExpr, field, n.info) # create flowVar: - result.add newFastAsgnStmt(fvField, callProc(spawnExpr[2])) + result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1])) if barrier == nil: result.add callCodegenProc("nimFlowVarCreateSemaphore", fvField) @@ -584,7 +587,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; let wrapper = createWrapperProc(fn, threadParam, argsParam, varSection, varInit, call, barrierAsExpr, fvAsExpr, spawnKind) - result.add callCodegenProc("nimSpawn", wrapper.newSymNode, - genAddrOf(scratchObj.newSymNode)) + result.add callCodegenProc("nimSpawn" & $spawnExpr.len, wrapper.newSymNode, + genAddrOf(scratchObj.newSymNode), nil, spawnExpr) if spawnKind == srFlowVar: result.add fvField diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index cd6ba37535..be1461c3cc 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1727,13 +1727,17 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = dec c.inParallelStmt of mSpawn: result = setMs(n, s) - result.sons[1] = semExpr(c, n.sons[1]) - if not result[1].typ.isEmptyType: - if spawnResult(result[1].typ, c.inParallelStmt > 0) == srFlowVar: - result.typ = createFlowVar(c, result[1].typ, n.info) + for i in 1 .. 0) == srFlowVar: + result.typ = createFlowVar(c, typ, n.info) else: - result.typ = result[1].typ - result.add instantiateCreateFlowVarCall(c, result[1].typ, n.info).newSymNode + result.typ = typ + result.add instantiateCreateFlowVarCall(c, typ, n.info).newSymNode + else: + result.add emptyNode of mProcCall: result = setMs(n, s) result.sons[1] = semExpr(c, n.sons[1]) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index adf03be641..517c37b0fc 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -237,9 +237,10 @@ proc useVar(a: PEffects, n: PNode) = message(n.info, warnUninit, s.name.s) # prevent superfluous warnings about the same variable: a.init.add s.id - if {sfGlobal, sfThread} * s.flags == {sfGlobal} and s.kind in {skVar, skLet}: + if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet}: if s.guard != nil: guardGlobal(a, n, s.guard) - if (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): + if {sfGlobal, sfThread} * s.flags == {sfGlobal} and + (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(a, s) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 9f1e53fb8a..10117183a9 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -267,6 +267,10 @@ proc nimArgsPassingDone(p: pointer) {.compilerProc.} = const MaxThreadPoolSize* = 256 ## maximal size of the thread pool. 256 threads ## should be good enough for anybody ;-) + MaxDistinguishedThread* = 32 ## maximal number of "distinguished" threads. + +type + ThreadId* = range[0..MaxDistinguishedThread-1] var currentPoolSize: int @@ -291,10 +295,24 @@ proc slave(w: ptr Worker) {.thread.} = w.shutdown = false atomicDec currentPoolSize +proc distinguishedSlave(w: ptr Worker) {.thread.} = + while true: + when declared(atomicStoreN): + atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST) + else: + w.ready = true + await(w.taskArrived) + assert(not w.ready) + w.f(w, w.data) + if w.q.len != 0: w.cleanFlowVars + var workers: array[MaxThreadPoolSize, TThread[ptr Worker]] workersData: array[MaxThreadPoolSize, Worker] + distinguished: array[MaxDistinguishedThread, TThread[ptr Worker]] + distinguishedData: array[MaxDistinguishedThread, Worker] + proc setMinPoolSize*(size: range[1..MaxThreadPoolSize]) = ## sets the minimal thread pool size. The default value of this is 4. minPoolSize = size @@ -308,7 +326,7 @@ proc setMaxPoolSize*(size: range[1..MaxThreadPoolSize]) = let w = addr(workersData[i]) w.shutdown = true -proc activateThread(i: int) {.noinline.} = +proc activateWorkerThread(i: int) {.noinline.} = workersData[i].taskArrived = createSemaphore() workersData[i].taskStarted = createSemaphore() workersData[i].initialized = true @@ -316,10 +334,18 @@ proc activateThread(i: int) {.noinline.} = initLock(workersData[i].q.lock) createThread(workers[i], slave, addr(workersData[i])) +proc activateDistinguishedThread(i: int) {.noinline.} = + distinguishedData[i].taskArrived = createSemaphore() + distinguishedData[i].taskStarted = createSemaphore() + distinguishedData[i].initialized = true + distinguishedData[i].q.empty = createSemaphore() + initLock(distinguishedData[i].q.lock) + createThread(distinguished[i], distinguishedSlave, addr(distinguishedData[i])) + proc setup() = currentPoolSize = min(countProcessors(), MaxThreadPoolSize) readyWorker = addr(workersData[0]) - for i in 0.. Date: Thu, 28 May 2015 12:44:31 +0200 Subject: [PATCH 6/9] minor tut1 update --- doc/tut1.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/tut1.txt b/doc/tut1.txt index 58ace1dbe0..fef4cb4389 100644 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -1325,11 +1325,11 @@ define operators which accept Slice objects to define ranges. b = "Slices are useless." echo a[7..12] # --> 'a prog' - b[11.. -2] = "useful" + b[11..^2] = "useful" echo b # --> 'Slices are useful.' -In the previous example slices are used to modify a part of a string, and even -a negative index is used. The slice's bounds can hold any value supported by +In the previous example slices are used to modify a part of a string. The +slice's bounds can hold any value supported by their type, but it is the proc using the slice object which defines what values are accepted. From c5a479d536eee243846282f3817ebb3916f6bcf4 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 28 May 2015 12:44:58 +0200 Subject: [PATCH 7/9] minor manual update --- doc/manual/types.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/types.txt b/doc/manual/types.txt index bdf51941da..1461b67b26 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -564,7 +564,7 @@ the ``of`` operator can be used to determine the object's type. .. code-block:: nim type - Person {.inheritable.} = object + Person = object of RootObj name*: string # the * means that `name` is accessible from other modules age: int # no * means that the field is hidden From 21ea8e6913fbfc16192ad3fd157e8e18e559219d Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 28 May 2015 12:49:08 +0200 Subject: [PATCH 8/9] some progress on making async multithreaded --- lib/pure/asyncdispatch.nim | 5 +- lib/pure/collections/sharedstrings.nim | 154 ++++++++++++++++++ lib/pure/collections/sharedtables.nim | 105 ++++++++++++ lib/pure/collections/tableimpl.nim | 132 +++++++++++++++ lib/pure/collections/tables.nim | 217 ++++++------------------- lib/pure/selectors.nim | 134 +++++++-------- 6 files changed, 504 insertions(+), 243 deletions(-) create mode 100644 lib/pure/collections/sharedstrings.nim create mode 100644 lib/pure/collections/sharedtables.nim create mode 100644 lib/pure/collections/tableimpl.nim diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index a4d7a16324..2a271b841e 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -122,7 +122,6 @@ export Port, SocketFlag ## Limitations/Bugs ## ---------------- ## -## * ``except`` statement (without `try`) does not work inside async procedures. ## * The effect system (``raises: []``) does not work with async procedures. ## * Can't await in a ``except`` body @@ -929,7 +928,7 @@ else: result = newRawSocket(domain, typ, protocol).TAsyncFD result.SocketHandle.setBlocking(false) when defined(macosx): - result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) register(result) proc newAsyncRawSocket*(domain: Domain = AF_INET, @@ -938,7 +937,7 @@ else: result = newRawSocket(domain, typ, protocol).TAsyncFD result.SocketHandle.setBlocking(false) when defined(macosx): - result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) register(result) proc closeSocket*(sock: TAsyncFD) = diff --git a/lib/pure/collections/sharedstrings.nim b/lib/pure/collections/sharedstrings.nim new file mode 100644 index 0000000000..10ab307672 --- /dev/null +++ b/lib/pure/collections/sharedstrings.nim @@ -0,0 +1,154 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Shared string support for Nim. + +const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000 + +type + UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char] + +type + Buffer = ptr object + refcount: int + capacity, realLen: int + data: UncheckedCharArray + + SharedString* = object ## A string that can be shared. Slicing is O(1). + buffer: Buffer + first, len: int + +proc decRef(b: Buffer) {.inline.} = + if atomicDec(b.refcount) <= 0: + deallocShared(b) + +proc incRef(b: Buffer) {.inline.} = + atomicInc(b.refcount) + +{.experimental.} + +proc `=destroy`*(s: SharedString) = + #echo "destroyed" + if not s.buffer.isNil: + decRef(s.buffer) + +when false: + proc `=`*(dest: var SharedString; src: SharedString) = + incRef(src.buffer) + if not dest.buffer.isNil: + decRef(dest.buffer) + dest.buffer = src.buffer + dest.first = src.first + dest.len = src.len + +proc len*(s: SharedString): int = s.len + +proc `[]`*(s: SharedString; i: Natural): char = + if i < s.len: result = s.buffer.data[i+s.first] + else: raise newException(IndexError, "index out of bounds") + +proc `[]=`*(s: var SharedString; i: Natural; value: char) = + if i < s.len: s.buffer.data[i+s.first] = value + else: raise newException(IndexError, "index out of bounds") + +proc `[]`*(s: SharedString; ab: Slice[int]): SharedString = + #incRef(src.buffer) + if ab.a < s.len: + result.buffer = s.buffer + result.first = ab.a + result.len = min(s.len, ab.b - ab.a + 1) + # else: produce empty string ;-) + +proc newBuffer(cap, len: int): Buffer = + assert cap >= len + result = cast[Buffer](allocShared0(sizeof(int)*3 + cap)) + result.refcount = 0 + result.capacity = cap + result.realLen = len + +proc newSharedString*(len: Natural): SharedString = + if len != 0: + # optimization: Don't have an underlying buffer when 'len == 0' + result.buffer = newBuffer(len, len) + result.first = 0 + result.len = len + +proc newSharedString*(s: string): SharedString = + let len = s.len + if len != 0: + # optimization: Don't have an underlying buffer when 'len == 0' + result.buffer = newBuffer(len, len) + copyMem(addr result.buffer.data[0], cstring(s), s.len) + result.first = 0 + result.len = len + +when declared(atomicLoadN): + template load(x): expr = atomicLoadN(addr x, ATOMIC_SEQ_CST) +else: + # XXX Fixme + template load(x): expr = x + +proc add*(s: var SharedString; t: cstring; len: Natural) = + if len == 0: return + let newLen = s.len + len + if s.buffer.isNil: + s.buffer = newBuffer(len, len) + copyMem(addr s.buffer.data[0], t, len) + s.len = len + elif newLen >= s.buffer.capacity or s.first != 0 or + s.len != s.buffer.realLen or load(s.buffer.refcount) > 1: + let oldBuf = s.buffer + s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen) + copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len) + copyMem(addr s.buffer.data[s.len], t, len) + decRef(oldBuf) + else: + copyMem(addr s.buffer.data[s.len], t, len) + s.buffer.realLen += len + s.len += len + +proc add*(s: var SharedString; t: string) = + s.add(t.cstring, t.len) + +proc rawData*(s: var SharedString): pointer = + if s.buffer.isNil: result = nil + else: result = addr s.buffer.data[s.first] + +proc add*(s: var SharedString; t: SharedString) = + if t.buffer.isNil: return + s.add(cast[cstring](addr s.buffer.data[s.first]), t.len) + +proc `$`*(s: SharedString): string = + result = newString(s.len) + if s.len > 0: + copyMem(addr result[0], addr s.buffer.data[s.first], s.len) + +proc `==`*(s: SharedString; t: string): bool = + if s.buffer.isNil: result = t.len == 0 + else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], + cstring(t), t.len) + +proc `==`*(s, t: SharedString): bool = + if s.buffer.isNil: result = t.len == 0 + else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], + addr t.buffer.data[t.first], t.len) + +iterator items*(s: SharedString): char = + let buf = s.buffer.data + let x = s.first + if buf != nil: + for i in 0..= 0 + if hasKey: result = t.data[index].val + if not hasKey: + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") + +proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = + ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. **Note**: This is inherently + ## unsafe in the context of multi-threading since it returns a pointer + ## to ``B``. + withLock t: + mgetOrPutImpl(enlarge) + +proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = + ## returns true iff `key` is in the table, otherwise inserts `value`. + withLock t: + hasKeyOrPutImpl(enlarge) + +proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = + ## puts a (key, value)-pair into `t`. + withLock t: + putImpl(enlarge) + +proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = + ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + withLock t: + addImpl(enlarge) + +proc del*[A, B](t: var SharedTable[A, B], key: A) = + ## deletes `key` from hash table `t`. + withLock t: + delImpl() + +proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] = + ## creates a new hash table that is empty. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math `_ module or the ``rightSize`` proc from this module. + assert isPowerOfTwo(initialSize) + result.counter = 0 + result.dataLen = initialSize + result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + sizeof(KeyValuePair[A, B]) * initialSize)) + initLock result.lock + +proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = + deallocShared(t.data) + deinitLock t.lock diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim new file mode 100644 index 0000000000..5900864fb8 --- /dev/null +++ b/lib/pure/collections/tableimpl.nim @@ -0,0 +1,132 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## An ``include`` file for the different table implementations. + +# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These +# two procs retain clarity of that encoding without the space cost of an enum. +proc isEmpty(hcode: THash): bool {.inline.} = + result = hcode == 0 + +proc isFilled(hcode: THash): bool {.inline.} = + result = hcode != 0 + +const + growthFactor = 2 + +proc mustRehash(length, counter: int): bool {.inline.} = + assert(length > counter) + result = (length * 2 < counter * 3) or (length - counter < 4) + +proc nextTry(h, maxHash: THash): THash {.inline.} = + result = (h + 1) and maxHash + +template rawGetKnownHCImpl() {.dirty.} = + var h: THash = hc and maxHash(t) # start with real hash value + while isFilled(t.data[h].hcode): + # Compare hc THEN key with boolean short circuit. This makes the common case + # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. + # It does slow down succeeding lookups by one extra THash cmp&and..usually + # just a few clock cycles, generally worth it for any non-integer-like A. + if t.data[h].hcode == hc and t.data[h].key == key: + return h + h = nextTry(h, maxHash(t)) + result = -1 - h # < 0 => MISSING; insert idx = -1 - result + +template rawGetImpl() {.dirty.} = + hc = hash(key) + if hc == 0: # This almost never taken branch should be very predictable. + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + rawGetKnownHCImpl() + +template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add + hc = hash(key) + if hc == 0: + hc = 314159265 + var h: THash = hc and maxHash(t) + while isFilled(t.data[h].hcode): + h = nextTry(h, maxHash(t)) + result = h + +template rawInsertImpl() {.dirty.} = + data[h].key = key + data[h].val = val + data[h].hcode = hc + +proc rawGetKnownHC[X, A](t: X, key: A, hc: THash): int {.inline.} = + rawGetKnownHCImpl() + +proc rawGetDeep[X, A](t: X, key: A, hc: var THash): int {.inline.} = + rawGetDeepImpl() + +proc rawGet[X, A](t: X, key: A, hc: var THash): int {.inline.} = + rawGetImpl() + +proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], + key: A, val: B, hc: THash, h: THash) = + rawInsertImpl() + +template addImpl(enlarge) {.dirty, immediate.} = + if mustRehash(t.dataLen, t.counter): enlarge(t) + var hc: THash + var j = rawGetDeep(t, key, hc) + rawInsert(t, t.data, key, val, hc, j) + inc(t.counter) + +template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = + if mustRehash(t.dataLen, t.counter): + enlarge(t) + index = rawGetKnownHC(t, key, hc) + index = -1 - index # important to transform for mgetOrPutImpl + rawInsert(t, t.data, key, val, hc, index) + inc(t.counter) + +template putImpl(enlarge) {.dirty, immediate.} = + var hc: THash + var index = rawGet(t, key, hc) + if index >= 0: t.data[index].val = val + else: maybeRehashPutImpl(enlarge) + +template mgetOrPutImpl(enlarge) {.dirty, immediate.} = + var hc: THash + var index = rawGet(t, key, hc) + if index < 0: + # not present: insert (flipping index) + maybeRehashPutImpl(enlarge) + # either way return modifiable val + result = t.data[index].val + +template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = + var hc: THash + var index = rawGet(t, key, hc) + if index < 0: + result = false + maybeRehashPutImpl(enlarge) + else: result = true + +template delImpl() {.dirty, immediate.} = + var hc: THash + var i = rawGet(t, key, hc) + let msk = maxHash(t) + if i >= 0: + t.data[i].hcode = 0 + dec(t.counter) + block outer: + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + t.data[i].hcode = 0 # mark current EMPTY + while true: + i = (i + 1) and msk # increment mod table size + if isEmpty(t.data[i].hcode): # end of collision cluster; So all done + break outer + r = t.data[i].hcode and msk # "home" location of key@i + if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + break + shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index a9357ce67d..c802d0221a 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -68,65 +68,20 @@ import hashes, math -{.pragma: myShallow.} - type KeyValuePair[A, B] = tuple[hcode: THash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] - Table* {.myShallow.}[A, B] = object ## generic hash table + Table*[A, B] = object ## generic hash table data: KeyValuePairSeq[A, B] counter: int TableRef*[A,B] = ref Table[A, B] {.deprecated: [TTable: Table, PTable: TableRef].} -when not defined(nimhygiene): - {.pragma: dirty.} +template maxHash(t): expr {.immediate.} = high(t.data) +template dataLen(t): expr = len(t.data) -# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These -# two procs retain clarity of that encoding without the space cost of an enum. -proc isEmpty(hcode: THash): bool {.inline.} = - result = hcode == 0 - -proc isFilled(hcode: THash): bool {.inline.} = - result = hcode != 0 - -proc len*[A, B](t: Table[A, B]): int = - ## returns the number of keys in `t`. - result = t.counter - -iterator pairs*[A, B](t: Table[A, B]): (A, B) = - ## iterates over any (key, value) pair in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) - -iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = - ## iterates over any (key, value) pair in the table `t`. The values - ## can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) - -iterator keys*[A, B](t: Table[A, B]): A = - ## iterates over any key in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].key - -iterator values*[A, B](t: Table[A, B]): B = - ## iterates over any value in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -iterator mvalues*[A, B](t: var Table[A, B]): var B = - ## iterates over any value in the table `t`. The values can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -const - growthFactor = 2 - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) +include tableimpl proc rightSize*(count: Natural): int {.inline.} = ## Return the value of `initialSize` to support `count` items. @@ -137,49 +92,9 @@ proc rightSize*(count: Natural): int {.inline.} = ## Internally, we want mustRehash(rightSize(x), x) == false. result = nextPowerOfTwo(count * 3 div 2 + 4) -proc nextTry(h, maxHash: THash): THash {.inline.} = - result = (h + 1) and maxHash - -template rawGetKnownHCImpl() {.dirty.} = - var h: THash = hc and high(t.data) # start with real hash value - while isFilled(t.data[h].hcode): - # Compare hc THEN key with boolean short circuit. This makes the common case - # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. - # It does slow down succeeding lookups by one extra THash cmp&and..usually - # just a few clock cycles, generally worth it for any non-integer-like A. - if t.data[h].hcode == hc and t.data[h].key == key: - return h - h = nextTry(h, high(t.data)) - result = -1 - h # < 0 => MISSING; insert idx = -1 - result - -template rawGetImpl() {.dirty.} = - hc = hash(key) - if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. - rawGetKnownHCImpl() - -template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add - hc = hash(key) - if hc == 0: - hc = 314159265 - var h: THash = hc and high(t.data) - while isFilled(t.data[h].hcode): - h = nextTry(h, high(t.data)) - result = h - -template rawInsertImpl() {.dirty.} = - data[h].key = key - data[h].val = val - data[h].hcode = hc - -proc rawGetKnownHC[A, B](t: Table[A, B], key: A, hc: THash): int {.inline.} = - rawGetKnownHCImpl() - -proc rawGetDeep[A, B](t: Table[A, B], key: A, hc: var THash): int {.inline.} = - rawGetDeepImpl() - -proc rawGet[A, B](t: Table[A, B], key: A, hc: var THash): int {.inline.} = - rawGetImpl() +proc len*[A, B](t: Table[A, B]): int = + ## returns the number of keys in `t`. + result = t.counter proc `[]`*[A, B](t: Table[A, B], key: A): B = ## retrieves the value at ``t[key]``. If `key` is not in `t`, @@ -215,9 +130,35 @@ proc hasKey*[A, B](t: Table[A, B], key: A): bool = var hc: THash result = rawGet(t, key, hc) >= 0 -proc rawInsert[A, B](t: var Table[A, B], data: var KeyValuePairSeq[A, B], - key: A, val: B, hc: THash, h: THash) = - rawInsertImpl() +iterator pairs*[A, B](t: Table[A, B]): (A, B) = + ## iterates over any (key, value) pair in the table `t`. + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = + ## iterates over any (key, value) pair in the table `t`. The values + ## can be modified. + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: Table[A, B]): A = + ## iterates over any key in the table `t`. + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].key + +iterator values*[A, B](t: Table[A, B]): B = + ## iterates over any value in the table `t`. + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + +iterator mvalues*[A, B](t: var Table[A, B]): var B = + ## iterates over any value in the table `t`. The values can be modified. + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + +proc del*[A, B](t: var Table[A, B], key: A) = + ## deletes `key` from hash table `t`. + delImpl() proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] @@ -228,81 +169,26 @@ proc enlarge[A, B](t: var Table[A, B]) = var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) -template addImpl() {.dirty.} = - if mustRehash(len(t.data), t.counter): enlarge(t) - var hc: THash - var j = rawGetDeep(t, key, hc) - rawInsert(t, t.data, key, val, hc, j) - inc(t.counter) - -template maybeRehashPutImpl() {.dirty.} = - if mustRehash(len(t.data), t.counter): - enlarge(t) - index = rawGetKnownHC(t, key, hc) - index = -1 - index # important to transform for mgetOrPutImpl - rawInsert(t, t.data, key, val, hc, index) - inc(t.counter) - -template putImpl() {.dirty.} = - var hc: THash - var index = rawGet(t, key, hc) - if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl() - -template mgetOrPutImpl() {.dirty.} = - var hc: THash - var index = rawGet(t, key, hc) - if index < 0: maybeRehashPutImpl() # not present: insert (flipping index) - result = t.data[index].val # either way return modifiable val - -template hasKeyOrPutImpl() {.dirty.} = - var hc: THash - var index = rawGet(t, key, hc) - if index < 0: - result = false - maybeRehashPutImpl() - else: result = true - proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way ## returning a value which can be modified. - mgetOrPutImpl() + mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = ## returns true iff `key` is in the table, otherwise inserts `value`. - hasKeyOrPutImpl() + hasKeyOrPutImpl(enlarge) proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a (key, value)-pair into `t`. - putImpl() + putImpl(enlarge) proc add*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. - addImpl() + addImpl(enlarge) -template doWhile(a: expr, b: stmt): stmt = - while true: - b - if not a: break - -proc del*[A, B](t: var Table[A, B], key: A) = - ## deletes `key` from hash table `t`. - var hc: THash - var i = rawGet(t, key, hc) - let msk = high(t.data) - if i >= 0: - t.data[i].hcode = 0 - dec(t.counter) - while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 - var j = i # The correctness of this depends on (h+1) in nextTry, - var r = j # though may be adaptable to other simple sequences. - t.data[i].hcode = 0 # mark current EMPTY - doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): - i = (i + 1) and msk # increment mod table size - if isEmpty(t.data[i].hcode): # end of collision cluster; So all done - return - r = t.data[i].hcode and msk # "home" location of key@i - shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop +proc len*[A, B](t: TableRef[A, B]): int = + ## returns the number of keys in `t`. + result = t.counter proc initTable*[A, B](initialSize=64): Table[A, B] = ## creates a new hash table that is empty. @@ -356,10 +242,6 @@ proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = for item in collection: result[index(item)] = item -proc len*[A, B](t: TableRef[A, B]): int = - ## returns the number of keys in `t`. - result = t.counter - iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): @@ -454,8 +336,7 @@ type OrderedKeyValuePair[A, B] = tuple[ hcode: THash, next: int, key: A, val: B] OrderedKeyValuePairSeq[A, B] = seq[OrderedKeyValuePair[A, B]] - OrderedTable* {. - myShallow.}[A, B] = object ## table that remembers insertion order + OrderedTable* [A, B] = object ## table that remembers insertion order data: OrderedKeyValuePairSeq[A, B] counter, first, last: int OrderedTableRef*[A, B] = ref OrderedTable[A, B] @@ -557,20 +438,20 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) = proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a (key, value)-pair into `t`. - putImpl() + putImpl(enlarge) proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. - addImpl() + addImpl(enlarge) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``value`` if not present, either way ## returning a value which can be modified. - mgetOrPutImpl() + mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = ## returns true iff `key` is in the table, otherwise inserts `value`. - hasKeyOrPutImpl() + hasKeyOrPutImpl(enlarge) proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = ## creates a new ordered hash table that is empty. @@ -741,7 +622,7 @@ proc sort*[A, B](t: OrderedTableRef[A, B], # ------------------------------ count tables ------------------------------- type - CountTable* {.myShallow.}[ + CountTable* [ A] = object ## table that counts the number of each key data: seq[tuple[key: A, val: int]] counter: int diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 6901ecf588..066d7dd52d 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -11,11 +11,11 @@ import tables, os, unsigned, hashes -when defined(linux): +when defined(linux): import posix, epoll -elif defined(windows): +elif defined(windows): import winlean -else: +else: import posix proc hash*(x: SocketHandle): THash {.borrow.} @@ -25,22 +25,23 @@ type Event* = enum EvRead, EvWrite, EvError - SelectorKey* = ref object + SelectorKey* = object fd*: SocketHandle events*: set[Event] ## The events which ``fd`` listens for. - data*: RootRef ## User object. + data*: pointer ## User object. ReadyInfo* = tuple[key: SelectorKey, events: set[Event]] + + when defined(nimdoc): type Selector* = ref object ## An object which holds file descriptors to be checked for read/write ## status. - fds: Table[SocketHandle, SelectorKey] proc register*(s: Selector, fd: SocketHandle, events: set[Event], - data: RootRef): SelectorKey {.discardable.} = + data: pointer): SelectorKey {.discardable.} = ## Registers file descriptor ``fd`` to selector ``s`` with a set of TEvent ## ``events``. @@ -57,7 +58,7 @@ when defined(nimdoc): proc select*(s: Selector, timeout: int): seq[ReadyInfo] = ## The ``events`` field of the returned ``key`` contains the original events ## for which the ``fd`` was bound. This is contrary to the ``events`` field - ## of the ``TReadyInfo`` tuple which determines which events are ready + ## of the ``ReadyInfo`` tuple which determines which events are ready ## on the ``fd``. proc newSelector*(): Selector = @@ -72,11 +73,11 @@ when defined(nimdoc): elif defined(linux): type - Selector* = ref object + Selector* = object epollFD: cint events: array[64, epoll_event] - fds: Table[SocketHandle, SelectorKey] - + fds: SharedTable[SocketHandle, SelectorKey] + proc createEventStruct(events: set[Event], fd: SocketHandle): epoll_event = if EvRead in events: result.events = EPOLLIN @@ -84,21 +85,19 @@ elif defined(linux): result.events = result.events or EPOLLOUT result.events = result.events or EPOLLRDHUP result.data.fd = fd.cint - + proc register*(s: Selector, fd: SocketHandle, events: set[Event], - data: RootRef): SelectorKey {.discardable.} = + data: pointer) = var event = createEventStruct(events, fd) if events != {}: if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: raiseOSError(osLastError()) var key = SelectorKey(fd: fd, events: events, data: data) - + s.fds[fd] = key - result = key - - proc update*(s: Selector, fd: SocketHandle, - events: set[Event]): SelectorKey {.discardable.} = + + proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = if s.fds[fd].events != events: if events == {}: # This fd is idle -- it should not be registered to epoll. @@ -108,7 +107,7 @@ elif defined(linux): # are therefore constantly ready. (leading to 100% CPU usage). if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: raiseOSError(osLastError()) - s.fds[fd].events = events + s.fds.mget(fd).events = events else: var event = createEventStruct(events, fd) if s.fds[fd].events == {}: @@ -119,22 +118,20 @@ elif defined(linux): else: if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: raiseOSError(osLastError()) - s.fds[fd].events = events - - result = s.fds[fd] - - proc unregister*(s: Selector, fd: SocketHandle): SelectorKey {.discardable.} = + s.fds.mget(fd).events = events + + proc unregister*(s: var Selector, fd: SocketHandle) = if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: let err = osLastError() - if err.cint notin {ENOENT, EBADF}: # TODO: Why do we sometimes get an EBADF? Is this normal? + if err.cint notin {ENOENT, EBADF}: + # TODO: Why do we sometimes get an EBADF? Is this normal? raiseOSError(err) - result = s.fds[fd] s.fds.del(fd) - proc close*(s: Selector) = + proc close*(s: var Selector) = + deinitSharedTable(s.fds) if s.epollFD.close() != 0: raiseOSError(osLastError()) - dealloc(addr s.events) # TODO: Test this - + proc epollHasFd(s: Selector, fd: SocketHandle): bool = result = true var event = createEventStruct(s.fds[fd].events, fd) @@ -142,9 +139,9 @@ elif defined(linux): let err = osLastError() if err.cint in {ENOENT, EBADF}: return false - raiseOSError(osLastError()) - - proc select*(s: Selector, timeout: int): seq[ReadyInfo] = + raiseOSError(err) + + proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = ## ## The ``events`` field of the returned ``key`` contains the original events ## for which the ``fd`` was bound. This is contrary to the ``events`` field @@ -156,11 +153,11 @@ elif defined(linux): let err = osLastError() if err.cint == EINTR: return @[] - raiseOSError(osLastError()) + raiseOSError(err) if evNum == 0: return @[] for i in 0 .. Date: Tue, 2 Jun 2015 16:21:55 +0200 Subject: [PATCH 9/9] proper waiting for the pinned thread --- lib/pure/concurrency/threadpool.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 10117183a9..0059e3897f 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -128,6 +128,7 @@ type initialized: bool # whether it has even been initialized shutdown: bool # the pool requests to shut down this worker thread q: ToFreeQueue + readyForTask: Semaphore proc await*(fv: FlowVarBase) = ## waits until the value for the flowVar arrives. Usually it is not necessary @@ -301,6 +302,7 @@ proc distinguishedSlave(w: ptr Worker) {.thread.} = atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST) else: w.ready = true + signal(w.readyForTask) await(w.taskArrived) assert(not w.ready) w.f(w, w.data) @@ -340,6 +342,7 @@ proc activateDistinguishedThread(i: int) {.noinline.} = distinguishedData[i].initialized = true distinguishedData[i].q.empty = createSemaphore() initLock(distinguishedData[i].q.lock) + distinguishedData[i].readyForTask = createSemaphore() createThread(distinguished[i], distinguishedSlave, addr(distinguishedData[i])) proc setup() = @@ -429,11 +432,11 @@ proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} = acquire(distinguishedLock) if not distinguishedData[id].initialized: activateDistinguishedThread(id) + release(distinguishedLock) while true: if selectWorker(addr(distinguishedData[id]), fn, data): break - cpuRelax() - # XXX exponential backoff? - release(distinguishedLock) + await(distinguishedData[id].readyForTask) + proc sync*() = ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate