From 6c4870d8122f2787feabb3f90647fed5fd31af45 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 30 Apr 2014 17:38:44 +0100 Subject: [PATCH 1/9] Get rid unsafe pointer type in GetQueuedCompletionStatus. --- lib/pure/asyncdispatch.nim | 5 +++-- lib/pure/asynchttpserver.nim | 4 +++- lib/windows/winlean.nim | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 93eddc2a60..f5dcf11a27 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -182,8 +182,9 @@ when defined(windows) or defined(nimdoc): var lpNumberOfBytesTransferred: DWORD var lpCompletionKey: ULONG var customOverlapped: PCustomOverlapped - let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, - addr lpCompletionKey, addr customOverlapped, llTimeout).bool + let res = GetQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POverlapped](addr customOverlapped), llTimeout).bool # http://stackoverflow.com/a/12277264/492186 # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 2ebd7036d9..6c2414d999 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -174,7 +174,9 @@ when isMainModule: proc cb(req: TRequest) {.async.} = #echo(req.reqMethod, " ", req.url) #echo(req.headers) - await req.respond(Http200, "Hello World") + let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", + "Content-type": "text/plain; charset=utf-8"} + await req.respond(Http200, "Hello World", headers.newStringTable()) server.serve(TPort(5555), cb) runForever() diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index d5dda4b637..a868f30250 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -672,7 +672,7 @@ proc CreateIoCompletionPort*(FileHandle: THANDLE, ExistingCompletionPort: THANDL proc GetQueuedCompletionStatus*(CompletionPort: THandle, lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG, - lpOverlapped: pointer, + lpOverlapped: ptr POverlapped, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} From 543687f34536eb146a509554035df733ba83cffc Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 30 Apr 2014 22:50:17 +0100 Subject: [PATCH 2/9] Fixes buffered recv in asyncnet. --- lib/pure/asyncnet.nim | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index b1abf627bb..9394078c83 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -97,29 +97,35 @@ proc recv*(socket: PAsyncSocket, size: int, ## to be read then the future will complete with a value of ``""``. if socket.isBuffered: result = newString(size) - - template returnNow(readBytes: int) = - result.setLen(readBytes) - # Only increase buffer position when not peeking. - if (flags and MSG_PEEK) != MSG_PEEK: - socket.currPos.inc(readBytes) - return + let originalBufPos = socket.currPos if socket.bufLen == 0: let res = await socket.readIntoBuf(flags and (not MSG_PEEK)) - if res == 0: returnNow(0) + if res == 0: + result.setLen(0) + return var read = 0 while read < size: if socket.currPos >= socket.bufLen: + if (flags and MSG_PEEK) == MSG_PEEK: + # We don't want to get another buffer if we're peeking. + result.setLen(read) + return let res = await socket.readIntoBuf(flags and (not MSG_PEEK)) - if res == 0: returnNow(read) + if res == 0: + result.setLen(read) + return let chunk = min(socket.bufLen-socket.currPos, size-read) - copyMem(addr(result[read]), addr(socket.buffer[socket.currPos+read]), chunk) + copyMem(addr(result[read]), addr(socket.buffer[socket.currPos]), chunk) read.inc(chunk) + socket.currPos.inc(chunk) - returnNow(read) + if (flags and MSG_PEEK) == MSG_PEEK: + # Restore old buffer cursor position. + socket.currPos = originalBufPos + result.setLen(read) else: result = await recv(socket.fd.TAsyncFD, size, flags) From a21289f5d5ac51bf0459f512eb566af8819a722c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 1 May 2014 23:27:43 +0100 Subject: [PATCH 3/9] Await is now supported in try statements. --- lib/pure/asyncdispatch.nim | 105 ++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 26 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index f5dcf11a27..85c31fdd03 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -37,10 +37,10 @@ type PFutureBase* = ref object of PObject cb: proc () {.closure,gcsafe.} finished: bool + error*: ref EBase PFuture*[T] = ref object of PFutureBase value: T - error*: ref EBase # TODO: This shouldn't be necessary, generics bug? proc newFuture*[T](): PFuture[T] = ## Creates a new future. @@ -114,7 +114,7 @@ proc finished*[T](future: PFuture[T]): bool = ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. future.finished -proc failed*[T](future: PFuture[T]): bool = +proc failed*(future: PFutureBase): bool = ## Determines whether ``future`` completed with an error. future.error != nil @@ -764,25 +764,56 @@ proc accept*(socket: TAsyncFD): PFuture[TAsyncFD] = template createCb*(retFutureSym, iteratorNameSym: expr): stmt {.immediate.} = var nameIterVar = iteratorNameSym proc cb {.closure,gcsafe.} = - if not nameIterVar.finished: - var next = nameIterVar() - if next == nil: - assert retFutureSym.finished, "Async procedure's return Future was not finished." - else: - next.callback = cb + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's return Future was not finished." + else: + next.callback = cb + except: + retFutureSym.fail(getCurrentException()) cb() +proc generateExceptionCheck(futSym, + exceptBranch, rootReceiver: PNimrodNode): PNimrodNode {.compileTime.} = + if exceptBranch == nil: + result = rootReceiver + else: + if exceptBranch[0].kind == nnkStmtList: + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), + exceptBranch[0] + ) + ) + else: + expectKind(exceptBranch[1], nnkStmtList) + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), + newIfStmt( + (infix(newDotExpr(futSym, newIdentNode("error")), "of", exceptBranch[0]), + exceptBranch[1]) + ) + ) + ) + let elseNode = newNimNode(nnkElse) + elseNode.add newNimNode(nnkStmtList) + elseNode[0].add rootReceiver + result.add elseNode + template createVar(futSymName: string, asyncProc: PNimrodNode, - valueReceiver: expr) {.immediate, dirty.} = - # TODO: Used template here due to bug #926 + valueReceiver, rootReceiver: expr) {.immediate, dirty.} = result = newNimNode(nnkStmtList) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future = y result.add newNimNode(nnkYieldStmt).add(futSym) # -> yield future valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future.read + result.add generateExceptionCheck(futSym, exceptBranch, rootReceiver) proc processBody(node, retFutureSym: PNimrodNode, - subtypeName: string): PNimrodNode {.compileTime.} = + subtypeName: string, + exceptBranch: PNimrodNode): PNimrodNode {.compileTime.} = + #echo(node.treeRepr) result = node case node.kind of nnkReturnStmt: @@ -795,7 +826,7 @@ proc processBody(node, retFutureSym: PNimrodNode, result.add newCall(newIdentNode("complete"), retFutureSym) else: result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subtypeName)) + node[0].processBody(retFutureSym, subtypeName, exceptBranch)) result.add newNimNode(nnkReturnStmt).add(newNilLit()) return # Don't process the children of this return stmt @@ -808,16 +839,16 @@ proc processBody(node, retFutureSym: PNimrodNode, of nnkCall: # await foo(p, x) var futureValue: PNimrodNode - createVar("future" & $node[1][0].toStrLit, node[1], futureValue) - result.add futureValue + createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue) else: error("Invalid node kind in 'await', got: " & $node[1].kind) elif node[1].kind == nnkCommand and node[1][0].kind == nnkIdent and node[1][0].ident == !"await": # foo await x var newCommand = node - createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1]) - result.add newCommand + createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand) of nnkVarSection, nnkLetSection: case node[0][2].kind @@ -826,8 +857,7 @@ proc processBody(node, retFutureSym: PNimrodNode, # var x = await y var newVarSection = node # TODO: Should this use copyNimNode? createVar("future" & $node[0][0].ident, node[0][2][1], - newVarSection[0][2]) - result.add newVarSection + newVarSection[0][2], newVarSection) else: discard of nnkAsgn: case node[1].kind @@ -835,19 +865,42 @@ proc processBody(node, retFutureSym: PNimrodNode, if node[1][0].ident == !"await": # x = await y var newAsgn = node - createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1]) - result.add newAsgn + createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn) else: discard of nnkDiscardStmt: # discard await x if node[0][0].kind == nnkIdent and node[0][0].ident == !"await": - var dummy = newNimNode(nnkStmtList) - createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], dummy) + var newDiscard = node + createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList) + proc processForTry(n: PNimrodNode, i: var int, + res: PNimrodNode): bool {.compileTime.} = + result = false + while i < n[0].len: + var processed = processBody(n[0][i], retFutureSym, subtypeName, n[1]) + if processed.kind != n[0][i].kind or processed.len != n[0][i].len: + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + discard processForTry(n, i, processed[2][1][0]) + res.add processed + result = true + else: + res.add n[0][i] + i.inc + var i = 0 + if not processForTry(node, i, result): + var temp = node + temp[0] = result + result = temp + return else: discard - + for i in 0 .. # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeName) + var procBody = prc[6].processBody(retFutureSym, subtypeName, nil) if subtypeName != "void": procBody.insert(0, newNimNode(nnkVarSection).add( newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T From 3adceb34e06f218d3f679b90c134b7aaf2de67b3 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 2 May 2014 17:17:12 -0500 Subject: [PATCH 4/9] fixed constant typo (SimulateCaasMemReset) --- compiler/main.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index 0e85405568..f833394f72 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -284,11 +284,11 @@ proc resetMemory = echo GC_getStatistics() const - SimiluateCaasMemReset = false + SimulateCaasMemReset = false PrintRopeCacheStats = false proc mainCommand* = - when SimiluateCaasMemReset: + when SimulateCaasMemReset: gGlobalOptions.incl(optCaasEnabled) # In "nimrod serve" scenario, each command must reset the registered passes @@ -454,6 +454,6 @@ proc mainCommand* = echo " efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float), ffDecimal, 3) - when SimiluateCaasMemReset: + when SimulateCaasMemReset: resetMemory() From dfeb573edb5563f1ff6bfd30f1c3914a88565332 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 3 May 2014 12:14:28 -0500 Subject: [PATCH 5/9] fixed paths because caasdriver is now in testament --- doc/idetools.txt | 2 +- tests/testament/caasdriver.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/idetools.txt b/doc/idetools.txt index d4f0f077de..0c554a80b6 100644 --- a/doc/idetools.txt +++ b/doc/idetools.txt @@ -528,7 +528,7 @@ suite is not integrated with the main test suite and you have to run it manually. First you have to compile the tester:: $ cd my/nimrod/checkout/tests - $ nimrod c caasdriver.nim + $ nimrod c testament/caasdriver.nim Running the ``caasdriver`` without parameters will attempt to process all the test cases in all three operation modes. If a test succeeds diff --git a/tests/testament/caasdriver.nim b/tests/testament/caasdriver.nim index 28f0bae9b9..22c5ed6fa5 100644 --- a/tests/testament/caasdriver.nim +++ b/tests/testament/caasdriver.nim @@ -25,7 +25,7 @@ const silentReplaceText = "--verbosity:0 --hints:off" var - TesterDir = getAppDir() + TesterDir = getAppDir() / ".." NimrodBin = TesterDir / "../bin/nimrod" proc replaceVars(session: var TNimrodSession, text: string): string = From 24babdedf135a85569251b47c03a294623016fc8 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 3 May 2014 21:05:45 +0100 Subject: [PATCH 6/9] DWORD -> PULONG for POverlapped. --- lib/windows/winlean.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index a868f30250..5f4224c2ac 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -640,8 +640,8 @@ proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, type TOVERLAPPED* {.pure, inheritable.} = object - Internal*: DWORD - InternalHigh*: DWORD + Internal*: PULONG + InternalHigh*: PULONG Offset*: DWORD OffsetHigh*: DWORD hEvent*: THANDLE From 8802688e9f13891b7f81b2d5cad401bc47cf52be Mon Sep 17 00:00:00 2001 From: EXetoC Date: Sat, 3 May 2014 23:32:14 +0200 Subject: [PATCH 7/9] Fix #1171. --- lib/pure/asyncdispatch.nim | 27 +++++++++++++------------ tests/async/tnestedpfuturetypeparam.nim | 8 ++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 tests/async/tnestedpfuturetypeparam.nim diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 85c31fdd03..fe9c16308e 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -811,7 +811,7 @@ template createVar(futSymName: string, asyncProc: PNimrodNode, result.add generateExceptionCheck(futSym, exceptBranch, rootReceiver) proc processBody(node, retFutureSym: PNimrodNode, - subtypeName: string, + subTypeIsVoid: bool, exceptBranch: PNimrodNode): PNimrodNode {.compileTime.} = #echo(node.treeRepr) result = node @@ -819,14 +819,14 @@ proc processBody(node, retFutureSym: PNimrodNode, of nnkReturnStmt: result = newNimNode(nnkStmtList) if node[0].kind == nnkEmpty: - if subtypeName != "void": + if not subtypeIsVoid: result.add newCall(newIdentNode("complete"), retFutureSym, newIdentNode("result")) else: result.add newCall(newIdentNode("complete"), retFutureSym) else: result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subtypeName, exceptBranch)) + node[0].processBody(retFutureSym, subtypeIsVoid, exceptBranch)) result.add newNimNode(nnkReturnStmt).add(newNilLit()) return # Don't process the children of this return stmt @@ -880,7 +880,7 @@ proc processBody(node, retFutureSym: PNimrodNode, res: PNimrodNode): bool {.compileTime.} = result = false while i < n[0].len: - var processed = processBody(n[0][i], retFutureSym, subtypeName, n[1]) + var processed = processBody(n[0][i], retFutureSym, subtypeIsVoid, n[1]) if processed.kind != n[0][i].kind or processed.len != n[0][i].len: expectKind(processed, nnkStmtList) expectKind(processed[2][1], nnkElse) @@ -900,7 +900,7 @@ proc processBody(node, retFutureSym: PNimrodNode, else: discard for i in 0 .. var retFuture = newFuture[T]() var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: returnType[1] outerProcBody.add( newVarStmt(retFutureSym, newCall( newNimNode(nnkBracketExpr).add( newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. - newIdentNode(subtypeName))))) # Get type from return type of this proc + subRetType)))) # Get type from return type of this proc # -> iterator nameIter(): PFutureBase {.closure.} = # -> var result: T # -> # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeName, nil) - if subtypeName != "void": + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + if not subtypeIsVoid: procBody.insert(0, newNimNode(nnkVarSection).add( newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T procBody.add( @@ -977,7 +978,7 @@ macro async*(prc: stmt): stmt {.immediate.} = for i in 0 .. Date: Sun, 4 May 2014 01:52:42 +0200 Subject: [PATCH 8/9] Fix #1170. --- lib/pure/asyncdispatch.nim | 3 ++- tests/async/tasyncdiscard.nim | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/async/tasyncdiscard.nim diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index fe9c16308e..fcf9478317 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -869,7 +869,8 @@ proc processBody(node, retFutureSym: PNimrodNode, else: discard of nnkDiscardStmt: # discard await x - if node[0][0].kind == nnkIdent and node[0][0].ident == !"await": + if node[0].kind != nnkEmpty and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": var newDiscard = node createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], newDiscard[0], newDiscard) diff --git a/tests/async/tasyncdiscard.nim b/tests/async/tasyncdiscard.nim new file mode 100644 index 0000000000..48d8a8c4d6 --- /dev/null +++ b/tests/async/tasyncdiscard.nim @@ -0,0 +1,39 @@ +discard """ + output: ''' +1 +2 +3 +4 +1 +2 +1 +6 +''' +""" +import asyncio, asyncdispatch, asyncnet + +proc main {.async.} = + proc f: PFuture[int] {.async.} = + discard + echo 1 + discard + result = 2 + discard + + let x = await f() + echo x + echo 3 + + proc g: PFuture[int] {.async.} = + discard + echo 4 + discard + result = 6 + discard + echo await f() + discard await f() + + discard await g() + echo 6 + +main() From 79891b6b9b0cb324081dbd6b558534f91ec134fe Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Sun, 4 May 2014 15:22:50 +0200 Subject: [PATCH 9/9] Added support for ref type hash tables. This reuses the hash table implementation for objects (and the associated tests). For efficiency reasons, iterator implementations are currently adapted rather than calling the TTable code. --- lib/pure/collections/tables.nim | 267 +++++++++++++++++++++++++++++++- tests/table/ptables.nim | 128 +++++++++++++++ tests/table/ptables2.nim | 20 +++ 3 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 tests/table/ptables.nim create mode 100644 tests/table/ptables2.nim diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 33e558aee2..848f4b8bae 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -66,6 +66,7 @@ type TTable* {.final, myShallow.}[A, B] = object ## generic hash table data: TKeyValuePairSeq[A, B] counter: int + PTable*[A,B] = ref TTable[A, B] when not defined(nimhygiene): {.pragma: dirty.} @@ -231,7 +232,7 @@ proc `$`*[A, B](t: TTable[A, B]): string = ## The `$` operator for hash tables. dollarImpl() -proc `==`*[A, B](s, t: TTable[A, B]): bool = +template equalsImpl() = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have # to use the slow route here: @@ -240,6 +241,9 @@ proc `==`*[A, B](s, t: TTable[A, B]): bool = if t[key] != val: return false return true +proc `==`*[A, B](s, t: TTable[A, B]): bool = + equalsImpl() + proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] @@ -247,6 +251,88 @@ proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] = for item in collection: result[index(item)] = item +proc len*[A, B](t: PTable[A, B]): int = + ## returns the number of keys in `t`. + result = t.counter + +iterator pairs*[A, B](t: PTable[A, B]): tuple[key: A, val: B] = + ## iterates over any (key, value) pair in the table `t`. + for h in 0..high(t.data): + if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: PTable[A, B]): tuple[key: A, val: var B] = + ## iterates over any (key, value) pair in the table `t`. The values + ## can be modified. + for h in 0..high(t.data): + if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: PTable[A, B]): A = + ## iterates over any key in the table `t`. + for h in 0..high(t.data): + if t.data[h].slot == seFilled: yield t.data[h].key + +iterator values*[A, B](t: PTable[A, B]): B = + ## iterates over any value in the table `t`. + for h in 0..high(t.data): + if t.data[h].slot == seFilled: yield t.data[h].val + +iterator mvalues*[A, B](t: PTable[A, B]): var B = + ## iterates over any value in the table `t`. The values can be modified. + for h in 0..high(t.data): + if t.data[h].slot == seFilled: yield t.data[h].val + +proc `[]`*[A, B](t: PTable[A, B], key: A): B = + ## retrieves the value at ``t[key]``. If `key` is not in `t`, + ## default empty value for the type `B` is returned + ## and no exception is raised. One can check with ``hasKey`` whether the key + ## exists. + result = t[][key] + +proc mget*[A, B](t: PTable[A, B], key: A): var B = + ## retrieves the value at ``t[key]``. The value can be modified. + ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. + t[].mget(key) + +proc hasKey*[A, B](t: PTable[A, B], key: A): bool = + ## returns true iff `key` is in the table `t`. + result = t[].hasKey(key) + +proc `[]=`*[A, B](t: PTable[A, B], key: A, val: B) = + ## puts a (key, value)-pair into `t`. + t[][key] = val + +proc add*[A, B](t: PTable[A, B], key: A, val: B) = + ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + t[].add(key, val) + +proc del*[A, B](t: PTable[A, B], key: A) = + ## deletes `key` from hash table `t`. + t[].del(key) + +proc newTable*[A, B](initialSize=64): PTable[A, B] = + new(result) + result[] = initTable[A, B](initialSize) + +proc newTable*[A, B](pairs: openArray[tuple[key: A, + val: B]]): PTable[A, B] = + ## creates a new hash table that contains the given `pairs`. + new(result) + result[] = toTable[A, B](pairs) + +proc `$`*[A, B](t: PTable[A, B]): string = + ## The `$` operator for hash tables. + dollarImpl() + +proc `==`*[A, B](s, t: PTable[A, B]): bool = + equalsImpl() + +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): PTable[C, B] = + ## Index the collection with the proc provided. + # TODO: As soon as supported, change collection: A to collection: A[B] + result = newTable[C, B]() + for item in collection: + result[index(item)] = item + # ------------------------------ ordered table ------------------------------ type @@ -257,6 +343,7 @@ type final, myShallow.}[A, B] = object ## table that remembers insertion order data: TOrderedKeyValuePairSeq[A, B] counter, first, last: int + POrderedTable*[A, B] = ref TOrderedTable[A, B] proc len*[A, B](t: TOrderedTable[A, B]): int {.inline.} = ## returns the number of keys in `t`. @@ -417,6 +504,96 @@ proc sort*[A, B](t: var TOrderedTable[A, B], t.first = list t.last = tail +proc len*[A, B](t: POrderedTable[A, B]): int {.inline.} = + ## returns the number of keys in `t`. + result = t.counter + +template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = + var h = t.first + while h >= 0: + var nxt = t.data[h].next + if t.data[h].slot == seFilled: yieldStmt + h = nxt + +iterator pairs*[A, B](t: POrderedTable[A, B]): tuple[key: A, val: B] = + ## iterates over any (key, value) pair in the table `t` in insertion + ## order. + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: POrderedTable[A, B]): tuple[key: A, val: var B] = + ## iterates over any (key, value) pair in the table `t` in insertion + ## order. The values can be modified. + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: POrderedTable[A, B]): A = + ## iterates over any key in the table `t` in insertion order. + forAllOrderedPairs: + yield t.data[h].key + +iterator values*[A, B](t: POrderedTable[A, B]): B = + ## iterates over any value in the table `t` in insertion order. + forAllOrderedPairs: + yield t.data[h].val + +iterator mvalues*[A, B](t: POrderedTable[A, B]): var B = + ## iterates over any value in the table `t` in insertion order. The values + ## can be modified. + forAllOrderedPairs: + yield t.data[h].val + +proc `[]`*[A, B](t: POrderedTable[A, B], key: A): B = + ## retrieves the value at ``t[key]``. If `key` is not in `t`, + ## default empty value for the type `B` is returned + ## and no exception is raised. One can check with ``hasKey`` whether the key + ## exists. + result = t[][key] + +proc mget*[A, B](t: POrderedTable[A, B], key: A): var B = + ## retrieves the value at ``t[key]``. The value can be modified. + ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. + result = t[].mget(key) + +proc hasKey*[A, B](t: POrderedTable[A, B], key: A): bool = + ## returns true iff `key` is in the table `t`. + result = t[].hasKey(key) + +proc `[]=`*[A, B](t: POrderedTable[A, B], key: A, val: B) = + ## puts a (key, value)-pair into `t`. + t[][key] = val + +proc add*[A, B](t: POrderedTable[A, B], key: A, val: B) = + ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + t[].add(key, val) + +proc newOrderedTable*[A, B](initialSize=64): POrderedTable[A, B] = + ## creates a new ordered hash table that is empty. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math `_ module. + new(result) + result[] = initOrderedTable[A, B]() + +proc newOrderedTable*[A, B](pairs: openArray[tuple[key: A, + val: B]]): POrderedTable[A, B] = + ## creates a new ordered hash table that contains the given `pairs`. + result = newOrderedTable[A, B](nextPowerOfTwo(pairs.len+10)) + for key, val in items(pairs): result[key] = val + +proc `$`*[A, B](t: POrderedTable[A, B]): string = + ## The `$` operator for ordered hash tables. + dollarImpl() + +proc sort*[A, B](t: POrderedTable[A, B], + cmp: proc (x,y: tuple[key: A, val: B]): int) = + ## sorts `t` according to `cmp`. This modifies the internal list + ## that kept the insertion order, so insertion order is lost after this + ## call but key lookup and insertions remain possible after `sort` (in + ## contrast to the `sort` for count tables). + t[].sort(cmp) + # ------------------------------ count tables ------------------------------- type @@ -424,6 +601,7 @@ type A] = object ## table that counts the number of each key data: seq[tuple[key: A, val: int]] counter: int + PCountTable*[A] = ref TCountTable[A] proc len*[A](t: TCountTable[A]): int = ## returns the number of keys in `t`. @@ -567,6 +745,93 @@ proc sort*[A](t: var TCountTable[A]) = if j < h: break if h == 1: break +proc len*[A](t: PCountTable[A]): int = + ## returns the number of keys in `t`. + result = t.counter + +iterator pairs*[A](t: PCountTable[A]): tuple[key: A, val: int] = + ## iterates over any (key, value) pair in the table `t`. + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A](t: PCountTable[A]): tuple[key: A, val: var int] = + ## iterates over any (key, value) pair in the table `t`. The values can + ## be modified. + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator keys*[A](t: PCountTable[A]): A = + ## iterates over any key in the table `t`. + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].key + +iterator values*[A](t: PCountTable[A]): int = + ## iterates over any value in the table `t`. + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + +iterator mvalues*[A](t: PCountTable[A]): var int = + ## iterates over any value in the table `t`. The values can be modified. + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + +proc `[]`*[A](t: PCountTable[A], key: A): int = + ## retrieves the value at ``t[key]``. If `key` is not in `t`, + ## 0 is returned. One can check with ``hasKey`` whether the key + ## exists. + result = t[][key] + +proc mget*[A](t: PCountTable[A], key: A): var int = + ## retrieves the value at ``t[key]``. The value can be modified. + ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. + result = t[].mget(key) + +proc hasKey*[A](t: PCountTable[A], key: A): bool = + ## returns true iff `key` is in the table `t`. + result = t[].hasKey(key) + +proc `[]=`*[A](t: PCountTable[A], key: A, val: int) = + ## puts a (key, value)-pair into `t`. `val` has to be positive. + assert val > 0 + t[][key] = val + +proc newCountTable*[A](initialSize=64): PCountTable[A] = + ## creates a new count table that is empty. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math `_ module. + new(result) + result[] = initCountTable[A](initialSize) + +proc newCountTable*[A](keys: openArray[A]): PCountTable[A] = + ## creates a new count table with every key in `keys` having a count of 1. + result = newCountTable[A](nextPowerOfTwo(keys.len+10)) + for key in items(keys): result[key] = 1 + +proc `$`*[A](t: PCountTable[A]): string = + ## The `$` operator for count tables. + dollarImpl() + +proc inc*[A](t: PCountTable[A], key: A, val = 1) = + ## increments `t[key]` by `val`. + t[].inc(key, val) + +proc smallest*[A](t: PCountTable[A]): tuple[key: A, val: int] = + ## returns the largest (key,val)-pair. Efficiency: O(n) + t[].smallest + +proc largest*[A](t: PCountTable[A]): tuple[key: A, val: int] = + ## returns the (key,val)-pair with the largest `val`. Efficiency: O(n) + t[].largest + +proc sort*[A](t: PCountTable[A]) = + ## sorts the count table so that the entry with the highest counter comes + ## first. This is destructive! You must not modify `t` afterwards! + ## You can use the iterators `pairs`, `keys`, and `values` to iterate over + ## `t` in the sorted order. + t[].sort + when isMainModule: type Person = object diff --git a/tests/table/ptables.nim b/tests/table/ptables.nim new file mode 100644 index 0000000000..ec52d08c33 --- /dev/null +++ b/tests/table/ptables.nim @@ -0,0 +1,128 @@ +discard """ + output: '''true''' +""" + +import hashes, tables + +const + data = { + "34": 123456, "12": 789, + "90": 343, "0": 34404, + "1": 344004, "2": 344774, + "3": 342244, "4": 3412344, + "5": 341232144, "6": 34214544, + "7": 3434544, "8": 344544, + "9": 34435644, "---00": 346677844, + "10": 34484, "11": 34474, "19": 34464, + "20": 34454, "30": 34141244, "40": 344114, + "50": 344490, "60": 344491, "70": 344492, + "80": 344497} + + sorteddata = { + "---00": 346677844, + "0": 34404, + "1": 344004, + "10": 34484, + "11": 34474, + "12": 789, + "19": 34464, + "2": 344774, "20": 34454, + "3": 342244, "30": 34141244, + "34": 123456, + "4": 3412344, "40": 344114, + "5": 341232144, "50": 344490, + "6": 34214544, "60": 344491, + "7": 3434544, "70": 344492, + "8": 344544, "80": 344497, + "9": 34435644, + "90": 343} + +block tableTest1: + var t = newTable[tuple[x, y: int], string]() + t[(0,0)] = "00" + t[(1,0)] = "10" + t[(0,1)] = "01" + t[(1,1)] = "11" + for x in 0..1: + for y in 0..1: + assert t[(x,y)] == $x & $y + assert($t == + "{(x: 0, y: 0): 00, (x: 0, y: 1): 01, (x: 1, y: 0): 10, (x: 1, y: 1): 11}") + +block tableTest2: + var t = newTable[string, float]() + t["test"] = 1.2345 + t["111"] = 1.000043 + t["123"] = 1.23 + t.del("111") + + t["012"] = 67.9 + t["123"] = 1.5 # test overwriting + + assert t["123"] == 1.5 + assert t["111"] == 0.0 # deleted + assert(not hasKey(t, "111")) + + for key, val in items(data): t[key] = val.toFloat + for key, val in items(data): assert t[key] == val.toFloat + + +block orderedTableTest1: + var t = newOrderedTable[string, int](2) + for key, val in items(data): t[key] = val + for key, val in items(data): assert t[key] == val + var i = 0 + # `pairs` needs to yield in insertion order: + for key, val in pairs(t): + assert key == data[i][0] + assert val == data[i][1] + inc(i) + + for key, val in mpairs(t): val = 99 + for val in mvalues(t): assert val == 99 + +block countTableTest1: + var s = data.toTable + var t = newCountTable[string]() + for k in s.Keys: t.inc(k) + for k in t.keys: assert t[k] == 1 + t.inc("90", 3) + t.inc("12", 2) + t.inc("34", 1) + assert t.largest()[0] == "90" + + t.sort() + var i = 0 + for k, v in t.pairs: + case i + of 0: assert k == "90" and v == 4 + of 1: assert k == "12" and v == 3 + of 2: assert k == "34" and v == 2 + else: break + inc i + +block SyntaxTest: + var x = newTable[int, string]({:}) + +proc orderedTableSortTest() = + var t = newOrderedTable[string, int](2) + for key, val in items(data): t[key] = val + for key, val in items(data): assert t[key] == val + t.sort(proc (x, y: tuple[key: string, val: int]): int = cmp(x.key, y.key)) + var i = 0 + # `pairs` needs to yield in sorted order: + for key, val in pairs(t): + doAssert key == sorteddata[i][0] + doAssert val == sorteddata[i][1] + inc(i) + + # check that lookup still works: + for key, val in pairs(t): + doAssert val == t[key] + # check that insert still works: + t["newKeyHere"] = 80 + + +orderedTableSortTest() +echo "true" + diff --git a/tests/table/ptables2.nim b/tests/table/ptables2.nim new file mode 100644 index 0000000000..939de2b84f --- /dev/null +++ b/tests/table/ptables2.nim @@ -0,0 +1,20 @@ +discard """ + output: '''true''' +""" + +import tables + +proc TestHashIntInt() = + var tab = newTable[int,int]() + for i in 1..1_000_000: + tab[i] = i + for i in 1..1_000_000: + var x = tab[i] + if x != i : echo "not found ", i + +proc run1() = # occupied Memory stays constant, but + for i in 1 .. 50: # aborts at run: 44 on win32 with 3.2GB with out of memory + TestHashIntInt() + +run1() +echo "true"