From 1a50442c13bfc19fe314df0a0fec87dd1b3c5d48 Mon Sep 17 00:00:00 2001 From: Paul Tan Date: Fri, 18 Aug 2017 20:42:16 +0800 Subject: [PATCH 001/200] Generate deps file during C compilation The "genDepend" command was previously taught how to generate a "deps" file in 4910a87c6 (gendepend improvements; refs #5144). Such a deps file is useful in integrating the Nim compiler with an external build system or watch daemon, such that it's possible to only run the Nim compiler when any of the source files are modified. It's also useful to generate the deps file in the nimcache directory during C compilation, without needing to re-run the compilation passes with "genDepend". This would thus reduce overall project build times. --- compiler/main.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/main.nim b/compiler/main.nim index f662ded1ba..76e18a80b0 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -77,6 +77,7 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = let proj = changeFileExt(gProjectFull, "") extccomp.callCCompiler(proj) extccomp.writeJsonBuildInstructions(proj) + writeDepsFile(graph, toGeneratedFile(proj, "")) proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) From b06c0f97a4640c16b2205635b378a39fcef9b814 Mon Sep 17 00:00:00 2001 From: Paul Tan Date: Tue, 22 Aug 2017 23:36:03 +0800 Subject: [PATCH 002/200] writeDepsFile: write included files as well `writeDepsFile()` does not list files which were included with the `include` statement, e.g, with: import file1 include file2 `file1` will be written to the deps file, while `file2` would not. Fix this by modifying `writeDepsFile()` to write included files as well. Now, both `file1` and `file2` in the above example will be written to the deps file. --- compiler/main.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/main.nim b/compiler/main.nim index 76e18a80b0..994c28ccb8 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -16,7 +16,7 @@ import cgen, jsgen, json, nversion, platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen, docgen2, service, parser, modules, ccgutils, sigmatch, ropes, - modulegraphs + modulegraphs, tables from magicsys import systemModule, resetSysTypes @@ -36,6 +36,9 @@ proc writeDepsFile(g: ModuleGraph; project: string) = for m in g.modules: if m != nil: f.writeLine(toFullPath(m.position.int32)) + for k in g.inclToMod.keys: + if g.getModule(k).isNil: # don't repeat includes which are also modules + f.writeLine(k.toFullPath) f.close() proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) = From 277bf1098c2662d21314692963f0596abfc30d3e Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 17 Sep 2017 13:43:22 +0100 Subject: [PATCH 003/200] Add check for broken code-block in docs --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 095c3ec74f..6b8cdbe03b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,3 +48,5 @@ script: - ./koch csource - ./koch nimsuggest # - nim c -r nimsuggest/tester + - ( ! grep -F '.. code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html ) + - ( ! grep -F '..code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html ) From ff98fe1387a24fbe5a09f64264a75a1b25f086d5 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 17 Sep 2017 17:32:06 +0100 Subject: [PATCH 004/200] Fix broken code-block in docs --- lib/pure/times.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index c1d6c3e530..587ea1903b 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -305,6 +305,7 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval = proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval + ## ## .. code-block:: nim ## ## let day = -initInterval(hours=24) From 25831a83d799a0400fbd086ea9a6f704d4d6b216 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 11 Nov 2017 16:59:42 +0000 Subject: [PATCH 005/200] Add unittest suite/test name filters Support simple globbing --- lib/pure/unittest.nim | 91 +++++++++++++++++++++++++++++++++----- tests/stdlib/tunittest.nim | 38 ++++++++++++++++ 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 7a8d1dad09..fbce087ffd 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -21,13 +21,41 @@ ## ``nim c -r `` exits with 0 or 1 ## ## Running a single test -## --------------------- +## ===================== ## -## Simply specify the test name as a command line argument. +## Specify the test name as a command line argument. ## ## .. code:: ## -## nim c -r test "my super awesome test name" +## nim c -r test "my test name" "another test" +## +## Multiple arguments can be used. +## +## Running a single test suite +## =========================== +## +## Specify the suite name delimited by ``"::"``. +## +## .. code:: +## +## nim c -r test "my test name::" +## +## Selecting tests by pattern +## ========================== +## +## A single ``"*"`` can be used for globbing. +## +## Delimit the end of a suite name with ``"::"``. +## +## Tests matching **any** of the arguments are executed. +## +## .. code:: +## +## nim c -r test fast_suite::mytest1 fast_suite::mytest2 +## nim c -r test "fast_suite::mytest*" +## nim c -r test "auth*::" "crypto::hashing*" +## # Run suites starting with 'bug #' and standalone tests starting with '#' +## nim c -r test 'bug #*::' '::#*' ## ## Example ## ------- @@ -121,7 +149,7 @@ var checkpoints {.threadvar.}: seq[string] formatters {.threadvar.}: seq[OutputFormatter] - testsToRun {.threadvar.}: HashSet[string] + testsFilters {.threadvar.}: HashSet[string] when declared(stdout): abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") @@ -300,22 +328,63 @@ method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) = method suiteEnded*(formatter: JUnitOutputFormatter) = formatter.stream.writeLine("\t") -proc shouldRun(testName: string): bool = - if testsToRun.len == 0: +proc glob(matcher, filter: string): bool = + ## Globbing using a single `*`. Empty `filter` matches everything. + if filter.len == 0: return true - result = testName in testsToRun + if not filter.contains('*'): + return matcher == filter + + let beforeAndAfter = filter.split('*', maxsplit=1) + if beforeAndAfter.len == 1: + # "foo*" + return matcher.startswith(beforeAndAfter[0]) + + if matcher.len < filter.len - 1: + return false # "12345" should not match "123*345" + + return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith(beforeAndAfter[1]) + +proc matchFilter(suiteName, testName, filter: string): bool = + if filter == "": + return true + if testName == filter: + # corner case for tests containing "::" in their name + return true + let suiteAndTestFilters = filter.split("::", maxsplit=1) + + if suiteAndTestFilters.len == 1: + # no suite specified + let test_f = suiteAndTestFilters[0] + return glob(testName, test_f) + + return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) + +when defined(testing): export matchFilter + +proc shouldRun(currentSuiteName, testName: string): bool = + ## Check if a test should be run by matching suiteName and testName against + ## test filters. + if testsFilters.len == 0: + return true + + for f in testsFilters: + if matchFilter(currentSuiteName, testName, f): + return true + + return false proc ensureInitialized() = if formatters == nil: formatters = @[OutputFormatter(defaultConsoleFormatter())] - if not testsToRun.isValid: - testsToRun.init() + if not testsFilters.isValid: + testsFilters.init() when declared(paramCount): # Read tests to run from the command line. for i in 1 .. paramCount(): - testsToRun.incl(paramStr(i)) + testsFilters.incl(paramStr(i)) # These two procs are added as workarounds for # https://github.com/nim-lang/Nim/issues/5549 @@ -395,7 +464,7 @@ template test*(name, body) {.dirty.} = ensureInitialized() - if shouldRun(name): + if shouldRun(when declared(testSuiteName): testSuiteName else: "", name): checkpoints = @[] var testStatusIMPL {.inject.} = OK diff --git a/tests/stdlib/tunittest.nim b/tests/stdlib/tunittest.nim index e4a8018713..86b9fd0370 100644 --- a/tests/stdlib/tunittest.nim +++ b/tests/stdlib/tunittest.nim @@ -13,6 +13,8 @@ discard """ [Suite] bug #5784 +[Suite] test name filtering + ''' """ @@ -120,3 +122,39 @@ suite "bug #5784": field: int var obj: Obj check obj.isNil or obj.field == 0 + +when defined(testing): + suite "test name filtering": + test "test name": + check matchFilter("suite1", "foo", "") + check matchFilter("suite1", "foo", "foo") + check matchFilter("suite1", "foo", "::") + check matchFilter("suite1", "foo", "*") + check matchFilter("suite1", "foo", "::foo") + check matchFilter("suite1", "::foo", "::foo") + + test "test name - glob": + check matchFilter("suite1", "foo", "f*") + check matchFilter("suite1", "foo", "*oo") + check matchFilter("suite1", "12345", "12*345") + check matchFilter("suite1", "q*wefoo", "q*wefoo") + check false == matchFilter("suite1", "foo", "::x") + check false == matchFilter("suite1", "foo", "::x*") + check false == matchFilter("suite1", "foo", "::*x") + # overlap + check false == matchFilter("suite1", "12345", "123*345") + check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f") + + test "suite name": + check matchFilter("suite1", "foo", "suite1::") + check false == matchFilter("suite1", "foo", "suite2::") + check matchFilter("suite1", "qwe::foo", "qwe::foo") + check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo") + + test "suite name - glob": + check matchFilter("suite1", "foo", "::*") + check matchFilter("suite1", "foo", "*::*") + check matchFilter("suite1", "foo", "*::foo") + check false == matchFilter("suite1", "foo", "*ite2::") + check matchFilter("suite1", "q**we::foo", "q**we::foo") + check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e") From 6a2b31226e1c43b15a2758c3a6bf7495dd7696ca Mon Sep 17 00:00:00 2001 From: Veladus Date: Sat, 2 Dec 2017 17:54:35 +0100 Subject: [PATCH 006/200] Compiler now catches when an expression is raised which is no Exception --- compiler/msgs.nim | 3 ++- compiler/semstmts.nim | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 2668c72ae0..f22e766c92 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -61,7 +61,7 @@ type errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects, errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX, errCannotInstantiateX, errExprHasNoAddress, errXStackEscape, - errVarForOutParamNeededX, + errVarForOutParamNeededX, errExprIsNoException, errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, errAmbiguousCallXYZ, errWrongNumberOfArguments, errWrongNumberOfArgumentsInCall, @@ -269,6 +269,7 @@ const errExprHasNoAddress: "expression has no address", errXStackEscape: "address of '$1' may not escape its stack frame", errVarForOutParamNeededX: "for a \'var\' type a variable needs to be passed; but '$1' is immutable", + errExprIsNoException: "raised object does not inherit from Exception", errPureTypeMismatch: "type mismatch", errTypeMismatch: "type mismatch: got (", errButExpected: "but expected one of: ", diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index e01f867faf..2f69da62ae 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -721,6 +721,8 @@ proc semFor(c: PContext, n: PNode): PNode = result.typ = enforceVoidContext closeScope(c) +var exceptionID = -1 + proc semRaise(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) @@ -729,6 +731,20 @@ proc semRaise(c: PContext, n: PNode): PNode = var typ = n.sons[0].typ if typ.kind != tyRef or typ.lastSon.kind != tyObject: localError(n.info, errExprCannotBeRaised) + + # check if the given object inherits from Exception + var base = typ.lastSon + while true: + if exceptionID == -1: + if base.sym.name.s == "Exception": + exceptionID = base.id + break + elif base.id == exceptionID: + break + if base.lastSon == nil: + localError(n.info, errExprIsNoException) + return + base = base.lastSon proc addGenericParamListToScope(c: PContext, n: PNode) = if n.kind != nkGenericParams: illFormedAst(n) From 2c886823e77c2244b1407a037e87b9da62f84a92 Mon Sep 17 00:00:00 2001 From: Veladus Date: Sat, 2 Dec 2017 20:57:18 +0100 Subject: [PATCH 007/200] Fixed for diffrent Typeids of Excpetion for diffrent compilation units --- compiler/semstmts.nim | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2f69da62ae..d804beff56 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -721,8 +721,6 @@ proc semFor(c: PContext, n: PNode): PNode = result.typ = enforceVoidContext closeScope(c) -var exceptionID = -1 - proc semRaise(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) @@ -735,11 +733,7 @@ proc semRaise(c: PContext, n: PNode): PNode = # check if the given object inherits from Exception var base = typ.lastSon while true: - if exceptionID == -1: - if base.sym.name.s == "Exception": - exceptionID = base.id - break - elif base.id == exceptionID: + if base.sym.name.s == "Exception": break if base.lastSon == nil: localError(n.info, errExprIsNoException) From 59d45305629e54b8cc9a1f2e0991609363d792a4 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Mon, 11 Dec 2017 21:12:07 +0200 Subject: [PATCH 008/200] Remove `-3` as marker of exited process. Cache exiting process for Windows to omit unnecessary syscalls. Fix closing hThread for Windows. Fix for pause/resume on Windows. Fix process handle leak on Windows. Change behavior for waitForExit on Windows. --- lib/pure/osproc.nim | 116 ++++++++++++++++++++++++++-------------- lib/windows/winlean.nim | 1 + 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 2a1ce0c58b..f762a713b4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -47,6 +47,7 @@ type ProcessObj = object of RootObj when defined(windows): fProcessHandle: Handle + fThreadHandle: Handle inHandle, outHandle, errHandle: FileHandle id: Handle else: @@ -54,6 +55,7 @@ type inStream, outStream, errStream: Stream id: Pid exitStatus: cint + exitFlag: bool options: set[ProcessOption] Process* = ref ProcessObj ## represents an operating system process @@ -237,11 +239,13 @@ proc execProcesses*(cmds: openArray[string], if n > 1: var i = 0 var q = newSeq[Process](n) - var m = min(n, cmds.len) when defined(windows): var w: WOHandleArray + var m = min(min(n, MAXIMUM_WAIT_OBJECTS), cmds.len) var wcount = m + else: + var m = min(n, cmds.len) while i < m: if beforeRunEvent != nil: @@ -262,8 +266,17 @@ proc execProcesses*(cmds: openArray[string], discard elif ret == WAIT_FAILED: raiseOSError(osLastError()) + else: + var status: int32 + for r in 0..m-1: + if not isNil(q[r]) and q[r].fProcessHandle == w[ret]: + discard getExitCodeProcess(q[r].fProcessHandle, status) + q[r].exitFlag = true + q[r].exitStatus = status + discard closeHandle(q[r].fProcessHandle) + break else: - var status : cint = 1 + var status: cint = 1 # waiting for all children, get result if any child exits let res = waitpid(-1, status, 0) if res > 0: @@ -271,6 +284,7 @@ proc execProcesses*(cmds: openArray[string], if not isNil(q[r]) and q[r].id == res: # we updating `exitStatus` manually, so `running()` can work. if WIFEXITED(status) or WIFSIGNALED(status): + q[r].exitFlag = true q[r].exitStatus = status break else: @@ -491,6 +505,7 @@ when defined(Windows) and not defined(useNimRtl): hi, ho, he: Handle new(result) result.options = options + result.exitFlag = true si.cb = sizeof(si).cint if poParentStreams notin options: si.dwFlags = STARTF_USESTDHANDLES # STARTF_USESHOWWINDOW or @@ -559,28 +574,31 @@ when defined(Windows) and not defined(useNimRtl): "Requested command not found: '$1'. OS error:" % command) else: raiseOSError(lastError, command) - # Close the handle now so anyone waiting is woken: - discard closeHandle(procInfo.hThread) + result.fProcessHandle = procInfo.hProcess + result.fThreadHandle = procInfo.hThread result.id = procInfo.dwProcessId + result.exitFlag = false proc close(p: Process) = if poInteractive in p.options: - # somehow this is not always required on Windows: discard closeHandle(p.inHandle) discard closeHandle(p.outHandle) discard closeHandle(p.errHandle) - #discard closeHandle(p.FProcessHandle) + discard closeHandle(p.fProcessHandle) proc suspend(p: Process) = - discard suspendThread(p.fProcessHandle) + discard suspendThread(p.fThreadHandle) proc resume(p: Process) = - discard resumeThread(p.fProcessHandle) + discard resumeThread(p.fThreadHandle) proc running(p: Process): bool = - var x = waitForSingleObject(p.fProcessHandle, 50) - return x == WAIT_TIMEOUT + if p.exitFlag: + return false + else: + var x = waitForSingleObject(p.fProcessHandle, 0) + return x == WAIT_TIMEOUT proc terminate(p: Process) = if running(p): @@ -590,22 +608,35 @@ when defined(Windows) and not defined(useNimRtl): terminate(p) proc waitForExit(p: Process, timeout: int = -1): int = - discard waitForSingleObject(p.fProcessHandle, timeout.int32) + if p.exitFlag: + return p.exitStatus - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - result = res - p.exitStatus = res - discard closeHandle(p.fProcessHandle) + let res = waitForSingleObject(p.fProcessHandle, timeout.int32) + if res == WAIT_TIMEOUT: + terminate(p) + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + if status != STILL_ACTIVE: + p.exitFlag = true + p.exitStatus = status + discard closeHandle(p.fProcessHandle) + result = status + else: + result = -1 proc peekExitCode(p: Process): int = - var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT - if b: result = -1 - else: - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - if res == 0: return p.exitStatus - return res + if p.exitFlag: + return p.exitStatus + + result = -1 + var b = waitForSingleObject(p.fProcessHandle, 0) == WAIT_TIMEOUT + if not b: + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + p.exitFlag = true + p.exitStatus = status + discard closeHandle(p.fProcessHandle) + result = status proc inputStream(p: Process): Stream = streamAccess(p) @@ -737,7 +768,8 @@ elif not defined(useNimRtl): pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options - result.exitStatus = -3 # for ``waitForExit`` + result.exitFlag = true + if poParentStreams notin options: if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or pipe(pStderr) != 0'i32: @@ -792,6 +824,7 @@ elif not defined(useNimRtl): if poEchoCmd in options: echo(command, " ", join(args, " ")) result.id = pid + result.exitFlag = false if poParentStreams in options: # does not make much sense, but better than nothing: @@ -968,14 +1001,14 @@ elif not defined(useNimRtl): if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) proc running(p: Process): bool = - if p.exitStatus != -3: + if p.exitFlag: return false else: - var ret : int - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) + var status: cint = 1 + let ret = waitpid(p.id, status, WNOHANG) if ret == int(p.id): if isExitStatus(status): + p.exitFlag = true p.exitStatus = status return false else: @@ -998,13 +1031,14 @@ elif not defined(useNimRtl): import kqueue, times proc waitForExit(p: Process, timeout: int = -1): int = - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) if timeout == -1: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status else: var kqFD = kqueue() @@ -1025,7 +1059,7 @@ elif not defined(useNimRtl): try: while true: - var status : cint = 1 + var status: cint = 1 var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, addr(tmspec)) if count < 0: @@ -1038,12 +1072,14 @@ elif not defined(useNimRtl): raiseOSError(osLastError()) if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1083,17 +1119,14 @@ elif not defined(useNimRtl): s.tv_sec = b.tv_sec s.tv_nsec = b.tv_nsec - #if waitPid(p.id, p.exitStatus, 0) == int(p.id): - # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitStatus`` for us. Since ``p.exitStatus`` is - # initialized with -3, wrong success exit codes are prevented. - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) if timeout == -1: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status else: var nmask, omask: Sigset @@ -1125,9 +1158,10 @@ elif not defined(useNimRtl): let res = sigtimedwait(nmask, sinfo, tmspec) if res == SIGCHLD: if sinfo.si_pid == p.id: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1148,9 +1182,10 @@ elif not defined(useNimRtl): # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1168,12 +1203,13 @@ elif not defined(useNimRtl): proc peekExitCode(p: Process): int = var status = cint(0) result = -1 - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) var ret = waitpid(p.id, status, WNOHANG) if ret > 0: if isExitStatus(status): + p.exitFlag = true p.exitStatus = status result = exitStatus(status) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 7eb268a9a4..a833377e5e 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -111,6 +111,7 @@ const WAIT_TIMEOUT* = 0x00000102'i32 WAIT_FAILED* = 0xFFFFFFFF'i32 INFINITE* = -1'i32 + STILL_ACTIVE* = 0x00000103'i32 STD_INPUT_HANDLE* = -10'i32 STD_OUTPUT_HANDLE* = -11'i32 From a15ddf4013c5157e7532c71aa2045e07947782fb Mon Sep 17 00:00:00 2001 From: Veladus Date: Mon, 11 Dec 2017 21:48:22 +0100 Subject: [PATCH 009/200] Improved error reporting --- compiler/semstmts.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d804beff56..a44b2fafc4 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -736,7 +736,7 @@ proc semRaise(c: PContext, n: PNode): PNode = if base.sym.name.s == "Exception": break if base.lastSon == nil: - localError(n.info, errExprIsNoException) + localError(n.info, "raised object of type $1 does not inherit from Exception", [typ.sym.name.s]) return base = base.lastSon From 15f72d0cf1390971e047f22dabf7a9e5b24fd85b Mon Sep 17 00:00:00 2001 From: Veladus Date: Mon, 11 Dec 2017 21:49:28 +0100 Subject: [PATCH 010/200] Now analyzes over magics instead of symbol names; but dosn't compile for me --- compiler/ast.nim | 3 ++- compiler/semstmts.nim | 2 +- lib/system.nim | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 5bf4184c95..5b923acb25 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -639,7 +639,8 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, - mNimvm, mIntDefine, mStrDefine, mRunnableExamples + mNimvm, mIntDefine, mStrDefine, mRunnableExamples, + mException # things that we can evaluate safely at compile time, even if not asked for it: const diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a44b2fafc4..c85de35cd2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -733,7 +733,7 @@ proc semRaise(c: PContext, n: PNode): PNode = # check if the given object inherits from Exception var base = typ.lastSon while true: - if base.sym.name.s == "Exception": + if base.sym.magic == mException: break if base.lastSon == nil: localError(n.info, "raised object of type $1 does not inherit from Exception", [typ.sym.name.s]) diff --git a/lib/system.nim b/lib/system.nim index b9f01c3065..ca17f70f6a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -463,7 +463,7 @@ type line*: int ## line number of the proc that is currently executing filename*: cstring ## filename of the proc that is currently executing - Exception* {.compilerproc.} = object of RootObj ## \ + Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ ## Base exception class. ## ## Each exception has to inherit from `Exception`. See the full `exception From 56aa16b1ded1cba0380e6b62973126650d65207e Mon Sep 17 00:00:00 2001 From: Veladus Date: Mon, 11 Dec 2017 22:09:29 +0100 Subject: [PATCH 011/200] removed unused constants --- compiler/msgs.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index f22e766c92..2668c72ae0 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -61,7 +61,7 @@ type errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects, errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX, errCannotInstantiateX, errExprHasNoAddress, errXStackEscape, - errVarForOutParamNeededX, errExprIsNoException, + errVarForOutParamNeededX, errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, errAmbiguousCallXYZ, errWrongNumberOfArguments, errWrongNumberOfArgumentsInCall, @@ -269,7 +269,6 @@ const errExprHasNoAddress: "expression has no address", errXStackEscape: "address of '$1' may not escape its stack frame", errVarForOutParamNeededX: "for a \'var\' type a variable needs to be passed; but '$1' is immutable", - errExprIsNoException: "raised object does not inherit from Exception", errPureTypeMismatch: "type mismatch", errTypeMismatch: "type mismatch: got (", errButExpected: "but expected one of: ", From d3f966922ef4ddd05c137f82e5b2329b3d5dc485 Mon Sep 17 00:00:00 2001 From: Gerke Max Preussner Date: Tue, 12 Dec 2017 08:43:12 -0500 Subject: [PATCH 012/200] Fixed koch warning when installing Nim. (#6898) --- compiler/dfa.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 22a110a1f4..fbf71d95ca 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -382,9 +382,9 @@ proc dfa(code: seq[Instr]) = else: pc2 = pc + 1 if code[pc].kind == fork: - let l = pc + code[pc].dest - if sid >= 0 and s[l].missingOrExcl(sid): - w.add l + let lidx = pc + code[pc].dest + if sid >= 0 and s[lidx].missingOrExcl(sid): + w.add lidx if sid >= 0 and s[pc2].missingOrExcl(sid): pc = pc2 From e6722498595eabf6ef0ae314ad099c4108fde346 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 12 Dec 2017 16:53:09 +0200 Subject: [PATCH 013/200] Windows: Fix invalid handle value for `execProcesses`. Windows. Fix named pipes leak. --- lib/pure/osproc.nim | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index f762a713b4..f610456630 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -273,7 +273,6 @@ proc execProcesses*(cmds: openArray[string], discard getExitCodeProcess(q[r].fProcessHandle, status) q[r].exitFlag = true q[r].exitStatus = status - discard closeHandle(q[r].fProcessHandle) break else: var status: cint = 1 @@ -313,11 +312,14 @@ proc execProcesses*(cmds: openArray[string], w[r] = q[r].fProcessHandle inc(i) else: - q[r] = nil when defined(windows): - for c in r..MAXIMUM_WAIT_OBJECTS - 2: - w[c] = w[c + 1] - dec(wcount) + for k in 0..wcount - 1: + if w[k] == q[r].fProcessHandle: + w[k] = w[wcount - 1] + w[wcount - 1] = 0 + dec(wcount) + break + q[r] = nil dec(ecount) else: for i in 0..high(cmds): @@ -574,17 +576,17 @@ when defined(Windows) and not defined(useNimRtl): "Requested command not found: '$1'. OS error:" % command) else: raiseOSError(lastError, command) - result.fProcessHandle = procInfo.hProcess result.fThreadHandle = procInfo.hThread result.id = procInfo.dwProcessId result.exitFlag = false proc close(p: Process) = - if poInteractive in p.options: + if poParentStreams notin p.options: discard closeHandle(p.inHandle) discard closeHandle(p.outHandle) discard closeHandle(p.errHandle) + discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) proc suspend(p: Process) = @@ -619,6 +621,7 @@ when defined(Windows) and not defined(useNimRtl): if status != STILL_ACTIVE: p.exitFlag = true p.exitStatus = status + discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) result = status else: @@ -635,6 +638,7 @@ when defined(Windows) and not defined(useNimRtl): discard getExitCodeProcess(p.fProcessHandle, status) p.exitFlag = true p.exitStatus = status + discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) result = status From 0429f41e9862f7b6c1570ccbcd4cce541767fdad Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 12 Dec 2017 20:00:14 +0200 Subject: [PATCH 014/200] execProcesses optimization. --- lib/pure/osproc.nim | 53 +++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index f610456630..d72ed1772c 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -257,6 +257,7 @@ proc execProcesses*(cmds: openArray[string], var ecount = len(cmds) while ecount > 0: + var rexit = -1 when defined(windows): # waiting for all children, get result if any child exits var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, @@ -273,6 +274,7 @@ proc execProcesses*(cmds: openArray[string], discard getExitCodeProcess(q[r].fProcessHandle, status) q[r].exitFlag = true q[r].exitStatus = status + rexit = r break else: var status: cint = 1 @@ -281,16 +283,21 @@ proc execProcesses*(cmds: openArray[string], if res > 0: for r in 0..m-1: if not isNil(q[r]) and q[r].id == res: - # we updating `exitStatus` manually, so `running()` can work. if WIFEXITED(status) or WIFSIGNALED(status): q[r].exitFlag = true q[r].exitStatus = status + rexit = r break else: let err = osLastError() if err == OSErrorCode(ECHILD): # some child exits, we need to check our childs exit codes - discard + for r in 0..m-1: + if (not isNil(q[r])) and (not running(q[r])): + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break elif err == OSErrorCode(EINTR): # signal interrupted our syscall, lets repeat it continue @@ -298,29 +305,27 @@ proc execProcesses*(cmds: openArray[string], # all other errors are exceptions raiseOSError(err) - for r in 0..m-1: - if not isNil(q[r]): - if not running(q[r]): - result = max(result, q[r].peekExitCode()) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - close(q[r]) - if i < len(cmds): - if beforeRunEvent != nil: beforeRunEvent(i) - q[r] = startProcess(cmds[i], + if rexit >= 0: + result = max(result, q[rexit].peekExitCode()) + if afterRunEvent != nil: afterRunEvent(rexit, q[rexit]) + close(q[rexit]) + if i < len(cmds): + if beforeRunEvent != nil: beforeRunEvent(i) + q[rexit] = startProcess(cmds[i], options = options + {poEvalCommand}) - when defined(windows): - w[r] = q[r].fProcessHandle - inc(i) - else: - when defined(windows): - for k in 0..wcount - 1: - if w[k] == q[r].fProcessHandle: - w[k] = w[wcount - 1] - w[wcount - 1] = 0 - dec(wcount) - break - q[r] = nil - dec(ecount) + when defined(windows): + w[rexit] = q[rexit].fProcessHandle + inc(i) + else: + when defined(windows): + for k in 0..wcount - 1: + if w[k] == q[rexit].fProcessHandle: + w[k] = w[wcount - 1] + w[wcount - 1] = 0 + dec(wcount) + break + q[rexit] = nil + dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: From 1a5a09b83504e2b39de29ef13a92d5ca57c71906 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 12 Dec 2017 19:35:40 +0100 Subject: [PATCH 015/200] make tfragment_gc more robust --- tests/fragmentation/tfragment_gc.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 6e0ec37cea..1781f66105 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -20,7 +20,10 @@ let total = getTotalMem() # Concrete values on Win64: 58.152MiB / 188.285MiB -echo "occupied ok: ", occ < 60 * 1024 * 1024 +let occupiedOk = occ < 64 * 1024 * 1024 +if not occupiedOk: + echo "occupied ", formatSize(occ) +echo "occupied ok: ", occupiedOk let totalOk = total < 210 * 1024 * 1024 if not totalOk: echo "total peak memory ", formatSize(total) From 6f8e98cff2c7fb99325042a811751dc21e972ee3 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 12 Dec 2017 19:55:46 +0100 Subject: [PATCH 016/200] improve the docs for tables.add --- lib/pure/collections/sharedtables.nim | 1 + lib/pure/collections/tables.nim | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index fc50ea41c7..211a6ce6ac 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -183,6 +183,7 @@ proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 01a42efab4..48f8eed67f 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -308,6 +308,7 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = proc add*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! addImpl(enlarge) proc len*[A, B](t: TableRef[A, B]): int = @@ -430,6 +431,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = proc add*[A, B](t: TableRef[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! t[].add(key, val) proc del*[A, B](t: TableRef[A, B], key: A) = @@ -604,6 +606,7 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! addImpl(enlarge) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = @@ -770,6 +773,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! t[].add(key, val) proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = From e952ada1ba81675b0f6d22e76012afe290b4356c Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 13 Dec 2017 00:36:14 +0200 Subject: [PATCH 017/200] Fix --- lib/pure/osproc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index d72ed1772c..f0542ea980 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -315,7 +315,7 @@ proc execProcesses*(cmds: openArray[string], options = options + {poEvalCommand}) when defined(windows): w[rexit] = q[rexit].fProcessHandle - inc(i) + inc(i) else: when defined(windows): for k in 0..wcount - 1: From 542d45f8826ec3566108ce6e2c3b456c7798af58 Mon Sep 17 00:00:00 2001 From: GULPF Date: Wed, 13 Dec 2017 02:52:35 +0100 Subject: [PATCH 018/200] Fix counttable smallest (#6912) --- lib/pure/collections/tables.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 48f8eed67f..28fbaa6323 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -993,9 +993,10 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) = proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) assert t.len > 0 - var minIdx = 0 + var minIdx = -1 for h in 1..high(t.data): - if t.data[h].val > 0 and t.data[minIdx].val > t.data[h].val: minIdx = h + if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val): + minIdx = h result.key = t.data[minIdx].key result.val = t.data[minIdx].val @@ -1329,3 +1330,7 @@ when isMainModule: assert((a == b) == true) assert((b == a) == true) + block: # CountTable.smallest + var t = initCountTable[int]() + for v in items([4, 4, 5, 5, 5]): t.inc(v) + doAssert t.smallest == (4, 2) From c35788b97c1d5dffbb5bb7948946029bce2527bf Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 13 Dec 2017 14:37:19 +0100 Subject: [PATCH 019/200] make hidden visibility the default for Unix --- compiler/ccgtypes.nim | 2 ++ lib/nimbase.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index cfa2afdd95..0c7e60eac9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -925,6 +925,8 @@ proc genProcHeader(m: BModule, prc: PSym): Rope = result.add "N_LIB_EXPORT " elif prc.typ.callConv == ccInline: result.add "static " + else: + result.add "N_LIB_PRIVATE " var check = initIntSet() fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) genProcParams(m, prc.typ, rettype, params, check) diff --git a/lib/nimbase.h b/lib/nimbase.h index ac2cc097c1..b12d8e34dd 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -159,6 +159,7 @@ __clang__ /* ------------------------------------------------------------------- */ #if defined(WIN32) || defined(_WIN32) /* only Windows has this mess... */ +# define N_LIB_PRIVATE # define N_CDECL(rettype, name) rettype __cdecl name # define N_STDCALL(rettype, name) rettype __stdcall name # define N_SYSCALL(rettype, name) rettype __syscall name @@ -178,6 +179,7 @@ __clang__ # endif # define N_LIB_IMPORT extern __declspec(dllimport) #else +# define N_LIB_PRIVATE __attribute__((visibility("hidden"))) # if defined(__GNUC__) # define N_CDECL(rettype, name) rettype name # define N_STDCALL(rettype, name) rettype name From 422c117a770010d9cf28631d4f6e73eea58117b5 Mon Sep 17 00:00:00 2001 From: konqoro Date: Wed, 13 Dec 2017 16:14:01 +0200 Subject: [PATCH 020/200] Small fix for js dom --- lib/js/dom.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index cdefc772c5..aa7f5d8396 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -134,9 +134,9 @@ type # https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement HtmlElement* = ref object of Element - contentEditable*: string + contentEditable*: cstring isContentEditable*: bool - dir*: string + dir*: cstring offsetHeight*: int offsetWidth*: int offsetLeft*: int From be16dfd19576ac972031006bbfdb1c881ab4fdfd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 13 Dec 2017 22:16:37 +0100 Subject: [PATCH 021/200] make tests green again --- compiler/ccgtypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 0c7e60eac9..c9cd3b1250 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -925,7 +925,7 @@ proc genProcHeader(m: BModule, prc: PSym): Rope = result.add "N_LIB_EXPORT " elif prc.typ.callConv == ccInline: result.add "static " - else: + elif {sfImportc, sfExportc} * prc.flags == {}: result.add "N_LIB_PRIVATE " var check = initIntSet() fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) From 9e87531f041525b23ca12c7886412d762930005a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 14 Dec 2017 03:23:47 -0600 Subject: [PATCH 022/200] Genode: constrain `osTryAllocPages` to RAM quota (#6883) Genode software components all start with an explicit RAM resource quota which may or may not be upgraded during runtime by the parent process. With this patch `osTryAllocPages` will fail if allocation exceeds quotas set by the parent and the `osAllocPages` procedure will trigger a blocking request to the parent to increase quotas. The previous behavior could potentially block both procedures indefinitely for a quota upgrade rather than fail and trigger garbage collection. This patch also adds tracking of Genode dataspace mappings into the component address space so they can be detached and freed. --- lib/system/genodealloc.nim | 114 +++++++++++++++++++++++++++++++++++++ lib/system/osalloc.nim | 12 +--- 2 files changed, 115 insertions(+), 11 deletions(-) create mode 100644 lib/system/genodealloc.nim diff --git a/lib/system/genodealloc.nim b/lib/system/genodealloc.nim new file mode 100644 index 0000000000..3646a842d8 --- /dev/null +++ b/lib/system/genodealloc.nim @@ -0,0 +1,114 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level dataspace allocator for Genode. + +when not defined(genode): + {.error: "Genode only module".} + +type DataspaceCapability {. + importcpp: "Genode::Dataspace_capability", pure.} = object + +type + Map = object + attachment: pointer + size: int + ds: DataspaceCapability + + SlabMeta = object + next: ptr MapSlab + ds: DataspaceCapability + + MapSlab = object + meta: SlabMeta + maps: array[1,Map] + +const SlabBackendSize = 4096 + +proc ramAvail(): int {. + importcpp: "genodeEnv->pd().avail_ram().value".} + ## Return number of bytes available for allocation. + +proc capsAvail(): int {. + importcpp: "genodeEnv->pd().avail_caps().value".} + ## Return the number of available capabilities. + ## Each dataspace allocation consumes a capability. + +proc allocDataspace(size: int): DataspaceCapability {. + importcpp: "genodeEnv->pd().alloc(@)".} + ## Allocate a dataspace and its capability. + +proc attachDataspace(ds: DataspaceCapability): pointer {. + importcpp: "genodeEnv->rm().attach(@)".} + ## Attach a dataspace into the component address-space. + +proc detachAddress(p: pointer) {. + importcpp: "genodeEnv->rm().detach(@)".} + ## Detach a dataspace from the component address-space. + +proc freeDataspace(ds: DataspaceCapability) {. + importcpp: "genodeEnv->pd().free(@)".} + ## Free a dataspace. + +proc newMapSlab(): ptr MapSlab = + let + ds = allocDataspace SlabBackendSize + p = attachDataspace ds + result = cast[ptr MapSlab](p) + result.meta.ds = ds + +iterator items(s: ptr MapSlab): ptr Map = + let mapCount = (SlabBackendSize - sizeof(SlabMeta)) div sizeof(Map) + for i in 0 .. = size and capsAvail() > 1: + result = osAllocPages size + +proc osDeallocPages(p: pointer; size: int) = + var slab = slabs + while not slab.isNil: + # lookup first free spot in slabs + for m in slab.items: + if m.attachment == p: + if m.size != size: + echo "cannot partially detach dataspace" + quit -1 + detachAddress m.attachment + freeDataspace m.ds + m[] = Map() + return + slab = slab.meta.next diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 444f113069..1ad4cf6952 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -78,17 +78,7 @@ when defined(emscripten): munmap(mmapDescr.realPointer, mmapDescr.realSize) elif defined(genode): - - proc osAllocPages(size: int): pointer {. - importcpp: "genodeEnv->rm().attach(genodeEnv->ram().alloc(@))".} - - proc osTryAllocPages(size: int): pointer = - {.emit: """try {""".} - result = osAllocPages size - {.emit: """} catch (...) { }""".} - - proc osDeallocPages(p: pointer, size: int) {. - importcpp: "genodeEnv->rm().detach(#)".} + include genodealloc # osAllocPages, osTryAllocPages, osDeallocPages elif defined(posix): const From e860334377365d903b607583e6253209a356ed08 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 14 Dec 2017 12:46:09 +0100 Subject: [PATCH 023/200] added SQL parser test --- tests/stdlib/somesql.sql | 298 ++++++++++++++++++++++++++++++++++++ tests/stdlib/tsqlparser.nim | 12 ++ 2 files changed, 310 insertions(+) create mode 100644 tests/stdlib/somesql.sql create mode 100644 tests/stdlib/tsqlparser.nim diff --git a/tests/stdlib/somesql.sql b/tests/stdlib/somesql.sql new file mode 100644 index 0000000000..285f93cec3 --- /dev/null +++ b/tests/stdlib/somesql.sql @@ -0,0 +1,298 @@ +create table anon40( + anon41 anon42 primary key default anon43(), + anon44 text unique not null, + anon45 text unique not null, + anon46 text not null, + anon47 text not null, + anon48 text default null, + anon49 text default null, + anon50 text default null, + anon51 text default null, + anon52 text default null, + anon53 text default null, + anon54 text default null, + anon55 text default null, + anon56 text default null, + anon57 text default null, + anon58 text default null, + anon59 text default null, + anon60 text default null, + anon61 text default null, + anon62 varchar(30) default null, + anon63 varchar(30) default null); +create table anon64( + anon41 serial primary key, + anon65 varchar(30) not null unique, + anon46 varchar(30) not null, + anon66 varchar(30) not null, + anon47 varchar(30) not null, + anon67 text not null, + anon55 varchar(30) not null unique, + anon68 varchar(30) default 'o', + anon69 boolean default true, + anon70 int not null references anon40(anon41)); +create table anon71( + anon72 varchar(30) not null primary key, + anon73 varchar(30) not null unique, + anon70 int not null references anon40(anon41)); +create table anon74( + anon72 varchar(30) not null primary key, + anon73 varchar(30) not null unique, + anon75 varchar(30) not null, + anon70 int not null references anon40(anon41), + foreign key(anon75) references anon71(anon72)); +create table anon76( + anon41 serial primary key, + anon72 varchar(30) not null unique, + anon73 varchar(30) not null unique, + anon77 varchar(30) not null, + anon70 int not null references anon40(anon41), + foreign key(anon77) references anon74(anon72)); +create table anon78( + anon41 serial primary key, + anon72 varchar(30) not null unique, + anon73 varchar(30) not null unique, + anon79 int not null, + anon80 varchar(30) default null, + anon81 int not null, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41), + foreign key(anon79) references anon78(anon41), + foreign key(anon81) references anon76(anon41)); +create table anon82( + anon41 serial primary key, + anon72 varchar(30) not null unique, + anon73 text not null unique, + anon79 int not null, + anon80 text default null, + anon83 varchar(30) not null default 'd', + anon84 decimal default 0.00, + anon69 boolean not null default true, + anon85 decimal default 0.00, + anon86 decimal default 0.00, + anon87 decimal default 0.00, + anon70 int not null references anon40(anon41), + foreign key(anon79) references anon78(anon41)); +create table anon88( + anon41 serial primary key, + anon72 varchar(30) not null unique, + anon80 text default '', + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon89( + anon90 int not null primary key, + anon91 anon92 default 0.00, + anon93 varchar(30), + anon69 boolean not null default true, + anon70 int not null references anon40(anon41), + foreign key(anon90) references anon82(anon41)); +create table anon94( + anon41 serial primary key, + anon73 text unique not null, + anon80 text default null, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon95( + anon41 serial primary key, + anon73 text unique not null, + anon96 int not null references anon94(anon41), + anon80 text default null, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon97( + anon41 serial primary key, + anon73 text unique not null, + anon98 int not null references anon95(anon41), + anon80 text default null, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon99( + anon41 serial primary key, + anon73 varchar(30) unique not null, + anon100 varchar(30) default null, + anon101 anon102 default 0, + anon103 varchar(30) default 'g', + anon104 int not null, + anon105 decimal not null default 1, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon106( + anon107 varchar(30) default 'g', + anon108 int references anon99(anon41) not null, + anon109 decimal default 1, + anon110 int references anon99(anon41) not null, + anon70 int not null references anon40(anon41)); +create table anon111( + anon41 serial primary key, + anon112 text unique not null, + anon73 text unique not null, + anon113 anon102 references anon97(anon41) not null, + anon114 varchar(30) not null, + anon115 int not null references anon88(anon41), + anon116 int not null references anon82(anon41), + anon117 int not null references anon82(anon41), + anon118 int not null references anon82(anon41), + anon119 int not null references anon82(anon41), + anon120 int not null references anon82(anon41), + anon121 int not null references anon82(anon41), + anon122 int references anon99(anon41) not null, + anon123 decimal default 0.00, + anon124 decimal default 0.00, + anon69 boolean default true, + anon70 int not null references anon40(anon41)); +create table anon125( + anon41 serial primary key, + anon126 int references anon111(anon41) not null, + anon80 text not null, + anon127 varchar(30) not null, + anon128 decimal default 0.00, + anon129 decimal default 0, + anon130 decimal default 0, + anon131 decimal default 0, + anon132 decimal default 0, + anon133 decimal default 0.00, + anon134 decimal default 0.00, + anon135 decimal default 0.00, + anon70 int not null references anon40(anon41), constraint anon136 check anon137(anon126, anon127, anon129)); +create table anon138( + anon41 serial primary key, + anon126 int references anon111(anon41) not null, + anon80 text not null, + anon127 varchar(30) not null, + anon139 date not null, + anon129 decimal default 0, + anon130 decimal default 0, + anon131 decimal default 0, + anon132 decimal default 0, + anon70 int not null references anon40(anon41), constraint anon136 check anon137(anon127, anon129)); +create table anon140( + anon41 serial primary key, + anon141 text unique not null, + anon46 text default null, + anon47 text default null, + anon57 varchar(30) default null, + anon142 text default null, + anon51 text default null, + anon143 varchar(30) default null, + anon53 text default null, + anon54 text default null, + anon55 text default null, + anon45 text default null, + anon69 boolean default true, + anon70 int not null references anon40(anon41)); +create table anon144( + anon41 serial primary key, + anon72 varchar(30) unique not null, + anon73 varchar(30) unique not null, + anon80 varchar(30) default null, + anon69 boolean default true, + anon70 int not null references anon40(anon41)); +create table anon145( + anon41 serial primary key, + anon72 varchar(30) unique not null, + anon73 varchar(30) unique not null, + anon146 int not null, + anon147 anon92 default 1, + anon148 anon92 default 9999999, + anon80 varchar(30) default null, + anon69 boolean default true, + anon149 int default 0, + anon150 int not null, + anon151 anon92 default 0, + anon70 int not null references anon40(anon41), + foreign key(anon150) references anon82(anon41), + foreign key(anon146) references anon144(anon41)); +create table anon152( + anon41 serial primary key, + anon73 varchar(30) not null unique, + anon153 varchar(30) not null unique, + anon80 text default null, + anon69 boolean not null default true, + anon70 int not null references anon40(anon41)); +create table anon154( + anon41 serial primary key not null, + anon155 int not null unique, + date date default anon156 not null, + anon157 anon102 references anon140(anon41) not null, + anon158 anon102 references anon64(anon41) not null, + anon159 decimal default 0 not null, + anon160 decimal default 0 not null, + anon161 decimal default 0 not null, + anon162 decimal default 0 not null, + anon163 decimal default 0 not null, + anon164 decimal default 0 not null, + anon165 decimal default 0.00, + anon166 decimal default 0 not null, + anon167 decimal default 0.00, + anon168 decimal default 0 not null, + anon169 boolean default false, + anon170 varchar(30) default 'ca', + anon171 varchar(30) default 'n', + anon172 varchar(30) not null default 'd', + anon173 decimal default 0.00, + anon174 decimal default 0.00, + anon175 int, + anon176 varchar(30) default null, + anon177 varchar(30) default '', + anon178 varchar(30) default null, + anon70 int not null references anon40(anon41)); +create table anon179( + anon41 serial primary key not null, + anon180 anon102 references anon154(anon41) not null, + anon181 int references anon125(anon41) not null, + anon182 int references anon82(anon41) not null, + anon122 int references anon99(anon41) not null, + anon183 decimal not null, + anon184 decimal default 0.00, + anon174 decimal default 0, + anon160 decimal default 0.00, + anon185 decimal default 0, + anon162 decimal default 0.00, + anon186 decimal default 0, + anon163 decimal default 0.00, + anon187 decimal default 0, + anon164 decimal default 0.00, + anon188 decimal default 0, + anon161 decimal default 0.00, + anon189 decimal default 0.00, + anon168 decimal default 0.00, + anon190 decimal not null, + anon191 decimal default 0, + anon83 varchar(30) not null default 't', + anon192 decimal default 0, + anon193 decimal not null, + anon194 decimal not null, + anon70 int not null references anon40(anon41)); +create table anon195( + anon41 serial not null, + anon196 int not null, + anon175 char not null, + anon90 int not null references anon82, + anon165 decimal default 0.00, + anon70 int not null references anon40(anon41), primary key(anon196, anon175)); +create table anon197( + anon41 serial not null, + anon196 int not null, + anon175 char not null, + anon198 int not null, + anon189 decimal default 0.00, + anon199 varchar(30) default null, + anon200 varchar(30) default null, + anon70 int not null references anon40(anon41), + primary key(anon196, anon175), + foreign key(anon198) references anon145(anon41)); +create table anon201( + anon41 serial primary key, + anon202 varchar(30) not null, + anon203 varchar(30) not null, + anon204 varchar(30) not null, + anon205 varchar(30) not null, + anon206 boolean default null, + anon70 int not null references anon40(anon41)); +create table anon207( + anon41 serial primary key, + anon208 varchar(30) not null, + anon209 varchar(30) not null, + anon204 varchar(30) default null, + anon70 int not null references anon40(anon41)); + diff --git a/tests/stdlib/tsqlparser.nim b/tests/stdlib/tsqlparser.nim new file mode 100644 index 0000000000..4a7b2f7d7f --- /dev/null +++ b/tests/stdlib/tsqlparser.nim @@ -0,0 +1,12 @@ +discard """ + output: '''true''' +""" + +# Just check that we can parse 'somesql' and render it without crashes. + +import parsesql, streams, os + +var tree = parseSql(newFileStream(getAppDir() / "somesql.sql"), "somesql") +discard renderSql(tree) + +echo "true" From 6df6ec27ec573fc7f619f7bf9fece6d6b0dc931f Mon Sep 17 00:00:00 2001 From: Fabian Keller Date: Thu, 14 Dec 2017 14:02:13 +0100 Subject: [PATCH 024/200] Improved collection-to-string behavior (#6825) --- changelog.md | 3 + lib/pure/collections/critbits.nim | 6 +- lib/pure/collections/deques.nim | 2 +- lib/pure/collections/lists.nim | 2 +- lib/pure/collections/sets.nim | 2 +- lib/pure/collections/tables.nim | 4 +- lib/pure/strutils.nim | 24 +---- lib/system.nim | 73 +++++++++++-- tests/array/troof1.nim | 2 +- tests/ccgbugs/t6279.nim | 2 +- tests/ccgbugs/tbug1081.nim | 2 +- tests/ccgbugs/tconstobj.nim | 2 +- tests/ccgbugs/tobjconstr_regression.nim | 2 +- tests/collections/tcollections_to_string.nim | 106 +++++++++++++++++++ tests/collections/ttables.nim | 2 +- tests/collections/ttablesref.nim | 4 +- tests/concepts/t3330.nim | 2 +- tests/exprs/tstmtexprs.nim | 2 +- tests/fields/timplicitfieldswithpartial.nim | 2 +- tests/generics/treentranttypes.nim | 4 +- tests/js/trefbyvar.nim | 2 +- tests/metatype/ttypedesc2.nim | 2 +- tests/misc/tlocals.nim | 2 +- tests/objects/tobjconstr.nim | 20 ++-- tests/showoff/thello2.nim | 2 +- tests/stdlib/tlists.nim | 2 +- tests/stdlib/treloop.nim | 2 +- tests/system/toString.nim | 8 +- tests/typerel/typeof_in_template.nim | 2 +- tests/types/tinheritpartialgeneric.nim | 4 +- tests/types/tparameterizedparent2.nim | 2 +- tests/vm/tableinstatic.nim | 2 +- tests/vm/tconstobj.nim | 2 +- 33 files changed, 226 insertions(+), 74 deletions(-) create mode 100644 tests/collections/tcollections_to_string.nim diff --git a/changelog.md b/changelog.md index 94cea7dbf4..6065ccd101 100644 --- a/changelog.md +++ b/changelog.md @@ -123,3 +123,6 @@ This now needs to be written as: to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and ``dealloc`` became O(1) operations. +- The behavior of ``$`` has been changed for all standard library collections. The + collection-to-string implementations now perform proper quoting and escaping of + strings and chars. diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 19f1f2e58d..34f5c54708 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -141,8 +141,8 @@ proc excl*[T](c: var CritBitTree[T], key: string) = proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = ## Returns true iff `c` does not contain the given `key`. If the key - ## does exist, c.excl(key) is performed. - let oldCount = c.count + ## does exist, c.excl(key) is performed. + let oldCount = c.count var n = exclImpl(c, key) result = c.count == oldCount @@ -326,7 +326,7 @@ proc `$`*[T](c: CritBitTree[T]): string = result.add($key) when T isnot void: result.add(": ") - result.add($val) + result.addQuoted(val) result.add("}") when isMainModule: diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 1e0cb82d2d..328308a9b3 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -185,7 +185,7 @@ proc `$`*[T](deq: Deque[T]): string = result = "[" for x in deq: if result.len > 1: result.add(", ") - result.add($x) + result.addQuoted(x) result.add("]") when isMainModule: diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 560273dfa3..e69acc8d95 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -135,7 +135,7 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string = result = "[" for x in nodes(L): if result.len > 1: result.add(", ") - result.add($x.value) + result.addQuoted(x.value) result.add("]") proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index f936b3ecaa..9e9152fc8c 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -406,7 +406,7 @@ template dollarImpl() {.dirty.} = result = "{" for key in items(s): if result.len > 1: result.add(", ") - result.add($key) + result.addQuoted(key) result.add("}") proc `$`*[A](s: HashSet[A]): string = diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 28fbaa6323..38f8f97f5e 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -338,9 +338,9 @@ template dollarImpl(): untyped {.dirty.} = result = "{" for key, val in pairs(t): if result.len > 1: result.add(", ") - result.add($key) + result.addQuoted(key) result.add(": ") - result.add($val) + result.addQuoted(val) result.add("}") proc `$`*[A, B](t: Table[A, B]): string = diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 62ceaa2e8b..dbb4db7811 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1761,29 +1761,15 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. + ## Escapes a string `s`. See `system.addEscapedChar `_ + ## for the escaping scheme. ## - ## This does these operations (at the same time): - ## * replaces any ``\`` by ``\\`` - ## * replaces any ``'`` by ``\'`` - ## * replaces any ``"`` by ``\"`` - ## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}`` - ## by ``\xHH`` where ``HH`` is its hexadecimal value. - ## The procedure has been designed so that its output is usable for many - ## different common syntaxes. The resulting string is prefixed with - ## `prefix` and suffixed with `suffix`. Both may be empty strings. - ## **Note**: This is not correct for producing Ansi C code! + ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. + ## Both may be empty strings. result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): - case c - of '\0'..'\31', '\127'..'\255': - add(result, "\\x") - add(result, toHex(ord(c), 2)) - of '\\': add(result, "\\\\") - of '\'': add(result, "\\'") - of '\"': add(result, "\\\"") - else: add(result, c) + result.addEscapedChar(c) add(result, suffix) proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, diff --git a/lib/system.nim b/lib/system.nim index b9f01c3065..e66699ae4a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1325,6 +1325,7 @@ proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} ## tmp.add("cd") ## assert(tmp == "abcd") + type Endianness* = enum ## is a type describing the endianness of a processor. littleEndian, bigEndian @@ -2503,9 +2504,9 @@ proc `$`*[T: tuple|object](x: T): string = when compiles($value): when compiles(value.isNil): if value.isNil: result.add "nil" - else: result.add($value) + else: result.addQuoted(value) else: - result.add($value) + result.addQuoted(value) firstElement = false else: result.add("...") @@ -2525,12 +2526,9 @@ proc collectionToString[T](x: T, prefix, separator, suffix: string): string = if value.isNil: result.add "nil" else: - result.add($value) - # prevent temporary string allocation - elif compiles(result.add(value)): - result.add(value) + result.addQuoted(value) else: - result.add($value) + result.addQuoted(value) result.add(suffix) @@ -3893,6 +3891,65 @@ proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime. when declared(initDebugger): initDebugger() +proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = + ## Adds a char to string `s` and applies the following escaping: + ## + ## * replaces any ``\`` by ``\\`` + ## * replaces any ``'`` by ``\'`` + ## * replaces any ``"`` by ``\"`` + ## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}`` + ## by ``\xHH`` where ``HH`` is its hexadecimal value. + ## + ## The procedure has been designed so that its output is usable for many + ## different common syntaxes. + ## **Note**: This is not correct for producing Ansi C code! + case c + of '\0'..'\31', '\127'..'\255': + add(s, "\\x") + const HexChars = "0123456789ABCDEF" + let n = ord(c) + s.add(HexChars[int((n and 0xF0) shr 4)]) + s.add(HexChars[int(n and 0xF)]) + of '\\': add(s, "\\\\") + of '\'': add(s, "\\'") + of '\"': add(s, "\\\"") + else: add(s, c) + +proc addQuoted*[T](s: var string, x: T) = + ## Appends `x` to string `s` in place, applying quoting and escaping + ## if `x` is a string or char. See + ## `addEscapedChar `_ + ## for the escaping scheme. + ## + ## The Nim standard library uses this function on the elements of + ## collections when producing a string representation of a collection. + ## It is recommended to use this function as well for user-side collections. + ## Users may overload `addQuoted` for custom (string-like) types if + ## they want to implement a customized element representation. + ## + ## .. code-block:: Nim + ## var tmp = "" + ## tmp.addQuoted(1) + ## tmp.add(", ") + ## tmp.addQuoted("string") + ## tmp.add(", ") + ## tmp.addQuoted('c') + ## assert(tmp == """1, "string", 'c'""") + when T is string: + s.add("\"") + for c in x: + s.addEscapedChar(c) + s.add("\"") + elif T is char: + s.add("'") + s.addEscapedChar(x) + s.add("'") + # prevent temporary string allocation + elif compiles(s.add(x)): + s.add(x) + else: + s.add($x) + when hasAlloc: # XXX: make these the default (or implement the NilObject optimization) proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect.} = @@ -4034,4 +4091,4 @@ template doAssertRaises*(exception, code: untyped): typed = except Exception as exc: raiseAssert(astToStr(exception) & " wasn't raised, another error was raised instead by:\n"& - astToStr(code)) \ No newline at end of file + astToStr(code)) diff --git a/tests/array/troof1.nim b/tests/array/troof1.nim index 594ad98a57..b486c34482 100644 --- a/tests/array/troof1.nim +++ b/tests/array/troof1.nim @@ -4,7 +4,7 @@ discard """ 3 @[(Field0: 1, Field1: 2), (Field0: 3, Field1: 5)] 2 -@[a, new one, c] +@["a", "new one", "c"] @[1, 2, 3]''' """ diff --git a/tests/ccgbugs/t6279.nim b/tests/ccgbugs/t6279.nim index 37345d807b..5a3d6768cc 100644 --- a/tests/ccgbugs/t6279.nim +++ b/tests/ccgbugs/t6279.nim @@ -1,6 +1,6 @@ discard """ cmd: "nim c -r -d:fulldebug -d:smokeCycles --gc:refc $file" -output: '''@[a]''' +output: '''@["a"]''' """ # bug #6279 diff --git a/tests/ccgbugs/tbug1081.nim b/tests/ccgbugs/tbug1081.nim index c9a9e6aa47..baef99d844 100644 --- a/tests/ccgbugs/tbug1081.nim +++ b/tests/ccgbugs/tbug1081.nim @@ -3,7 +3,7 @@ discard """ 0 0 0 -x = [a, b, c, 0, 1, 2, 3, 4, 5, 6] and y = [a, b, c, 0, 1, 2, 3, 4, 5, 6]''' +x = ['a', 'b', 'c', '0', '1', '2', '3', '4', '5', '6'] and y = ['a', 'b', 'c', '0', '1', '2', '3', '4', '5', '6']''' """ proc `1/1`() = echo(1 div 1) diff --git a/tests/ccgbugs/tconstobj.nim b/tests/ccgbugs/tconstobj.nim index 98f441e83d..51cf661ee8 100644 --- a/tests/ccgbugs/tconstobj.nim +++ b/tests/ccgbugs/tconstobj.nim @@ -1,5 +1,5 @@ discard """ - output: '''(FirstName: James, LastName: Franco)''' + output: '''(FirstName: "James", LastName: "Franco")''' """ # bug #1547 diff --git a/tests/ccgbugs/tobjconstr_regression.nim b/tests/ccgbugs/tobjconstr_regression.nim index 87d0378940..d29abad974 100644 --- a/tests/ccgbugs/tobjconstr_regression.nim +++ b/tests/ccgbugs/tobjconstr_regression.nim @@ -1,5 +1,5 @@ discard """ - output: "@[(username: user, role: admin, description: desc, email_addr: email), (username: user, role: admin, description: desc, email_addr: email)]" + output: '''@[(username: "user", role: "admin", description: "desc", email_addr: "email"), (username: "user", role: "admin", description: "desc", email_addr: "email")]''' """ type diff --git a/tests/collections/tcollections_to_string.nim b/tests/collections/tcollections_to_string.nim new file mode 100644 index 0000000000..6cc8a84ff0 --- /dev/null +++ b/tests/collections/tcollections_to_string.nim @@ -0,0 +1,106 @@ +discard """ + exitcode: 0 + output: "" +""" +import sets +import tables +import deques +import lists +import critbits + +# Tests for tuples +doAssert $(1, 2, 3) == "(Field0: 1, Field1: 2, Field2: 3)" +doAssert $("1", "2", "3") == """(Field0: "1", Field1: "2", Field2: "3")""" +doAssert $('1', '2', '3') == """(Field0: '1', Field1: '2', Field2: '3')""" + +# Tests for seqs +doAssert $(@[1, 2, 3]) == "@[1, 2, 3]" +doAssert $(@["1", "2", "3"]) == """@["1", "2", "3"]""" +doAssert $(@['1', '2', '3']) == """@['1', '2', '3']""" + +# Tests for sets +doAssert $(toSet([1])) == "{1}" +doAssert $(toSet(["1"])) == """{"1"}""" +doAssert $(toSet(['1'])) == """{'1'}""" +doAssert $(toOrderedSet([1, 2, 3])) == "{1, 2, 3}" +doAssert $(toOrderedSet(["1", "2", "3"])) == """{"1", "2", "3"}""" +doAssert $(toOrderedSet(['1', '2', '3'])) == """{'1', '2', '3'}""" + +# Tests for tables +doAssert $({1: "1", 2: "2"}.toTable) == """{1: "1", 2: "2"}""" +doAssert $({"1": 1, "2": 2}.toTable) == """{"1": 1, "2": 2}""" + +# Tests for deques +block: + var d = initDeque[int]() + d.addLast(1) + doAssert $d == "[1]" +block: + var d = initDeque[string]() + d.addLast("1") + doAssert $d == """["1"]""" +block: + var d = initDeque[char]() + d.addLast('1') + doAssert $d == "['1']" + +# Tests for lists +block: + var l = initDoublyLinkedList[int]() + l.append(1) + l.append(2) + l.append(3) + doAssert $l == "[1, 2, 3]" +block: + var l = initDoublyLinkedList[string]() + l.append("1") + l.append("2") + l.append("3") + doAssert $l == """["1", "2", "3"]""" +block: + var l = initDoublyLinkedList[char]() + l.append('1') + l.append('2') + l.append('3') + doAssert $l == """['1', '2', '3']""" + +# Tests for critbits +block: + var t: CritBitTree[int] + t["a"] = 1 + doAssert $t == "{a: 1}" +block: + var t: CritBitTree[string] + t["a"] = "1" + doAssert $t == """{a: "1"}""" +block: + var t: CritBitTree[char] + t["a"] = '1' + doAssert $t == "{a: '1'}" + + +# Test escaping behavior +block: + var s = "" + s.addQuoted('\0') + s.addQuoted('\31') + s.addQuoted('\127') + s.addQuoted('\255') + doAssert s == "'\\x00''\\x1F''\\x7F''\\xFF'" +block: + var s = "" + s.addQuoted('\\') + s.addQuoted('\'') + s.addQuoted('\"') + doAssert s == """'\\''\'''\"'""" + +# Test customized element representation +type CustomString = object + +proc addQuoted(s: var string, x: CustomString) = + s.add("") + +block: + let s = @[CustomString()] + doAssert $s == "@[]" + diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 01dab44fca..2b8af5bd97 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -48,7 +48,7 @@ block tableTest1: for y in 0..1: assert t[(x,y)] == $x & $y assert($t == - "{(x: 0, y: 1): 01, (x: 0, y: 0): 00, (x: 1, y: 0): 10, (x: 1, y: 1): 11}") + "{(x: 0, y: 1): \"01\", (x: 0, y: 0): \"00\", (x: 1, y: 0): \"10\", (x: 1, y: 1): \"11\"}") block tableTest2: var t = initTable[string, float]() diff --git a/tests/collections/ttablesref.nim b/tests/collections/ttablesref.nim index 12af1ccbb8..a4030e0dce 100644 --- a/tests/collections/ttablesref.nim +++ b/tests/collections/ttablesref.nim @@ -47,7 +47,7 @@ block tableTest1: for y in 0..1: assert t[(x,y)] == $x & $y assert($t == - "{(x: 0, y: 1): 01, (x: 0, y: 0): 00, (x: 1, y: 0): 10, (x: 1, y: 1): 11}") + "{(x: 0, y: 1): \"01\", (x: 0, y: 0): \"00\", (x: 1, y: 0): \"10\", (x: 1, y: 1): \"11\"}") block tableTest2: var t = newTable[string, float]() @@ -139,7 +139,7 @@ proc orderedTableSortTest() = block anonZipTest: let keys = @['a','b','c'] let values = @[1, 2, 3] - doAssert "{a: 1, b: 2, c: 3}" == $ toTable zip(keys, values) + doAssert "{'a': 1, 'b': 2, 'c': 3}" == $ toTable zip(keys, values) block clearTableTest: var t = newTable[string, float]() diff --git a/tests/concepts/t3330.nim b/tests/concepts/t3330.nim index fcd5054ef4..722c0a0e0a 100644 --- a/tests/concepts/t3330.nim +++ b/tests/concepts/t3330.nim @@ -6,10 +6,10 @@ but expected one of: proc test(foo: Foo[int]) t3330.nim(25, 8) Hint: Non-matching candidates for add(k, string, T) proc add(x: var string; y: string) +proc add(result: var string; x: float) proc add(x: var string; y: char) proc add(result: var string; x: int64) proc add(x: var string; y: cstring) -proc add(result: var string; x: float) proc add[T](x: var seq[T]; y: openArray[T]) proc add[T](x: var seq[T]; y: T) diff --git a/tests/exprs/tstmtexprs.nim b/tests/exprs/tstmtexprs.nim index 01f429b07f..9283f72682 100644 --- a/tests/exprs/tstmtexprs.nim +++ b/tests/exprs/tstmtexprs.nim @@ -1,6 +1,6 @@ discard """ output: '''24 -(bar: bar) +(bar: "bar") 1244 6 abcdefghijklmnopqrstuvwxyz diff --git a/tests/fields/timplicitfieldswithpartial.nim b/tests/fields/timplicitfieldswithpartial.nim index a315cc5d0b..9378332577 100644 --- a/tests/fields/timplicitfieldswithpartial.nim +++ b/tests/fields/timplicitfieldswithpartial.nim @@ -1,5 +1,5 @@ discard """ - output: '''(foo: 38, other: string here) + output: '''(foo: 38, other: "string here") 43 100 90''' diff --git a/tests/generics/treentranttypes.nim b/tests/generics/treentranttypes.nim index 9b4774e9bc..2ef049ce2f 100644 --- a/tests/generics/treentranttypes.nim +++ b/tests/generics/treentranttypes.nim @@ -1,6 +1,6 @@ discard """ output: ''' -(Field0: 10, Field1: (Field0: test, Field1: 1.2)) +(Field0: 10, Field1: (Field0: "test", Field1: 1.2)) 3x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0], [2.0, 0.0, 5.0]] 2x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]] @@ -43,7 +43,7 @@ type Matrix*[M: static[int]; N: static[int]; T] = Vector[M, Vector[N, T]] - + proc arrayTest = # every kind of square matrix works just fine let mat_good: Matrix[3, 3, float] = [[0.0, 2.0, 3.0], diff --git a/tests/js/trefbyvar.nim b/tests/js/trefbyvar.nim index d440fcc644..5b168044ee 100644 --- a/tests/js/trefbyvar.nim +++ b/tests/js/trefbyvar.nim @@ -66,4 +66,4 @@ proc initTypeA1(a: int; b: string; c: pointer = nil): TypeA1 = result.c_impl = c let x = initTypeA1(1, "a") -doAssert($x == "(a_impl: 1, b_impl: a, c_impl: ...)") +doAssert($x == "(a_impl: 1, b_impl: \"a\", c_impl: ...)") diff --git a/tests/metatype/ttypedesc2.nim b/tests/metatype/ttypedesc2.nim index 7650a6f6b4..4b6cfe6bc7 100644 --- a/tests/metatype/ttypedesc2.nim +++ b/tests/metatype/ttypedesc2.nim @@ -1,5 +1,5 @@ discard """ - output: "(x: a)" + output: '''(x: 'a')''' """ type diff --git a/tests/misc/tlocals.nim b/tests/misc/tlocals.nim index 3e240d3c87..09b7432f57 100644 --- a/tests/misc/tlocals.nim +++ b/tests/misc/tlocals.nim @@ -1,5 +1,5 @@ discard """ - output: "(x: string here, a: 1)" + output: '''(x: "string here", a: 1)''' """ proc simple[T](a: T) = diff --git a/tests/objects/tobjconstr.nim b/tests/objects/tobjconstr.nim index 12478f6218..b7da176aae 100644 --- a/tests/objects/tobjconstr.nim +++ b/tests/objects/tobjconstr.nim @@ -1,14 +1,14 @@ discard """ - output: '''(k: kindA, a: (x: abc, z: [1, 1, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 2, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 3, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 4, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 5, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 6, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 7, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 8, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 9, 3]), method: ()) -(k: kindA, a: (x: abc, z: [1, 10, 3]), method: ()) + output: '''(k: kindA, a: (x: "abc", z: [1, 1, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 2, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 3, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 4, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 5, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 6, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 7, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 8, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 9, 3]), method: ()) +(k: kindA, a: (x: "abc", z: [1, 10, 3]), method: ()) (x: 123) (x: 123) (z: 89, y: 0, x: 128) diff --git a/tests/showoff/thello2.nim b/tests/showoff/thello2.nim index d2e2f62271..3ccb4e3be1 100644 --- a/tests/showoff/thello2.nim +++ b/tests/showoff/thello2.nim @@ -1,5 +1,5 @@ discard """ - output: '''(a: 3, b: 4, s: abc)''' + output: '''(a: 3, b: 4, s: "abc")''' """ type diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 4caa05c90d..37e73c53fa 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -17,7 +17,7 @@ block SinglyLinkedListTest1: block SinglyLinkedListTest2: var L: TSinglyLinkedList[string] for d in items(data): L.prepend($d) - assert($L == "[6, 5, 4, 3, 2, 1]") + assert($L == """["6", "5", "4", "3", "2", "1"]""") assert("4" in L) diff --git a/tests/stdlib/treloop.nim b/tests/stdlib/treloop.nim index 35236708cb..b4221525d2 100644 --- a/tests/stdlib/treloop.nim +++ b/tests/stdlib/treloop.nim @@ -1,5 +1,5 @@ discard """ - output: "@[(, +, 1, 2, )]" + output: '''@["(", "+", " 1", " 2", ")"]''' """ import re diff --git a/tests/system/toString.nim b/tests/system/toString.nim index 3e7fc7ddbd..ea9d6b05b8 100644 --- a/tests/system/toString.nim +++ b/tests/system/toString.nim @@ -4,12 +4,12 @@ discard """ doAssert "@[23, 45]" == $(@[23, 45]) doAssert "[32, 45]" == $([32, 45]) -doAssert "@[, foo, bar]" == $(@["", "foo", "bar"]) -doAssert "[, foo, bar]" == $(["", "foo", "bar"]) +doAssert """@["", "foo", "bar"]""" == $(@["", "foo", "bar"]) +doAssert """["", "foo", "bar"]""" == $(["", "foo", "bar"]) # bug #2395 let alphaSet: set[char] = {'a'..'c'} -doAssert "{a, b, c}" == $alphaSet +doAssert "{'a', 'b', 'c'}" == $alphaSet doAssert "2.3242" == $(2.3242) doAssert "2.982" == $(2.982) doAssert "123912.1" == $(123912.1) @@ -49,5 +49,5 @@ import strutils # array test let arr = ['H','e','l','l','o',' ','W','o','r','l','d','!','\0'] -doAssert $arr == "[H, e, l, l, o, , W, o, r, l, d, !, \0]" +doAssert $arr == "['H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!', '\\x00']" doAssert $cstring(unsafeAddr arr) == "Hello World!" diff --git a/tests/typerel/typeof_in_template.nim b/tests/typerel/typeof_in_template.nim index 9ec06f2e31..3724cc9943 100644 --- a/tests/typerel/typeof_in_template.nim +++ b/tests/typerel/typeof_in_template.nim @@ -1,5 +1,5 @@ discard """ - output: '''@[a, c]''' + output: '''@["a", "c"]''' """ # bug #3230 diff --git a/tests/types/tinheritpartialgeneric.nim b/tests/types/tinheritpartialgeneric.nim index a00df26fad..1845778bf3 100644 --- a/tests/types/tinheritpartialgeneric.nim +++ b/tests/types/tinheritpartialgeneric.nim @@ -1,6 +1,6 @@ discard """ - output: '''(c: hello, a: 10, b: 12.0) -(a: 15.5, b: hello) + output: '''(c: "hello", a: 10, b: 12.0) +(a: 15.5, b: "hello") (a: 11.75, b: 123)''' """ diff --git a/tests/types/tparameterizedparent2.nim b/tests/types/tparameterizedparent2.nim index 999db2ac58..e96b9edbeb 100644 --- a/tests/types/tparameterizedparent2.nim +++ b/tests/types/tparameterizedparent2.nim @@ -2,7 +2,7 @@ discard """ output: '''(width: 11, color: 13) (width: 15, weight: 13, taste: 11, color: 14) (width: 17, color: 16) -(width: 12.0, taste: yummy, color: 13) +(width: 12.0, taste: "yummy", color: 13) (width: 0, tast_e: 0.0, kind: Smooth, skin: 1.5, color: 12)''' """ # bug #5264 diff --git a/tests/vm/tableinstatic.nim b/tests/vm/tableinstatic.nim index 54e7c11f0e..b0d24b4771 100644 --- a/tests/vm/tableinstatic.nim +++ b/tests/vm/tableinstatic.nim @@ -2,7 +2,7 @@ discard """ nimout: '''0 0 0 -{hallo: 123, welt: 456}''' +{"hallo": "123", "welt": "456"}''' """ import tables diff --git a/tests/vm/tconstobj.nim b/tests/vm/tconstobj.nim index 51f30fb78f..38fcdd844f 100644 --- a/tests/vm/tconstobj.nim +++ b/tests/vm/tconstobj.nim @@ -1,5 +1,5 @@ discard """ - output: '''(name: hello) + output: '''(name: "hello") (-1, 0)''' """ From c6b33de127ada9d715c16c7215f88cde7bb5a0c6 Mon Sep 17 00:00:00 2001 From: treeform Date: Wed, 13 Dec 2017 23:32:54 +0000 Subject: [PATCH 025/200] fix --- lib/pure/parsesql.nim | 415 +++++++++++++++++++++-------------- tests/stdlib/tparsesql.nim | 438 +++++++++++++++++++++++++++++++++++++ 2 files changed, 694 insertions(+), 159 deletions(-) create mode 100644 tests/stdlib/tparsesql.nim diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 6891e2ff74..b53f46f82c 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -462,27 +462,27 @@ proc errorStr(L: SqlLexer, msg: string): string = # ----------------------------- parser ---------------------------------------- -# Operator/Element Associativity Description -# . left table/column name separator -# :: left PostgreSQL-style typecast -# [ ] left array element selection -# - right unary minus -# ^ left exponentiation -# * / % left multiplication, division, modulo -# + - left addition, subtraction -# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL -# ISNULL test for null -# NOTNULL test for not null -# (any other) left all other native and user-defined oprs -# IN set membership -# BETWEEN range containment -# OVERLAPS time interval overlap -# LIKE ILIKE SIMILAR string pattern matching -# < > less than, greater than -# = right equality, assignment -# NOT right logical negation -# AND left logical conjunction -# OR left logical disjunction +# Operator/Element Associativity Description +# . left table/column name separator +# :: left PostgreSQL-style typecast +# [ ] left array element selection +# - right unary minus +# ^ left exponentiation +# * / % left multiplication, division, modulo +# + - left addition, subtraction +# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL +# ISNULL test for null +# NOTNULL test for not null +# (any other) left all other native and user-defined oprs +# IN set membership +# BETWEEN range containment +# OVERLAPS time interval overlap +# LIKE ILIKE SIMILAR string pattern matching +# < > less than, greater than +# = right equality, assignment +# NOT right logical negation +# AND left logical conjunction +# OR left logical disjunction type SqlNodeKind* = enum ## kind of SQL abstract syntax tree @@ -518,11 +518,15 @@ type nkSelect, nkSelectDistinct, nkSelectColumns, + nkSelectPair, nkAsgn, nkFrom, + nkFromItemPair, nkGroup, + nkLimit, nkHaving, nkOrder, + nkJoin, nkDesc, nkUnion, nkIntersect, @@ -670,6 +674,7 @@ proc getPrecedence(p: SqlParser): int = result = - 1 proc parseExpr(p: var SqlParser): SqlNode +proc parseSelect(p: var SqlParser): SqlNode proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind @@ -921,6 +926,19 @@ proc parseWhere(p: var SqlParser): SqlNode = result = newNode(nkWhere) result.add(parseExpr(p)) +proc parseFromItem(p: var SqlParser): SqlNode = + result = newNode(nkFromItemPair) + if p.tok.kind == tkParLe: + getTok(p) + var select = parseSelect(p) + result.add(select) + eat(p, tkParRi) + else: + result.add(parseExpr(p)) + if isKeyw(p, "as"): + getTok(p) + result.add(parseExpr(p)) + proc parseIndexDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateIndex) if isKeyw(p, "primary"): @@ -1019,7 +1037,12 @@ proc parseSelect(p: var SqlParser): SqlNode = a.add(newNode(nkIdent, "*")) getTok(p) else: - a.add(parseExpr(p)) + var pair = newNode(nkSelectPair) + pair.add(parseExpr(p)) + a.add(pair) + if isKeyw(p, "as"): + getTok(p) + pair.add(parseExpr(p)) if p.tok.kind != tkComma: break getTok(p) result.add(a) @@ -1027,7 +1050,7 @@ proc parseSelect(p: var SqlParser): SqlNode = var f = newNode(nkFrom) while true: getTok(p) - f.add(parseExpr(p)) + f.add(parseFromItem(p)) if p.tok.kind != tkComma: break result.add(f) if isKeyw(p, "where"): @@ -1041,6 +1064,11 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(g) + if isKeyw(p, "limit"): + getTok(p) + var l = newNode(nkLimit) + l.add(parseExpr(p)) + result.add(l) if isKeyw(p, "having"): var h = newNode(nkHaving) while true: @@ -1073,6 +1101,18 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(n) + if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): + var join = newNode(nkJoin) + result.add(join) + if isKeyw(p, "join"): + join.add(newNode(nkIdent, "")) + getTok(p) + else: + join.add(parseExpr(p)) + eat(p, "join") + join.add(parseFromItem(p)) + eat(p, "on") + join.add(parseExpr(p)) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1104,7 +1144,7 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = elif isKeyw(p, "begin"): getTok(p) else: - sqlError(p, "CREATE expected") + sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") proc open(p: var SqlParser, input: Stream, filename: string) = ## opens the parser `p` and assigns the input stream `input` to it. @@ -1120,6 +1160,8 @@ proc parse(p: var SqlParser): SqlNode = result = newNode(nkStmtList) while p.tok.kind != tkEof: parseStmt(p, result) + if p.tok.kind == tkEof: + break eat(p, tkSemicolon) if result.len == 1: result = result.sons[0] @@ -1139,19 +1181,69 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc ra(n: SqlNode, s: var string, indent: int) +proc parseSQL*(input: string, filename=""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `EInvalidSql` exception. + parseSQL(newStringStream(input), "") -proc rs(n: SqlNode, s: var string, indent: int, - prefix = "(", suffix = ")", - sep = ", ") = + +type + SqlWriter = object + indent: int + upperCase: bool + buffer: string + +proc add(s: var SqlWriter, thing: string) = + s.buffer.add(thing) + +proc add(s: var SqlWriter, thing: char) = + s.buffer.add(thing) + +proc addKeyw(s: var SqlWriter, thing: string) = + if s.buffer.len > 0 and s.buffer[^1] notin " ,\L(": + s.buffer.add(" ") + if s.upperCase: + s.buffer.add(thing.toUpper()) + else: + s.buffer.add(thing) + s.buffer.add(" ") + +proc rm(s: var SqlWriter, chars = " \L,") = + while s.buffer[^1] in chars: + s.buffer = s.buffer[0..^2] + +proc newLine(s: var SqlWriter) = + s.rm(" \L") + s.buffer.add("\L") + for i in 0.. 0: s.add(prefix) for i in 0 .. n.len-1: if i > 0: s.add(sep) - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(suffix) -proc ra(n: SqlNode, s: var string, indent: int) = +proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard @@ -1169,217 +1261,222 @@ proc ra(n: SqlNode, s: var string, indent: int) = of nkIntegerLit, nkNumericLit: s.add(n.strVal) of nkPrimaryKey: - s.add(" primary key") - rs(n, s, indent) + s.addKeyw("primary key") + rs(n, s) of nkForeignKey: - s.add(" foreign key") - rs(n, s, indent) + s.addKeyw("foreign key") + rs(n, s) of nkNotNull: - s.add(" not null") + s.addKeyw("not null") of nkNull: - s.add(" null") + s.addKeyw("null") of nkDot: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(".") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkDotDot: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(". .") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkPrefix: s.add('(') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[1], s, indent) + ra(n.sons[1], s) s.add(')') of nkInfix: s.add('(') - ra(n.sons[1], s, indent) + ra(n.sons[1], s) s.add(' ') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[2], s, indent) + ra(n.sons[2], s) s.add(')') of nkCall, nkColumnReference: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(')') of nkReferences: - s.add(" references ") - ra(n.sons[0], s, indent) + s.addKeyw("references") + ra(n.sons[0], s) of nkDefault: - s.add(" default ") - ra(n.sons[0], s, indent) + s.addKeyw("default") + ra(n.sons[0], s) of nkCheck: - s.add(" check ") - ra(n.sons[0], s, indent) + s.addKeyw("check") + ra(n.sons[0], s) of nkConstraint: - s.add(" constraint ") - ra(n.sons[0], s, indent) - s.add(" check ") - ra(n.sons[1], s, indent) + s.addKeyw("constraint") + ra(n.sons[0], s) + s.addKeyw("check") + ra(n.sons[1], s) of nkUnique: - s.add(" unique") - rs(n, s, indent) + s.addKeyw("unique") + rs(n, s) of nkIdentity: - s.add(" identity") + s.addKeyw("identity") of nkColumnDef: s.add("\n ") - rs(n, s, indent, "", "", " ") + rs(n, s, "", "", " ") of nkStmtList: for i in 0..n.len-1: - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add("\n") of nkInsert: assert n.len == 3 - s.add("insert into ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) + s.addKeyw("insert into") + ra(n.sons[0], s) + ra(n.sons[1], s) if n.sons[2].kind == nkDefault: - s.add("default values") + s.addKeyw("default values") else: - s.add("\n") - ra(n.sons[2], s, indent) + s.newLine() + ra(n.sons[2], s) s.add(';') of nkUpdate: - s.add("update ") - ra(n.sons[0], s, indent) - s.add(" set ") + s.addKeyw("update") + ra(n.sons[0], s) + s.addKeyw("set") var L = n.len for i in 1 .. L-2: if i > 1: s.add(", ") var it = n.sons[i] assert it.kind == nkAsgn - ra(it, s, indent) - ra(n.sons[L-1], s, indent) + ra(it, s) + ra(n.sons[L-1], s) s.add(';') of nkDelete: - s.add("delete from ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) + s.addKeyw("delete from") + ra(n.sons[0], s) + ra(n.sons[1], s) s.add(';') of nkSelect, nkSelectDistinct: - s.add("select ") + s.addKeyw("select") if n.kind == nkSelectDistinct: - s.add("distinct ") - rs(n.sons[0], s, indent, "", "", ", ") - for i in 1 .. n.len-1: ra(n.sons[i], s, indent) + s.addKeyw("distinct") + s.inner: + for son in n.sons[0].sons: + ra(son, s) + s.add(',') + s.newLine() + s.rm() + for i in 1 .. n.len-1: + ra(n.sons[i], s) s.add(';') of nkSelectColumns: assert(false) + of nkSelectPair: + ra(n.sons[0], s) + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) + of nkFromItemPair: + if n.sons[0].kind == nkIdent: + ra(n.sons[0], s) + else: + assert n.sons[0].kind == nkSelect + s.add("(") + s.inner: + ra(n.sons[0], s) + s.rm("; \L") + s.newLine() + s.add(")") + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) of nkAsgn: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(" = ") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkFrom: - s.add("\nfrom ") - rs(n, s, indent, "", "", ", ") + s.innerKeyw("from"): + rs(n, s, "", "", ", ") of nkGroup: - s.add("\ngroup by") - rs(n, s, indent, "", "", ", ") + s.innerKeyw("group by"): + rs(n, s, "", "", ", ") + of nkLimit: + s.innerKeyw("limit"): + rs(n, s, "", "", ", ") of nkHaving: - s.add("\nhaving") - rs(n, s, indent, "", "", ", ") + s.innerKeyw("having"): + rs(n, s, "", "", ", ") of nkOrder: - s.add("\norder by ") - rs(n, s, indent, "", "", ", ") + s.addKeyw("order by") + rs(n, s, "", "", ", ") + of nkJoin: + var joinType = n.sons[0].strVal + if joinType == "": + joinType = "join" + else: + joinType &= " " & "join" + s.innerKeyw(joinType): + ra(n.sons[1], s) + s.innerKeyw("on"): + ra(n.sons[2], s) of nkDesc: - ra(n.sons[0], s, indent) - s.add(" desc") + ra(n.sons[0], s) + s.addKeyw("desc") of nkUnion: - s.add(" union") + s.addKeyw("union") of nkIntersect: - s.add(" intersect") + s.addKeyw("intersect") of nkExcept: - s.add(" except") + s.addKeyw("except") of nkColumnList: - rs(n, s, indent) + rs(n, s) of nkValueList: - s.add("values ") - rs(n, s, indent) + s.addKeyw("values") + rs(n, s) of nkWhere: - s.add("\nwhere ") - ra(n.sons[0], s, indent) + s.newLine() + s.addKeyw("where") + s.inner: + ra(n.sons[0], s) of nkCreateTable, nkCreateTableIfNotExists: - s.add("create table ") + s.addKeyw("create table") if n.kind == nkCreateTableIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + if i > 1: s.add(",") + ra(n.sons[i], s) s.add(");") of nkCreateType, nkCreateTypeIfNotExists: - s.add("create type ") + s.addKeyw("create type") if n.kind == nkCreateTypeIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" as ") - ra(n.sons[1], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("as") + ra(n.sons[1], s) s.add(';') of nkCreateIndex, nkCreateIndexIfNotExists: - s.add("create index ") + s.addKeyw("create index") if n.kind == nkCreateIndexIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" on ") - ra(n.sons[1], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("on") + ra(n.sons[1], s) s.add('(') for i in 2..n.len-1: if i > 2: s.add(", ") - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(");") of nkEnumDef: - s.add("enum ") - rs(n, s, indent) + s.addKeyw("enum") + rs(n, s) -# What I want: -# -#select(columns = [T1.all, T2.name], -# fromm = [T1, T2], -# where = T1.name ==. T2.name, -# orderby = [name]): -# -#for row in dbQuery(db, """select x, y, z -# from a, b -# where a.name = b.name"""): -# - -#select x, y, z: -# fromm: Table1, Table2 -# where: x.name == y.name -#db.select(fromm = [t1, t2], where = t1.name == t2.name): -#for x, y, z in db.select(fromm = a, b where = a.name == b.name): -# writeLine x, y, z - -proc renderSQL*(n: SqlNode): string = +proc renderSQL*(n: SqlNode, upperCase=false): string = ## Converts an SQL abstract syntax tree to its string representation. - result = "" - ra(n, result, 0) + var s: SqlWriter + s.buffer = "" + s.upperCase = upperCase + ra(n, s) + return s.buffer proc `$`*(n: SqlNode): string = ## an alias for `renderSQL`. renderSQL(n) - -when not defined(testing) and isMainModule: - echo(renderSQL(parseSQL(newStringStream(""" - CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); - CREATE TABLE holidays ( - num_weeks int, - happiness happiness - ); - CREATE INDEX table1_attr1 ON table1(attr1); - - SELECT * FROM myTab WHERE col1 = 'happy'; - """), "stdin"))) - -# CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); -# CREATE TABLE holidays ( -# num_weeks int, -# happiness happiness -# ); -# CREATE INDEX table1_attr1 ON table1(attr1) diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim new file mode 100644 index 0000000000..e73a1d7ee2 --- /dev/null +++ b/tests/stdlib/tparsesql.nim @@ -0,0 +1,438 @@ +import unittest + +import sequtils +import strutils +import parsesql + +proc fold(str: string): string = + var + lines = str.split("\L") + minCount = 1000 + while lines.len > 0 and lines[0].strip().len == 0: + lines.delete(0, 0) + while lines.len > 0 and lines[lines.len-1].strip().len == 0: + lines.delete(lines.len, lines.len) + for line in lines: + var count = 0 + while line[count] == ' ': + inc count + if minCount > count: + minCount = count + for i, line in lines: + lines[i] = line[minCount..^1] + return lines.join("\L") + +proc parseCheck(have: string, need: string) = + var + sql = parseSQL(have) + sqlHave = renderSQL(sql, true).strip() + sqlNeed = need.fold().strip() + var + haveLines = sqlHave.split("\L") + needLines = sqlNeed.split("\L") + for i in 0.. Date: Wed, 13 Dec 2017 23:34:44 +0000 Subject: [PATCH 026/200] fix --- lib/pure/parsesql.nim | 41 +- tests/stdlib/tparsesql.nim | 808 +++++++++++++++++-------------------- 2 files changed, 397 insertions(+), 452 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index b53f46f82c..f266beef78 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -662,10 +662,12 @@ proc getPrecedence(p: SqlParser): int = elif isOpr(p, "=") or isOpr(p, "<") or isOpr(p, ">") or isOpr(p, ">=") or isOpr(p, "<=") or isOpr(p, "<>") or isOpr(p, "!=") or isKeyw(p, "is") or isKeyw(p, "like"): - result = 3 + result = 4 elif isKeyw(p, "and"): - result = 2 + result = 3 elif isKeyw(p, "or"): + result = 2 + elif isKeyw(p, "between"): result = 1 elif p.tok.kind == tkOperator: # user-defined operator: @@ -1015,6 +1017,8 @@ proc parseUpdate(p: var SqlParser): SqlNode = proc parseDelete(p: var SqlParser): SqlNode = getTok(p) + if isOpr(p, "*"): + getTok(p) result = newNode(nkDelete) eat(p, "from") result.add(primary(p)) @@ -1156,7 +1160,7 @@ proc open(p: var SqlParser, input: Stream, filename: string) = proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. result = newNode(nkStmtList) while p.tok.kind != tkEof: parseStmt(p, result) @@ -1173,7 +1177,7 @@ proc close(p: var SqlParser) = proc parseSQL*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. var p: SqlParser open(p, input, filename) try: @@ -1184,7 +1188,7 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = proc parseSQL*(input: string, filename=""): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. parseSQL(newStringStream(input), "") @@ -1210,7 +1214,7 @@ proc addKeyw(s: var SqlWriter, thing: string) = s.buffer.add(" ") proc rm(s: var SqlWriter, chars = " \L,") = - while s.buffer[^1] in chars: + while s.buffer.len > 0 and s.buffer[^1] in chars: s.buffer = s.buffer[0..^2] proc newLine(s: var SqlWriter) = @@ -1253,6 +1257,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: + # TODO add e'' as an option? s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") @@ -1329,23 +1334,25 @@ proc ra(n: SqlNode, s: var SqlWriter) = assert n.len == 3 s.addKeyw("insert into") ra(n.sons[0], s) + s.add(" ") ra(n.sons[1], s) if n.sons[2].kind == nkDefault: s.addKeyw("default values") else: s.newLine() ra(n.sons[2], s) + s.rm(" ") s.add(';') of nkUpdate: - s.addKeyw("update") - ra(n.sons[0], s) - s.addKeyw("set") - var L = n.len - for i in 1 .. L-2: - if i > 1: s.add(", ") - var it = n.sons[i] - assert it.kind == nkAsgn - ra(it, s) + s.innerKeyw("update"): + ra(n.sons[0], s) + s.innerKeyw("set"): + var L = n.len + for i in 1 .. L-2: + if i > 1: s.add(", ") + var it = n.sons[i] + assert it.kind == nkAsgn + ra(it, s) ra(n.sons[L-1], s) s.add(';') of nkDelete: @@ -1404,8 +1411,8 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.innerKeyw("having"): rs(n, s, "", "", ", ") of nkOrder: - s.addKeyw("order by") - rs(n, s, "", "", ", ") + s.innerKeyw("order by"): + rs(n, s, "", "", ", ") of nkJoin: var joinType = n.sons[0].strVal if joinType == "": diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index e73a1d7ee2..fe64d3416f 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -1,438 +1,376 @@ -import unittest +discard """ + file: "tparsesql.nim" + output: '''select + foo +from + table; +select + foo +from + table; +select + foo +from + table +limit + 10; +select + foo, + bar, + baz +from + table +limit + 10; +select + foo as bar +from + table; +select + foo as foo_prime, + bar as bar_prime, + baz as baz_prime +from + table; +select + * +from + table; +select + * +from + table +where + ((a = b) and (c = d)); +select + * +from + table +where + (not b); +select + * +from + table +where + (a and (not b)); +select + * +from + table +where + (((a = b) and (c = d)) or ((n is null) and (((not b) + 1) = 3))); +select + * +from + table +having + ((a = b) and (c = d)); +select + a, + b +from + table +group by + a; +select + a, + b +from + table +group by + 1, 2; +select + t.a +from + t as t; +select + a, + b +from + ( + select + * + from + t + ); +select + a, + b +from + ( + select + * + from + t + ) as foo; +select + a, + b +from + ( + select + * + from + ( + select + * + from + ( + select + * + from + ( + select + * + from + inner as inner1 + ) as inner2 + ) as inner3 + ) as inner4 + ) as inner5; +select + a, + b +from + ( + select + * + from + a + ), ( + select + * + from + b + ), ( + select + * + from + c + ); +select + * +from + Products +where + (Price BETWEEN (10 AND 20)); +select + id +from + a +join + b +on + (a.id == b.id); +select + id +from + a +join + ( + select + id + from + c + ) as b +on + (a.id == b.id); +select + id +from + a +INNER join + b +on + (a.id == b.id); +select + id +from + a +OUTER join + b +on + (a.id == b.id); +select + id +from + a +CROSS join + b +on + (a.id == b.id); +create type happiness as enum ('happy', 'very happy', 'ecstatic'); +create table holidays( + num_weeks int, + happiness happiness); +create index table1_attr1 on table1(attr1); +select + * +from + myTab +where + (col1 = 'happy'); + +insert into Customers (CustomerName, ContactName, Address, City, PostalCode, Country) +values ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); +insert into TableName default values; + +update + Customers +set + ContactName = 'Alfred Schmidt', City = 'Frankfurt' +where + (CustomerID = 1); +delete from table_name; +delete from table_name; +select + * +from + Customers; +select + * +from + Customers +where + ((((CustomerName LIKE 'L%') OR (CustomerName LIKE 'R%')) OR (CustomerName LIKE 'W%')) AND (Country = 'USA')) +order by + CustomerName; + +''' +""" -import sequtils -import strutils import parsesql -proc fold(str: string): string = - var - lines = str.split("\L") - minCount = 1000 - while lines.len > 0 and lines[0].strip().len == 0: - lines.delete(0, 0) - while lines.len > 0 and lines[lines.len-1].strip().len == 0: - lines.delete(lines.len, lines.len) - for line in lines: - var count = 0 - while line[count] == ' ': - inc count - if minCount > count: - minCount = count - for i, line in lines: - lines[i] = line[minCount..^1] - return lines.join("\L") - -proc parseCheck(have: string, need: string) = - var - sql = parseSQL(have) - sqlHave = renderSQL(sql, true).strip() - sqlNeed = need.fold().strip() - var - haveLines = sqlHave.split("\L") - needLines = sqlNeed.split("\L") - for i in 0.. Date: Thu, 14 Dec 2017 18:31:43 +0000 Subject: [PATCH 027/200] fix --- lib/pure/parsesql.nim | 185 ++++++++--------- tests/stdlib/tparsesql.nim | 407 ++++++++++--------------------------- 2 files changed, 196 insertions(+), 396 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index f266beef78..ae192ab9a0 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -55,6 +55,13 @@ const ";", ":", ",", "(", ")", "[", "]", "." ] + reservedKeywords = @[ + # statements + "select", "from", "where", "group", "limit", "having", + # functions + "count", + ] + proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) L.filename = filename @@ -274,16 +281,16 @@ proc getSymbol(c: var SqlLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) = +proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') = var pos = c.bufpos + 1 var buf = c.buf tok.kind = tkQuotedIdentifier while true: var ch = buf[pos] - if ch == '\"': - if buf[pos+1] == '\"': + if ch == quote: + if buf[pos+1] == quote: inc(pos, 2) - add(tok.literal, '\"') + add(tok.literal, quote) else: inc(pos) break @@ -442,7 +449,8 @@ proc getTok(c: var SqlLexer, tok: var Token) = add(tok.literal, '.') of '0'..'9': getNumeric(c, tok) of '\'': getString(c, tok, tkStringConstant) - of '"': getQuotedIdentifier(c, tok) + of '"': getQuotedIdentifier(c, tok, '"') + of '`': getQuotedIdentifier(c, tok, '`') of lexbase.EndOfFile: tok.kind = tkEof tok.literal = "[EOF]" @@ -450,7 +458,7 @@ proc getTok(c: var SqlLexer, tok: var Token) = '\128'..'\255': getSymbol(c, tok) of '+', '-', '*', '/', '<', '>', '=', '~', '!', '@', '#', '%', - '^', '&', '|', '`', '?': + '^', '&', '|', '?': getOperator(c, tok) else: add(tok.literal, c.buf[c.bufpos]) @@ -504,6 +512,7 @@ type nkPrefix, nkInfix, nkCall, + nkPrGroup, nkColumnReference, nkReferences, nkDefault, @@ -700,7 +709,8 @@ proc identOrLiteral(p: var SqlParser): SqlNode = getTok(p) of tkParLe: getTok(p) - result = parseExpr(p) + result = newNode(nkPrGroup) + result.add(parseExpr(p)) eat(p, tkParRi) else: sqlError(p, "expression expected") @@ -752,7 +762,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal) + opNode = newNode(nkIdent, p.tok.literal.toLower()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -1112,7 +1122,8 @@ proc parseSelect(p: var SqlParser): SqlNode = join.add(newNode(nkIdent, "")) getTok(p) else: - join.add(parseExpr(p)) + join.add(newNode(nkIdent, p.tok.literal.toLower())) + getTok(p) eat(p, "join") join.add(parseFromItem(p)) eat(p, "on") @@ -1167,8 +1178,6 @@ proc parse(p: var SqlParser): SqlNode = if p.tok.kind == tkEof: break eat(p, tkSemicolon) - if result.len == 1: - result = result.sons[0] proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. @@ -1198,44 +1207,25 @@ type upperCase: bool buffer: string -proc add(s: var SqlWriter, thing: string) = - s.buffer.add(thing) - proc add(s: var SqlWriter, thing: char) = s.buffer.add(thing) -proc addKeyw(s: var SqlWriter, thing: string) = - if s.buffer.len > 0 and s.buffer[^1] notin " ,\L(": +proc add(s: var SqlWriter, thing: string) = + if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}: s.buffer.add(" ") + s.buffer.add(thing) + +proc addKeyw(s: var SqlWriter, thing: string) = + var keyw = thing if s.upperCase: - s.buffer.add(thing.toUpper()) - else: - s.buffer.add(thing) - s.buffer.add(" ") + keyw = keyw.toUpper() + s.add(keyw) -proc rm(s: var SqlWriter, chars = " \L,") = - while s.buffer.len > 0 and s.buffer[^1] in chars: - s.buffer = s.buffer[0..^2] - -proc newLine(s: var SqlWriter) = - s.rm(" \L") - s.buffer.add("\L") - for i in 0.. 0: + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = + if n.len > 0: + s.add(prefix) + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + s.add(suffix) + proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard of nkIdent: - if allCharsInSet(n.strVal, {'\33'..'\127'}): + if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords: s.add(n.strVal) else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: - # TODO add e'' as an option? s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") @@ -1277,33 +1280,33 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("null") of nkDot: ra(n.sons[0], s) - s.add(".") + s.add('.') ra(n.sons[1], s) of nkDotDot: ra(n.sons[0], s) s.add(". .") ra(n.sons[1], s) of nkPrefix: - s.add('(') ra(n.sons[0], s) s.add(' ') ra(n.sons[1], s) - s.add(')') of nkInfix: - s.add('(') ra(n.sons[1], s) s.add(' ') ra(n.sons[0], s) s.add(' ') ra(n.sons[2], s) - s.add(')') of nkCall, nkColumnReference: ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") + if i > 1: s.add(',') ra(n.sons[i], s) s.add(')') + of nkPrGroup: + s.add('(') + s.addMulti(n) + s.add(')') of nkReferences: s.addKeyw("references") ra(n.sons[0], s) @@ -1324,55 +1327,43 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkIdentity: s.addKeyw("identity") of nkColumnDef: - s.add("\n ") rs(n, s, "", "", " ") of nkStmtList: for i in 0..n.len-1: ra(n.sons[i], s) - s.add("\n") + s.add(';') of nkInsert: assert n.len == 3 s.addKeyw("insert into") ra(n.sons[0], s) - s.add(" ") + s.add(' ') ra(n.sons[1], s) if n.sons[2].kind == nkDefault: s.addKeyw("default values") else: - s.newLine() ra(n.sons[2], s) - s.rm(" ") - s.add(';') of nkUpdate: - s.innerKeyw("update"): - ra(n.sons[0], s) - s.innerKeyw("set"): - var L = n.len - for i in 1 .. L-2: - if i > 1: s.add(", ") - var it = n.sons[i] - assert it.kind == nkAsgn - ra(it, s) + s.addKeyw("update") + ra(n.sons[0], s) + s.addKeyw("set") + var L = n.len + for i in 1 .. L-2: + if i > 1: s.add(", ") + var it = n.sons[i] + assert it.kind == nkAsgn + ra(it, s) ra(n.sons[L-1], s) - s.add(';') of nkDelete: s.addKeyw("delete from") ra(n.sons[0], s) ra(n.sons[1], s) - s.add(';') of nkSelect, nkSelectDistinct: s.addKeyw("select") if n.kind == nkSelectDistinct: s.addKeyw("distinct") - s.inner: - for son in n.sons[0].sons: - ra(son, s) - s.add(',') - s.newLine() - s.rm() + s.addMulti(n.sons[0]) for i in 1 .. n.len-1: ra(n.sons[i], s) - s.add(';') of nkSelectColumns: assert(false) of nkSelectPair: @@ -1385,12 +1376,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) else: assert n.sons[0].kind == nkSelect - s.add("(") - s.inner: - ra(n.sons[0], s) - s.rm("; \L") - s.newLine() - s.add(")") + s.add('(') + ra(n.sons[0], s) + s.add(')') if n.sons.len == 2: s.addKeyw("as") ra(n.sons[1], s) @@ -1399,30 +1387,30 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.add(" = ") ra(n.sons[1], s) of nkFrom: - s.innerKeyw("from"): - rs(n, s, "", "", ", ") + s.addKeyw("from") + s.addMulti(n) of nkGroup: - s.innerKeyw("group by"): - rs(n, s, "", "", ", ") + s.addKeyw("group by") + s.addMulti(n) of nkLimit: - s.innerKeyw("limit"): - rs(n, s, "", "", ", ") + s.addKeyw("limit") + s.addMulti(n) of nkHaving: - s.innerKeyw("having"): - rs(n, s, "", "", ", ") + s.addKeyw("having") + s.addMulti(n) of nkOrder: - s.innerKeyw("order by"): - rs(n, s, "", "", ", ") + s.addKeyw("order by") + s.addMulti(n) of nkJoin: var joinType = n.sons[0].strVal if joinType == "": joinType = "join" else: joinType &= " " & "join" - s.innerKeyw(joinType): - ra(n.sons[1], s) - s.innerKeyw("on"): - ra(n.sons[2], s) + s.addKeyw(joinType) + ra(n.sons[1], s) + s.addKeyw("on") + ra(n.sons[2], s) of nkDesc: ra(n.sons[0], s) s.addKeyw("desc") @@ -1438,10 +1426,8 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("values") rs(n, s) of nkWhere: - s.newLine() s.addKeyw("where") - s.inner: - ra(n.sons[0], s) + ra(n.sons[0], s) of nkCreateTable, nkCreateTableIfNotExists: s.addKeyw("create table") if n.kind == nkCreateTableIfNotExists: @@ -1449,7 +1435,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(",") + if i > 1: s.add(',') ra(n.sons[i], s) s.add(");") of nkCreateType, nkCreateTypeIfNotExists: @@ -1459,7 +1445,6 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) s.addKeyw("as") ra(n.sons[1], s) - s.add(';') of nkCreateIndex, nkCreateIndexIfNotExists: s.addKeyw("create index") if n.kind == nkCreateIndexIfNotExists: diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index fe64d3416f..3dc949ea1c 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -1,346 +1,149 @@ discard """ file: "tparsesql.nim" - output: '''select - foo -from - table; -select - foo -from - table; -select - foo -from - table -limit - 10; -select - foo, - bar, - baz -from - table -limit - 10; -select - foo as bar -from - table; -select - foo as foo_prime, - bar as bar_prime, - baz as baz_prime -from - table; -select - * -from - table; -select - * -from - table -where - ((a = b) and (c = d)); -select - * -from - table -where - (not b); -select - * -from - table -where - (a and (not b)); -select - * -from - table -where - (((a = b) and (c = d)) or ((n is null) and (((not b) + 1) = 3))); -select - * -from - table -having - ((a = b) and (c = d)); -select - a, - b -from - table -group by - a; -select - a, - b -from - table -group by - 1, 2; -select - t.a -from - t as t; -select - a, - b -from - ( - select - * - from - t - ); -select - a, - b -from - ( - select - * - from - t - ) as foo; -select - a, - b -from - ( - select - * - from - ( - select - * - from - ( - select - * - from - ( - select - * - from - inner as inner1 - ) as inner2 - ) as inner3 - ) as inner4 - ) as inner5; -select - a, - b -from - ( - select - * - from - a - ), ( - select - * - from - b - ), ( - select - * - from - c - ); -select - * -from - Products -where - (Price BETWEEN (10 AND 20)); -select - id -from - a -join - b -on - (a.id == b.id); -select - id -from - a -join - ( - select - id - from - c - ) as b -on - (a.id == b.id); -select - id -from - a -INNER join - b -on - (a.id == b.id); -select - id -from - a -OUTER join - b -on - (a.id == b.id); -select - id -from - a -CROSS join - b -on - (a.id == b.id); -create type happiness as enum ('happy', 'very happy', 'ecstatic'); -create table holidays( - num_weeks int, - happiness happiness); -create index table1_attr1 on table1(attr1); -select - * -from - myTab -where - (col1 = 'happy'); - -insert into Customers (CustomerName, ContactName, Address, City, PostalCode, Country) -values ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); -insert into TableName default values; - -update - Customers -set - ContactName = 'Alfred Schmidt', City = 'Frankfurt' -where - (CustomerID = 1); -delete from table_name; -delete from table_name; -select - * -from - Customers; -select - * -from - Customers -where - ((((CustomerName LIKE 'L%') OR (CustomerName LIKE 'R%')) OR (CustomerName LIKE 'W%')) AND (Country = 'USA')) -order by - CustomerName; - -''' """ import parsesql -echo $parseSQL "SELECT foo FROM table;" -echo $parseSQL "SELECT foo FROM table" -echo $parseSQL "SELECT foo FROM table limit 10" -echo $parseSQL "SELECT foo, bar, baz FROM table limit 10" -echo $parseSQL "SELECT foo AS bar FROM table" -echo $parseSQL "SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table" -echo $parseSQL "SELECT * FROM table" +doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" +doAssert $parseSQL(""" +SELECT + CustomerName, + ContactName, + Address, + City, + PostalCode, + Country, + CustomerName, + ContactName, + Address, + City, + PostalCode, + Country +FROM table;""") == "select CustomerName, ContactName, Address, City, PostalCode, Country, CustomerName, ContactName, Address, City, PostalCode, Country from table;" + +doAssert $parseSQL("SELECT foo FROM table limit 10") == "select foo from table limit 10;" +doAssert $parseSQL("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" +doAssert $parseSQL("SELECT foo AS bar FROM table") == "select foo as bar from table;" +doAssert $parseSQL("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" +doAssert $parseSQL("SELECT * FROM table") == "select * from table;" + + #TODO add count(*) -#echo $parseSQL "SELECT COUNT(*) FROM table" -echo $parseSQL """ +#doAssert $parseSQL("SELECT COUNT(*) FROM table" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a = b and c = d -""" -echo $parseSQL """ +""") == "select * from table where a = b and c = d;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE not b -""" -echo $parseSQL """ +""") == "select * from table where not b;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a and not b -""" -echo $parseSQL """ +""") == "select * from table where a and not b;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a = b and c = d or n is null and not b + 1 = 3 -""" -echo $parseSQL """ +""") == "select * from table where a = b and c = d or n is null and not b + 1 = 3;" + +doAssert $parseSQL(""" +SELECT * FROM table +WHERE (a = b and c = d) or (n is null and not b + 1 = 3) +""") == "select * from table where(a = b and c = d) or (n is null and not b + 1 = 3);" + +doAssert $parseSQL(""" SELECT * FROM table HAVING a = b and c = d -""" -echo $parseSQL """ +""") == "select * from table having a = b and c = d;" + +doAssert $parseSQL(""" SELECT a, b FROM table GROUP BY a -""" -echo $parseSQL """ +""") == "select a, b from table group by a;" + +doAssert $parseSQL(""" SELECT a, b FROM table GROUP BY 1, 2 -""" -echo $parseSQL "SELECT t.a FROM t as t" -echo $parseSQL """ +""") == "select a, b from table group by 1, 2;" + +doAssert $parseSQL("SELECT t.a FROM t as t") == "select t.a from t as t;" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM t ) -""" -echo $parseSQL """ +""") == "select a, b from(select * from t);" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM t ) as foo -""" -echo $parseSQL """ +""") == "select a, b from(select * from t) as foo;" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM ( SELECT * FROM ( SELECT * FROM ( - SELECT * FROM inner as inner1 + SELECT * FROM innerTable as inner1 ) as inner2 ) as inner3 ) as inner4 ) as inner5 -""" -echo $parseSQL """ +""") == "select a, b from(select * from(select * from(select * from(select * from innerTable as inner1) as inner2) as inner3) as inner4) as inner5;" + +doAssert $parseSQL(""" SELECT a, b FROM (SELECT * FROM a), (SELECT * FROM b), (SELECT * FROM c) -""" -echo $parseSQL """ +""") == "select a, b from(select * from a),(select * from b),(select * from c);" + +doAssert $parseSQL(""" SELECT * FROM Products WHERE Price BETWEEN 10 AND 20; -""" -echo $parseSQL """ +""") == "select * from Products where Price between 10 and 20;" + +doAssert $parseSQL(""" SELECT id FROM a JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a JOIN (SELECT id from c) as b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a join(select id from c) as b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a INNER JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a inner join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a OUTER JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a outer join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a CROSS JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a cross join b on a.id == b.id;" + +doAssert $parseSQL(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); CREATE TABLE holidays ( num_weeks int, @@ -348,29 +151,41 @@ CREATE TABLE holidays ( ); CREATE INDEX table1_attr1 ON table1(attr1); SELECT * FROM myTab WHERE col1 = 'happy'; -""" -echo $parseSQL """ +""") == "create type happiness as enum ('happy' , 'very happy' , 'ecstatic' ); create table holidays(num_weeks int , happiness happiness );; create index table1_attr1 on table1(attr1 );; select * from myTab where col1 = 'happy';" + +doAssert $parseSQL(""" INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); -""" -echo $parseSQL """ +""") == "insert into Customers (CustomerName , ContactName , Address , City , PostalCode , Country ) values ('Cardinal' , 'Tom B. Erichsen' , 'Skagen 21' , 'Stavanger' , '4006' , 'Norway' );" + +doAssert $parseSQL(""" INSERT INTO TableName DEFAULT VALUES -""" -echo $parseSQL """ +""") == "insert into TableName default values;" + +doAssert $parseSQL(""" UPDATE Customers SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; -""" -echo $parseSQL "DELETE FROM table_name;" -echo $parseSQL "DELETE * FROM table_name;" -echo $parseSQL """ +""") == "update Customers set ContactName = 'Alfred Schmidt' , City = 'Frankfurt' where CustomerID = 1;" + +doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" + +doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" + +doAssert $parseSQL(""" --Select all: SELECT * FROM Customers; -""" -echo $parseSQL """ +""") == "select * from Customers;" + +doAssert $parseSQL(""" SELECT * FROM Customers WHERE (CustomerName LIKE 'L%' OR CustomerName LIKE 'R%' /*OR CustomerName LIKE 'S%' OR CustomerName LIKE 'T%'*/ OR CustomerName LIKE 'W%') AND Country='USA' ORDER BY CustomerName; -""" +""") == "select * from Customers where(CustomerName like 'L%' or CustomerName like 'R%' or CustomerName like 'W%') and Country = 'USA' order by CustomerName;" + +# parse keywords as identifires +doAssert $parseSQL(""" +SELECT `SELECT`, `FROM` as `GROUP` FROM `WHERE`; +""") == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" From c678ac3f33d67cf39df777ca2c8440a6cabcc0b7 Mon Sep 17 00:00:00 2001 From: Euan T Date: Thu, 16 Jun 2016 18:40:35 +0100 Subject: [PATCH 028/200] Expand dbQuote to handle backslashes --- lib/impure/db_mysql.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 1b79b35434..e1119ada6c 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -120,6 +120,7 @@ proc dbQuote*(s: string): string = result = "'" for c in items(s): if c == '\'': add(result, "''") + if c == '\\': add(result, "\\\\") else: add(result, c) add(result, '\'') From 21a9fce4142dca497b1e7c85b35479aba5f47a8c Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 31 Dec 2016 15:55:34 +0000 Subject: [PATCH 029/200] Fix onSignal example --- lib/posix/posix.nim | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index b635c0b0b1..a73b6090e3 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -962,10 +962,13 @@ proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "".} template onSignal*(signals: varargs[cint], body: untyped) = - ## Setup code to be executed when Unix signals are received. Example: - ## from posix import SIGINT, SIGTERM - ## onSignal(SIGINT, SIGTERM): - ## echo "bye" + ## Setup code to be executed when Unix signals are received. + ## Example: + ## + ## .. code-block:: nim + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" for s in signals: handle_signal(s, From 54ee368358b607e94299cbe1df65e770334e5d96 Mon Sep 17 00:00:00 2001 From: Jon Caldwell Date: Thu, 25 May 2017 01:19:55 -0700 Subject: [PATCH 030/200] Fix identifier in cursor* templates in terminal --- lib/pure/terminal.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 871ac5d391..a08a388baf 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -650,10 +650,10 @@ template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): template setCursorYPos(x: int) = setCursorYPos(stdout, x) -template cursorUp*(count=1) = cursorUp(stdout, f) -template cursorDown*(count=1) = cursorDown(stdout, f) -template cursorForward*(count=1) = cursorForward(stdout, f) -template cursorBackward*(count=1) = cursorBackward(stdout, f) +template cursorUp*(count=1) = cursorUp(stdout, count) +template cursorDown*(count=1) = cursorDown(stdout, count) +template cursorForward*(count=1) = cursorForward(stdout, count) +template cursorBackward*(count=1) = cursorBackward(stdout, count) template eraseLine*() = eraseLine(stdout) template eraseScreen*() = eraseScreen(stdout) template setStyle*(style: set[Style]) = From 55fdac46e33337c82877603fa277325aa27be7a1 Mon Sep 17 00:00:00 2001 From: treeform Date: Thu, 14 Dec 2017 18:40:20 +0000 Subject: [PATCH 031/200] fix --- lib/impure/db_mysql.nim | 1 - lib/posix/posix.nim | 11 ++++------- lib/pure/terminal.nim | 8 ++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index e1119ada6c..1b79b35434 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -120,7 +120,6 @@ proc dbQuote*(s: string): string = result = "'" for c in items(s): if c == '\'': add(result, "''") - if c == '\\': add(result, "\\\\") else: add(result, c) add(result, '\'') diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index a73b6090e3..b635c0b0b1 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -962,13 +962,10 @@ proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "".} template onSignal*(signals: varargs[cint], body: untyped) = - ## Setup code to be executed when Unix signals are received. - ## Example: - ## - ## .. code-block:: nim - ## from posix import SIGINT, SIGTERM - ## onSignal(SIGINT, SIGTERM): - ## echo "bye" + ## Setup code to be executed when Unix signals are received. Example: + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" for s in signals: handle_signal(s, diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index a08a388baf..871ac5d391 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -650,10 +650,10 @@ template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): template setCursorYPos(x: int) = setCursorYPos(stdout, x) -template cursorUp*(count=1) = cursorUp(stdout, count) -template cursorDown*(count=1) = cursorDown(stdout, count) -template cursorForward*(count=1) = cursorForward(stdout, count) -template cursorBackward*(count=1) = cursorBackward(stdout, count) +template cursorUp*(count=1) = cursorUp(stdout, f) +template cursorDown*(count=1) = cursorDown(stdout, f) +template cursorForward*(count=1) = cursorForward(stdout, f) +template cursorBackward*(count=1) = cursorBackward(stdout, f) template eraseLine*() = eraseLine(stdout) template eraseScreen*() = eraseScreen(stdout) template setStyle*(style: set[Style]) = From da2f689e09d9eb6a3a3af6c5f7f8f06fc17c48ea Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 14 Dec 2017 20:49:08 +0100 Subject: [PATCH 032/200] fixes #6033 --- changelog.md | 2 ++ {lib/pure => compiler}/securehash.nim | 0 doc/lib.rst | 2 -- web/website.ini | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename {lib/pure => compiler}/securehash.nim (100%) diff --git a/changelog.md b/changelog.md index 6065ccd101..e0f481f236 100644 --- a/changelog.md +++ b/changelog.md @@ -126,3 +126,5 @@ This now needs to be written as: - The behavior of ``$`` has been changed for all standard library collections. The collection-to-string implementations now perform proper quoting and escaping of strings and chars. +- Removed ``securehash`` stdlib module as it is not secure anymore. The module + is still available via ``compiler/securehash``. diff --git a/lib/pure/securehash.nim b/compiler/securehash.nim similarity index 100% rename from lib/pure/securehash.nim rename to compiler/securehash.nim diff --git a/doc/lib.rst b/doc/lib.rst index 21d3dc8cc7..959c3ef9bc 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -376,8 +376,6 @@ Cryptography and Hashing * `base64 `_ This module implements a base64 encoder and decoder. -* `securehash `_ - This module implements a sha1 encoder and decoder. Multimedia support ------------------ diff --git a/web/website.ini b/web/website.ini index 17a062eae7..a158e3b473 100644 --- a/web/website.ini +++ b/web/website.ini @@ -64,7 +64,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;pure/securehash;impure/rdstdin" +srcdoc2: "pure/fenv;impure/rdstdin" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile" From 1bb086e7a8e8cf836b99e8332203641c9e626ab6 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 14 Dec 2017 20:55:02 +0100 Subject: [PATCH 033/200] fixes #5999 --- compiler/semobjconstr.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 56d160aa42..a0bf084faa 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -39,13 +39,19 @@ proc mergeInitStatus(existing: var InitStatus, newStatus: InitStatus) = of initUnknown: discard +proc invalidObjConstr(n: PNode) = + if n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s[0] == ':': + localError(n.info, "incorrect object construction syntax; use a space after the colon") + else: + localError(n.info, "incorrect object construction syntax") + proc locateFieldInInitExpr(field: PSym, initExpr: PNode): PNode = # Returns the assignment nkExprColonExpr node or nil let fieldId = field.name.id for i in 1 ..< initExpr.len: let assignment = initExpr[i] if assignment.kind != nkExprColonExpr: - localError(initExpr.info, "incorrect object construction syntax") + invalidObjConstr(assignment) continue if fieldId == considerQuotedIdent(assignment[0]).id: @@ -284,7 +290,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = let field = result[i] if nfSem notin field.flags: if field.kind != nkExprColonExpr: - localError(n.info, "incorrect object construction syntax") + invalidObjConstr(field) continue let id = considerQuotedIdent(field[0]) # This node was not processed. There are two possible reasons: From 196977f623c3e3384e943c08cb85e6e0ded31109 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 14 Dec 2017 21:37:53 +0100 Subject: [PATCH 034/200] many improvements to random.nim; fixes #4726 --- changelog.md | 4 ++ lib/pure/random.nim | 125 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 27 deletions(-) diff --git a/changelog.md b/changelog.md index e0f481f236..ea31767055 100644 --- a/changelog.md +++ b/changelog.md @@ -128,3 +128,7 @@ This now needs to be written as: strings and chars. - Removed ``securehash`` stdlib module as it is not secure anymore. The module is still available via ``compiler/securehash``. +- The ``random`` procs in ``random.nim`` have all been deprecated. Instead use + the new ``rand`` procs. The module now exports the state of the random + number generator as type ``Rand`` so multiple threads can easily use their + own random number generators that do not require locking. diff --git a/lib/pure/random.nim b/lib/pure/random.nim index e6a9162c58..0e2e30a7f9 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -7,16 +7,16 @@ # distribution, for details about the copyright. # -## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. +## Nim's standard random number generator. Based on +## the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. ## * More information: http://xoroshiro.di.unimi.it/ ## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c ## -## Do not use this module for cryptographic use! +## **Do not use this module for cryptographic purposes!** include "system/inclrtl" {.push debugger:off.} -# XXX Expose RandomGenState when defined(JS): type ui = uint32 @@ -27,31 +27,34 @@ else: const randMax = 18_446_744_073_709_551_615u64 type - RandomGenState = object + Rand* = object ## State of the random number generator. + ## The procs that use the default state + ## are **not** thread-safe! a0, a1: ui when defined(JS): - var state = RandomGenState( + var state = Rand( a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility else: # racy for multi-threading but good enough for now: - var state = RandomGenState( + var state = Rand( a0: 0x69B4C98CB8530805u64, a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility proc rotl(x, k: ui): ui = result = (x shl k) or (x shr (ui(64) - k)) -proc next(s: var RandomGenState): uint64 = - let s0 = s.a0 - var s1 = s.a1 +proc next*(r: var Rand): uint64 = + ## Uses the state to compute a new ``uint64`` random number. + let s0 = r.a0 + var s1 = r.a1 result = s0 + s1 s1 = s1 xor s0 - s.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b - s.a1 = rotl(s1, 36) # c + r.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b + r.a1 = rotl(s1, 36) # c -proc skipRandomNumbers(s: var RandomGenState) = +proc skipRandomNumbers*(s: var Rand) = ## This is the jump function for the generator. It is equivalent ## to 2^64 calls to next(); it can be used to generate 2^64 ## non-overlapping subsequences for parallel computations. @@ -71,21 +74,23 @@ proc skipRandomNumbers(s: var RandomGenState) = s.a0 = s0 s.a1 = s1 -proc random*(max: int): int {.benign.} = +proc random*(max: int): int {.benign, deprecated.} = ## Returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. + ## number, i.e. a tickcount. **Deprecated since version 0.18.0**. + ## Use ``rand`` instead. while true: let x = next(state) if x < randMax - (randMax mod ui(max)): return int(x mod uint64(max)) -proc random*(max: float): float {.benign.} = +proc random*(max: float): float {.benign, deprecated.} = ## Returns a random number in the range 0.. Date: Fri, 15 Dec 2017 11:21:49 +0100 Subject: [PATCH 035/200] fixes #668 --- compiler/aliases.nim | 8 ++++++- compiler/ccgexprs.nim | 22 +++++++++++++++---- tests/ccgbugs/tobjconstr_bad_aliasing.nim | 26 +++++++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 tests/ccgbugs/tobjconstr_bad_aliasing.nim diff --git a/compiler/aliases.nim b/compiler/aliases.nim index c0371e1596..cd7e7f19ae 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -179,5 +179,11 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = result = isPartOf(a[0], b) if result == arNo: result = arMaybe else: discard + of nkObjConstr: + result = arNo + for i in 1..data[$2]", rdLoc(d), intLiteral(i)) + arr.r = rfmt(nil, "$1->data[$2]", rdLoc(dest[]), intLiteral(i)) arr.storage = OnHeap # we know that sequences are on the heap expr(p, n[i], arr) gcUsage(n) + if doesAlias: + if d.k == locNone: + d = tmp + else: + genAssignment(p, d, tmp, {}) proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = var elem, a, arr: TLoc diff --git a/tests/ccgbugs/tobjconstr_bad_aliasing.nim b/tests/ccgbugs/tobjconstr_bad_aliasing.nim new file mode 100644 index 0000000000..ea51ecacb6 --- /dev/null +++ b/tests/ccgbugs/tobjconstr_bad_aliasing.nim @@ -0,0 +1,26 @@ +discard """ + output: '''(10, (20, ))''' +""" + +import strutils, sequtils + +# bug #668 + +type + TThing = ref object + data: int + children: seq[TThing] + +proc `$`(t: TThing): string = + result = "($1, $2)" % @[$t.data, join(map(t.children, proc(th: TThing): string = $th), ", ")] + +proc somethingelse(): seq[TThing] = + result = @[TThing(data: 20, children: @[])] + +proc dosomething(): seq[TThing] = + result = somethingelse() + + result = @[TThing(data: 10, children: result)] + +when isMainModule: + echo($dosomething()[0]) From 8decf0f5ced146c755250896be2cbc3b55af96f7 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 15 Dec 2017 11:34:17 +0100 Subject: [PATCH 036/200] make JS tests green again --- compiler/jsgen.nim | 2 +- lib/pure/random.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index bc0f90e179..50dfa22f89 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2252,7 +2252,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = case n.kind of nkSym: genSym(p, n, r) - of nkCharLit..nkUInt32Lit: + of nkCharLit..nkUInt64Lit: if n.typ.kind == tyBool: r.res = if n.intVal == 0: rope"false" else: rope"true" else: diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 0e2e30a7f9..7edd93c088 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -210,7 +210,7 @@ when isMainModule: for i, oc in occur: if oc < 69: doAssert false, "too few occurrences of " & $i - elif oc > 130: + elif oc > 150: doAssert false, "too many occurrences of " & $i var a = [0, 1] From 8db5b32ff73fd01f5fabe50703f3a2f6206275fa Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 15 Dec 2017 12:16:03 +0100 Subject: [PATCH 037/200] make type vs proc ambiguous handling more consistent; fixes #6726; fixes #6693 --- changelog.md | 2 ++ compiler/importer.nim | 2 +- compiler/semexprs.nim | 4 ++-- tests/modules/mrange.nim | 2 ++ tests/modules/tambig_range.nim | 9 +++++++++ 5 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 tests/modules/mrange.nim create mode 100644 tests/modules/tambig_range.nim diff --git a/changelog.md b/changelog.md index ea31767055..bf83a79fa5 100644 --- a/changelog.md +++ b/changelog.md @@ -132,3 +132,5 @@ This now needs to be written as: the new ``rand`` procs. The module now exports the state of the random number generator as type ``Rand`` so multiple threads can easily use their own random number generators that do not require locking. +- The compiler is now more consistent in its treatment of ambiguous symbols: + Types that shadow procs and vice versa are marked as ambiguous (bug #6693). diff --git a/compiler/importer.nim b/compiler/importer.nim index 89ee00b9d4..46d675b275 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -27,7 +27,7 @@ proc rawImportSymbol(c: PContext, s: PSym) = # check if we have already a symbol of the same name: var check = strTableGet(c.importTable.symbols, s.name) if check != nil and check.id != s.id: - if s.kind notin OverloadableSyms: + if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms: # s and check need to be qualified: incl(c.ambiguousSymbols, s.id) incl(c.ambiguousSymbols, check.id) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 65b111d8f5..af6322b2b3 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2217,10 +2217,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # XXX think about this more (``set`` procs) if n.len == 2: result = semConv(c, n) - elif n.len == 1: - result = semObjConstr(c, n, flags) elif contains(c.ambiguousSymbols, s.id): errorUseQualifier(c, n.info, s) + elif n.len == 1: + result = semObjConstr(c, n, flags) elif s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) of skProc, skFunc, skMethod, skConverter, skIterator: diff --git a/tests/modules/mrange.nim b/tests/modules/mrange.nim new file mode 100644 index 0000000000..9b78bf24b4 --- /dev/null +++ b/tests/modules/mrange.nim @@ -0,0 +1,2 @@ + +proc range*() = echo "yo" \ No newline at end of file diff --git a/tests/modules/tambig_range.nim b/tests/modules/tambig_range.nim new file mode 100644 index 0000000000..48e0e9f528 --- /dev/null +++ b/tests/modules/tambig_range.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "ambiguous identifier: 'range' --use system.range or mrange.range" + line: 9 +""" + +# bug #6726 +import mrange + +range() From be87fe91768c48e8af45e3b5cbbac20bd5d049dd Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 15 Dec 2017 13:24:47 +0100 Subject: [PATCH 038/200] make tests green again --- compiler/semexprs.nim | 2 ++ tests/misc/tevents.nim | 26 +++++++++++++------------- tests/tuples/tuple_with_nil.nim | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index af6322b2b3..532566a184 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2123,6 +2123,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkIdent, nkAccQuoted: let checks = if efNoEvaluateGeneric in flags: {checkUndeclared, checkPureEnumFields} + elif efInCall in flags: + {checkUndeclared, checkModule, checkPureEnumFields} else: {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields} var s = qualifiedLookUp(c, n, checks) diff --git a/tests/misc/tevents.nim b/tests/misc/tevents.nim index 5f2169f29a..caf674084b 100644 --- a/tests/misc/tevents.nim +++ b/tests/misc/tevents.nim @@ -8,19 +8,19 @@ HandlePrintEvent2: Output -> printing for ME''' import events type - TPrintEventArgs = object of TEventArgs + PrintEventArgs = object of EventArgs user*: string -proc handleprintevent*(e: TEventArgs) = +proc handleprintevent*(e: EventArgs) = write(stdout, "HandlePrintEvent: Output -> Handled print event\n") -proc handleprintevent2*(e: TEventArgs) = - var args: TPrintEventArgs = TPrintEventArgs(e) +proc handleprintevent2*(e: EventArgs) = + var args: PrintEventArgs = PrintEventArgs(e) write(stdout, "HandlePrintEvent2: Output -> printing for " & args.user) var ee = initEventEmitter() -var eventargs: TPrintEventArgs +var eventargs: PrintEventArgs eventargs.user = "ME\n" ##method one test @@ -33,16 +33,16 @@ ee.emit("print", eventargs) ##method two test type - TSomeObject = object of TObject - PrintEvent: TEventHandler + SomeObject = object of RootObj + printEvent: EventHandler -var obj: TSomeObject -obj.PrintEvent = initEventHandler("print") -obj.PrintEvent.addHandler(handleprintevent2) +var obj: SomeObject +obj.printEvent = initEventHandler("print") +obj.printEvent.addHandler(handleprintevent2) -ee.emit(obj.PrintEvent, eventargs) +ee.emit(obj.printEvent, eventargs) -obj.PrintEvent.removeHandler(handleprintevent2) +obj.printEvent.removeHandler(handleprintevent2) -ee.emit(obj.PrintEvent, eventargs) +ee.emit(obj.printEvent, eventargs) diff --git a/tests/tuples/tuple_with_nil.nim b/tests/tuples/tuple_with_nil.nim index 7f5a359f5b..eb265f420b 100644 --- a/tests/tuples/tuple_with_nil.nim +++ b/tests/tuples/tuple_with_nil.nim @@ -485,7 +485,7 @@ proc writeformat(o: var Writer; b: bool; fmt: Format) = else: raise newException(FormatError, "Boolean values must of one of the following types: s,b,o,x,X,d,n") -proc writeformat(o: var Writer; ary: openarray[any]; fmt: Format) = +proc writeformat(o: var Writer; ary: openarray[system.any]; fmt: Format) = ## Write array `ary` according to format `fmt` using output object ## `o` and output function `add`. if ary.len == 0: return From cf9bee1702e41438ad3c55f33f64d179422df807 Mon Sep 17 00:00:00 2001 From: GULPF Date: Fri, 15 Dec 2017 13:59:32 +0100 Subject: [PATCH 039/200] Fix counttable smallest loop start (#6917) * Fix counttable smallest * Fix counttable smallest loop start --- lib/pure/collections/tables.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 38f8f97f5e..01767956ea 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -994,7 +994,7 @@ proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) assert t.len > 0 var minIdx = -1 - for h in 1..high(t.data): + for h in 0..high(t.data): if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val): minIdx = h result.key = t.data[minIdx].key @@ -1332,5 +1332,5 @@ when isMainModule: block: # CountTable.smallest var t = initCountTable[int]() - for v in items([4, 4, 5, 5, 5]): t.inc(v) - doAssert t.smallest == (4, 2) + for v in items([0, 0, 5, 5, 5]): t.inc(v) + doAssert t.smallest == (0, 2) From e0591d494190935c235fd8d89d2bf78d16e8c30d Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 15 Dec 2017 15:33:43 +0100 Subject: [PATCH 040/200] fixes #6626 --- changelog.md | 2 ++ compiler/lambdalifting.nim | 13 +++++++++++++ tests/async/tasync_in_seq_constr.nim | 5 +++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index bf83a79fa5..6007a2d466 100644 --- a/changelog.md +++ b/changelog.md @@ -134,3 +134,5 @@ This now needs to be written as: own random number generators that do not require locking. - The compiler is now more consistent in its treatment of ambiguous symbols: Types that shadow procs and vice versa are marked as ambiguous (bug #6693). +- ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably + in an array, seq or object constructor and is now prevented at compile-time. diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index f8d107c84d..8204395242 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -455,6 +455,7 @@ type LiftingPass = object processed: IntSet envVars: Table[int, PNode] + inContainer: int proc initLiftingPass(fn: PSym): LiftingPass = result.processed = initIntSet() @@ -597,6 +598,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; proc transformYield(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = + if c.inContainer > 0: + localError(n.info, "invalid control flow: 'yield' within a constructor") let state = getStateField(owner) assert state != nil assert state.typ != nil @@ -703,11 +706,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; if not c.processed.containsOrIncl(s.id): #if s.name.s == "temp": # echo renderTree(s.getBody, {renderIds}) + let oldInContainer = c.inContainer + c.inContainer = 0 let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) if c.envvars.getOrDefault(s.id).isNil: s.ast.sons[bodyPos] = body else: s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body) + c.inContainer = oldInContainer if s.typ.callConv == ccClosure: result = symToClosure(n, owner, d, c) elif s.id in d.capturedVars: @@ -733,9 +739,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; n.sons[1] = x.sons[1] of nkLambdaKinds, nkIteratorDef, nkFuncDef: if n.typ != nil and n[namePos].kind == nkSym: + let oldInContainer = c.inContainer + c.inContainer = 0 let m = newSymNode(n[namePos].sym) m.typ = n.typ result = liftCapturedVars(m, owner, d, c) + c.inContainer = oldInContainer of nkHiddenStdConv: if n.len == 2: n.sons[1] = liftCapturedVars(n[1], owner, d, c) @@ -750,8 +759,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; # special case 'when nimVm' due to bug #3636: n.sons[1] = liftCapturedVars(n[1], owner, d, c) return + + let inContainer = n.kind in {nkObjConstr, nkBracket} + if inContainer: inc c.inContainer for i in 0.. Date: Fri, 15 Dec 2017 16:46:27 +0100 Subject: [PATCH 041/200] make boostrapping work --- compiler/semtypes.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index b633c87ab8..f2fda3453b 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1620,6 +1620,7 @@ proc processMagicType(c: PContext, m: PSym) = rawAddSon(m.typ, newTypeS(tyNone, c)) of mPNimrodNode: incl m.typ.flags, tfTriggersCompileTime + of mException: discard else: localError(m.info, errTypeExpected) proc semGenericConstraints(c: PContext, x: PType): PType = From 7c3e00d469efc83e50dfa45ffb4c573a66d6deff Mon Sep 17 00:00:00 2001 From: Eduardo Bart Date: Fri, 15 Dec 2017 14:01:23 -0200 Subject: [PATCH 042/200] Fix icc compiler on linux (#6488) --- compiler/extccomp.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 42c341651b..150dc2eaaf 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -138,12 +138,14 @@ compiler icl: # Intel compilers try to imitate the native ones (gcc and msvc) when defined(windows): result = vcc() + result.name = "icl" + result.compilerExe = "icl" + result.linkerExe = "icl" else: result = gcc() - - result.name = "icl" - result.compilerExe = "icl" - result.linkerExe = "icl" + result.name = "icc" + result.compilerExe = "icc" + result.linkerExe = "icc" # Local C Compiler compiler lcc: From 7a711cc8e17ed356112b00ee237f8bef11f290bd Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 15 Dec 2017 19:12:39 +0100 Subject: [PATCH 043/200] cleanup of the Intel compiler handling; refs #6488 --- compiler/extccomp.nim | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 150dc2eaaf..7a473ea434 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -21,7 +21,7 @@ import type TSystemCC* = enum ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc, - ccTcc, ccPcc, ccUcc, ccIcl + ccTcc, ccPcc, ccUcc, ccIcl, ccIcc TInfoCCProp* = enum # properties of the C compiler: hasSwitchRange, # CC allows ranges in switch statements (GNU C) hasComputedGoto, # CC has computed goto (GNU C extension) @@ -135,17 +135,17 @@ compiler vcc: # Intel C/C++ Compiler compiler icl: - # Intel compilers try to imitate the native ones (gcc and msvc) - when defined(windows): - result = vcc() - result.name = "icl" - result.compilerExe = "icl" - result.linkerExe = "icl" - else: - result = gcc() - result.name = "icc" - result.compilerExe = "icc" - result.linkerExe = "icc" + result = vcc() + result.name = "icl" + result.compilerExe = "icl" + result.linkerExe = "icl" + +# Intel compilers try to imitate the native ones (gcc and msvc) +compiler icc: + result = gcc() + result.name = "icc" + result.compilerExe = "icc" + result.linkerExe = "icc" # Local C Compiler compiler lcc: @@ -329,7 +329,8 @@ const tcc(), pcc(), ucc(), - icl()] + icl(), + icc()] hExt* = ".h" From 1de393800898d6c4909ee3c1194a911d6ed4e350 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Fri, 15 Dec 2017 21:13:57 +0300 Subject: [PATCH 044/200] Added test duration output (#6619) --- tests/testament/tester.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 2f0485135e..ffd945d183 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -158,6 +158,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected, given: string, success: TResultEnum) = let name = test.name.extractFilename & " " & $target & test.options let duration = epochTime() - test.startTime + let durationStr = duration.formatFloat(ffDecimal, precision = 8) backend.writeTestResult(name = name, category = test.cat.string, target = $target, @@ -167,7 +168,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) if success == reSuccess: - styledEcho fgGreen, "PASS: ", fgCyan, name + styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" elif success == reIgnored: styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name else: From 35133d97ef955a4bd19b8922078ff75ff39f7782 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 16 Dec 2017 01:56:38 +0100 Subject: [PATCH 045/200] make niminst compile again --- tools/niminst/niminst.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index e4568dc3ab..ab0ce6a5bb 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -15,7 +15,7 @@ when haveZipLib: import os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - securehash + "../../compiler/securehash" const maxOS = 20 # max number of OSes @@ -283,7 +283,7 @@ proc yesno(p: var CfgParser, v: string): bool = else: quit(errorStr(p, "unknown value; use: yes|no")) proc incl(s: var seq[string], x: string): int = - for i in 0.. Date: Sun, 17 Dec 2017 10:57:05 +0100 Subject: [PATCH 046/200] Name error in example (#6935) Name error, example didn't compile . --- lib/pure/parsecsv.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index ca0f3f9e0d..071858b7c8 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -32,7 +32,7 @@ ## import parsecsv ## import os ## # Prepare a file -## var csv_content = """One,Two,Three,Four +## var content = """One,Two,Three,Four ## 1,2,3,4 ## 10,20,30,40 ## 100,200,300,400 From 822da4b21352ca733c85b0f0303d0cf8f90e5aa6 Mon Sep 17 00:00:00 2001 From: skilchen Date: Sun, 17 Dec 2017 10:57:45 +0100 Subject: [PATCH 047/200] fix #6931 terminal.eraseline() gives OverflowError on Windows (#6933) --- lib/pure/terminal.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 871ac5d391..205aecb33b 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -391,8 +391,8 @@ proc eraseLine*(f: File) = origin.X = 0'i16 if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) - var ht = scrbuf.dwSize.Y - origin.Y - var wt = scrbuf.dwSize.X - origin.X + var ht: DWORD = scrbuf.dwSize.Y - origin.Y + var wt: DWORD = scrbuf.dwSize.X - origin.X if fillConsoleOutputCharacter(h, ' ', ht*wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) From e06d76669aa627cc371d30634acd527c2c868006 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 13:15:19 +0100 Subject: [PATCH 048/200] renderer.nim: fixes a long standing bug that kept triple string literals from being rendered properly --- compiler/renderer.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 6f80afefa7..2092fc67c1 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -860,7 +860,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = a: TContext if n.comment != nil: pushCom(g, n) case n.kind # atoms: - of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal) + of nkTripleStrLit: put(g, tkTripleStrLit, atom(g, n)) of nkEmpty: discard of nkType: put(g, tkInvalid, atom(g, n)) of nkSym, nkIdent: gident(g, n) From eab46c5b7e2d2fe34b4d2c6c53d690940e9d4b99 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 13:42:56 +0100 Subject: [PATCH 049/200] runnableExamples feature: allow import statements and move them to the top level --- compiler/semexprs.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 532566a184..28a97068bf 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1775,6 +1775,13 @@ proc setMs(n: PNode, s: PSym): PNode = n.sons[0] = newSymNode(s) n.sons[0].info = n.info +proc extractImports(n: PNode; result: PNode) = + if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}: + result.add copyTree(n) + n.kind = nkEmpty + return + for i in 0.. Date: Sun, 17 Dec 2017 13:58:40 +0100 Subject: [PATCH 050/200] added new stdlib module 'strformat'; refs #5600; refs #6507 --- changelog.md | 3 + doc/lib.rst | 4 + lib/pure/strformat.nim | 590 +++++++++++++++++++++++++++++++++++++++++ web/website.ini | 2 +- 4 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 lib/pure/strformat.nim diff --git a/changelog.md b/changelog.md index 6007a2d466..b216b0f17f 100644 --- a/changelog.md +++ b/changelog.md @@ -136,3 +136,6 @@ This now needs to be written as: Types that shadow procs and vice versa are marked as ambiguous (bug #6693). - ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably in an array, seq or object constructor and is now prevented at compile-time. +- For string formatting / interpolation a new module + called [strformat](https://nim-lang.org/docs/strformat.html) has been added + to the stdlib. diff --git a/doc/lib.rst b/doc/lib.rst index 959c3ef9bc..6eaf6c788a 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -102,6 +102,10 @@ String handling case of a string, splitting a string into substrings, searching for substrings, replacing substrings. +* `strformat `_ + Macro based standard string interpolation / formatting. Inpired by + Python's ```f``-strings. + * `strmisc `_ This module contains uncommon string handling operations that do not fit with the commonly used operations in strutils. diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim new file mode 100644 index 0000000000..b2198aa406 --- /dev/null +++ b/lib/pure/strformat.nim @@ -0,0 +1,590 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +String `interpolation`:idx: / `format`:idx: inspired by +Python's ``f``-strings. + +Examples: + +.. code-block:: nim + + doAssert fmt"""{"abc":>4}""" == " abc" + doAssert fmt"""{"abc":<4}""" == "abc " + + doAssert fmt"{-12345:08}" == "-0012345" + doAssert fmt"{-1:3}" == "-1 " + doAssert fmt"{-1:03}" == "-01" + doAssert fmt"{16:#X}" == "0x10" + + doAssert fmt"{123.456}" == "123.456" + doAssert fmt"{123.456:>9.3f}" == " 123.456" + doAssert fmt"{123.456:9.3f}" == "123.456 " + doAssert fmt"{123.456:>9.4f}" == " 123.4560" + doAssert fmt"{123.456:>9.0f}" == " 123." + doAssert fmt"{123.456:<9.4f}" == "123.4560 " + + doAssert fmt"{123.456:e}" == "1.234560e+02" + doAssert fmt"{123.456:>13e}" == " 1.234560e+02" + doAssert fmt"{123.456:<13e}" == "1.234560e+02 " + + +An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: + +.. code-block:: nim + var temp = newStringOfCap(educatedCapGuess) + format(key, temp) + format(" is ", temp) + format(value, arg, temp) + format("{z}", temp) + temp + +Parts of the string that are enclosed in the curly braces are interpreted +as Nim code, to escape an ``{`` or ``}`` double it. + +``fmt`` delegates most of the work to an open overloaded set +of ``format`` procs. The required signature for a type ``T`` that supports +formatting is usually ``proc format(x: T; result: var string)`` for efficiency +but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are +used as the fallback implementation. + +This is the concrete lookup algorithm that ``fmt`` uses: + +.. code-block:: nim + + when compiles(format(arg, res)): + format(arg, res) + elif compiles(format(arg)): + res.add format(arg) + elif compiles(add(res, arg)): + res.add(arg) + else: + res.add($arg) + + +The subexpression after the colon +(``arg`` in ``fmt"{key} is {value:arg} {{z}}"``) is an optional argument +passed to ``format``. + +If an optional argument is present the following lookup algorithm is used: + +.. code-block:: nim + + when compiles(format(arg, option, res)): + format(arg, option, res) + else: + res.add format(arg, option) + + +For strings and numeric types the optional argument is a so-called +"standard format specifier". + + +Standard format specifier +========================= + + +The general form of a standard format specifier is:: + + [[fill]align][#][0][minimumwidth][.precision][type] + +The brackets ([]) indicate an optional element. + +The optional align flag can be one of the following: + +'<' + Forces the field to be left-aligned within the available + space (This is the default.) + +'>' + Forces the field to be right-aligned within the available space. + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the alignment +option has no meaning in this case. + +The optional 'fill' character defines the character to be used to pad +the field to the minimum width. The fill character, if present, must be +followed by an alignment flag. + +If the '#' character is present, integers use the 'alternate form' for formatting. +This means that binary, octal, and hexadecimal output will be prefixed +with '0b', '0o', and '0x', respectively. + +'width' is a decimal integer defining the minimum field width. If not specified, +then the field width will be determined by the content. + +If the width field is preceded by a zero ('0') character, this enables +zero-padding. + +The 'precision' is a decimal number indicating how many digits should be displayed +after the decimal point in a floating point conversion. For non-numeric types the +field indicates the maximum field size - in other words, how many characters will +be used from the field content. The precision is ignored for integer conversions. + +Finally, the 'type' determines how the data should be presented. + +The available integer presentation types are: + + +================= ==================================================== + Type Result +================= ==================================================== +``b`` Binary. Outputs the number in base 2. +``d`` Decimal Integer. Outputs the number in base 10. +``o`` Octal format. Outputs the number in base 8. +``x`` Hex format. Outputs the number in base 16, using + lower-case letters for the digits above 9. +``X`` Hex format. Outputs the number in base 16, using + uppercase letters for the digits above 9. +(None) the same as 'd' +================= ==================================================== + + +The available floating point presentation types are: + +================= ==================================================== + Type Result +================= ==================================================== +``e`` Exponent notation. Prints the number in scientific + notation using the letter 'e' to indicate the + exponent. +``E`` Exponent notation. Same as 'e' except it converts + the number to uppercase. +``f`` Fixed point. Displays the number as a fixed-point + number. +``F`` Fixed point. Same as 'f' except it converts the + number to uppercase. +``g`` General format. This prints the number as a + fixed-point number, unless the number is too + large, in which case it switches to 'e' + exponent notation. +``G`` General format. Same as 'g' except switches to 'E' + if the number gets to large. +'' (None) similar to 'g', except that it prints at least one + digit after the decimal point. +================= ==================================================== + + +Future directions +================= + +A curly expression with commas in it like ``{x, argA, argB}`` could be +transformed to ``format(x, argA, argB, res)`` in order to support +formatters that do not need to parse a custom language within a custom +language but instead prefer to use Nim's existing syntax. This also +helps in readability since there is only so much you can cram into +single letter DSLs. + +]## + +import macros, parseutils, unicode +import strutils + +template callFormat(res, arg) {.dirty.} = + when arg is string: + # workaround in order to circumvent 'strutils.format' which matches + # too but doesn't adhere to our protocol. + res.add arg + elif compiles(format(arg, res)): + format(arg, res) + elif compiles(format(arg)): + res.add format(arg) + elif compiles(add(res, arg)): + res.add(arg) + else: + res.add($arg) + +template callFormatOption(res, arg, option) {.dirty.} = + when compiles(format(arg, option, res)): + format(arg, option, res) + else: + res.add format(arg, option) + +macro fmt*(pattern: string): untyped = + ## For a specification of the ``fmt`` macro, see the module level documentation. + runnableExamples: + template check(actual, expected: string) = + doAssert actual == expected + + from strutils import toUpperAscii, repeat + + # Basic tests + let s = "string" + check fmt"{0} {s}", "0 string" + check fmt"{s[0..2].toUpperAscii}", "STR" + check fmt"{-10:04}", "-010" + check fmt"{-10:<04}", "-010" + check fmt"{-10:>04}", "-010" + check fmt"0x{10:02X}", "0x0A" + + check fmt"{10:#04X}", "0x0A" + + check fmt"""{"test":#>5}""", "#test" + check fmt"""{"test":>5}""", " test" + + check fmt"""{"test": <5}""", "test " + check fmt"""{"test":<5}""", "test " + check fmt"{1f:.3f}", "1.000" + check fmt"Hello, {s}!", "Hello, string!" + + # Tests for identifers without parenthesis + check fmt"{s} works{s}", "string worksstring" + check fmt"{s:>7}", " string" + doAssert(not compiles(fmt"{s_works}")) # parsed as identifier `s_works` + + # Misc general tests + check fmt"{{}}", "{}" + check fmt"{0}%", "0%" + check fmt"{0}%asdf", "0%asdf" + check fmt("\n{\"\\n\"}\n"), "\n\n\n" + check fmt"""{"abc"}s""", "abcs" + + # String tests + check fmt"""{"abc"}""", "abc" + check fmt"""{"abc":>4}""", " abc" + check fmt"""{"abc":<4}""", "abc " + check fmt"""{"":>4}""", " " + check fmt"""{"":<4}""", " " + + # Int tests + check fmt"{12345}", "12345" + check fmt"{ - 12345}", "-12345" + check fmt"{12345:6}", "12345 " + check fmt"{12345:>6}", " 12345" + check fmt"{12345:4}", "12345" + check fmt"{12345:08}", "00012345" + check fmt"{-12345:08}", "-0012345" + check fmt"{0:0}", "0" + check fmt"{0:02}", "00" + check fmt"{-1:3}", "-1 " + check fmt"{-1:03}", "-01" + check fmt"{10}", "10" + check fmt"{16:#X}", "0x10" + + # Hex tests + check fmt"{0:x}", "0" + check fmt"{-0:x}", "0" + check fmt"{255:x}", "ff" + check fmt"{255:X}", "FF" + check fmt"{-255:x}", "-ff" + check fmt"{-255:X}", "-FF" + check fmt"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check fmt"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check fmt"{255:>4x}", " ff" + check fmt"{255:04x}", "00ff" + check fmt"{-255:>4x}", " -ff" + check fmt"{-255:04x}", "-0ff" + + # Float tests + check fmt"{123.456}", "123.456" + check fmt"{-123.456}", "-123.456" + check fmt"{123.456:.3f}", "123.456" + check fmt"{-123.456:.3f}", "-123.456" + check fmt"{123.456:1g}", "123.456" + check fmt"{123.456:.1f}", "123.5" + check fmt"{123.456:.0f}", "123." + check fmt"{123.456:>9.3f}", " 123.456" + check fmt"{123.456:9.3f}", "123.456 " + check fmt"{123.456:>9.4f}", " 123.4560" + check fmt"{123.456:>9.0f}", " 123." + check fmt"{123.456:<9.4f}", "123.4560 " + + # Float (scientific) tests + check fmt"{123.456:e}", "1.234560e+02" + check fmt"{123.456:>13e}", " 1.234560e+02" + check fmt"{123.456:<13e}", "1.234560e+02 " + check fmt"{123.456:.1e}", "1.2e+02" + check fmt"{123.456:.2e}", "1.23e+02" + check fmt"{123.456:.3e}", "1.235e+02" + + # Note: times.format adheres to the format protocol. Test that this + # works: + import times + + var nullTime: TimeInfo + check fmt"{nullTime:yyyy-mm-dd}", "0000-00-00" + + # Unicode string tests + check fmt"""{"αβγ"}""", "αβγ" + check fmt"""{"αβγ":>5}""", " αβγ" + check fmt"""{"αβγ":<5}""", "αβγ " + check fmt"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check fmt"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + # Invalid unicode sequences should be handled as plain strings. + # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 + let invalidUtf8 = [ + "\xc3\x28", "\xa0\xa1", + "\xe2\x28\xa1", "\xe2\x82\x28", + "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" + ] + for s in invalidUtf8: + check fmt"{s:>5}", repeat(" ", 5-s.len) & s + + if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: + error "fmt only works with string literals", pattern + let f = pattern.strVal + var i = 0 + let res = genSym(nskVar, "fmtRes") + result = newNimNode(nnkStmtListExpr, lineInfoFrom=pattern) + result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + count(f, '{')*10))) + var strlit = "" + while i < f.len: + if f[i] == '{': + inc i + if f[i] == '{': + inc i + strlit.add '{' + else: + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + strlit = "" + + var subexpr = "" + while i < f.len and f[i] != '}' and f[i] != ':': + subexpr.add f[i] + inc i + let x = parseExpr(subexpr) + + if f[i] == ':': + inc i + var options = "" + while i < f.len and f[i] != '}': + options.add f[i] + inc i + result.add getAst(callFormatOption(res, x, newLit(options))) + else: + result.add getAst(callFormat(res, x)) + if f[i] == '}': + inc i + else: + doAssert false, "invalid format string: missing '}'" + elif f[i] == '}': + if f[i+1] == '}': + strlit.add '}' + inc i, 2 + else: + doAssert false, "invalid format string: '}' instead of '}}'" + inc i + else: + strlit.add f[i] + inc i + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + result.add res + when defined(debugFmtDsl): + echo repr result + +proc mkDigit(v: int, typ: char): string {.inline.} = + assert(v < 26) + if v < 10: + result = $chr(ord('0') + v) + else: + result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10) + +proc alignString*(s: string, minimumWidth: int; align = '<'; fill = ' '): string = + ## Aligns ``s`` using ``fill`` char. + ## This is only of interest if you want to write a custom ``format`` proc that + ## should support the standard format specifiers. + if minimumWidth == 0: + result = s + else: + let sRuneLen = if s.validateUtf8 == -1: s.runeLen else: s.len + let toFill = minimumWidth - sRuneLen + if toFill <= 0: + result = s + elif align == '<': + result = s & repeat(fill, toFill) + else: + result = repeat(fill, toFill) & s + +type + StandardFormatSpecifier* = object ## Type that describes "standard format specifiers". + fill*, align*: char ## Desired fill and alignment. + when false: + sign: char ## Desired sign. + alternateForm*: bool ## Whether to prefix binary, octal and hex numbers + ## with ``0b``, ``0o``, ``0x``. + padWithZero*: bool ## Whether to pad with zeros rather than spaces. + minimumWidth*, precision*: int ## Desired minium width and precision. + typ*: char ## Type like 'f', 'g' or 'd'. + endPosition*: int ## End position in the format specifier after + ## ``parseStandardFormatSpecifier`` returned. + +proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = + ## Converts ``n`` to string. If ``n`` is `SomeReal`, it casts to `int64`. + ## Conversion is done using ``radix``. If result's length is lesser than + ## ``minimumWidth``, it aligns result to the right or left (depending on ``a``) + ## with ``fill`` char. + when n is SomeUnsignedInt: + var v = n.uint64 + let negative = false + else: + var v = n.int64 + let negative = v.int64 < 0 + if negative: + # FIXME: overflow error for low(int64) + v = v * -1 + + var xx = "" + if spec.alternateForm: + case spec.typ + of 'X': xx = "0x" + of 'x': xx = "0x" + of 'b': xx = "0b" + of 'o': xx = "0o" + else: discard + + if v == 0: + result = "0" + else: + result = "" + while v > type(v)(0): + let d = v mod type(v)(radix) + v = v div type(v)(radix) + result.add(mkDigit(d.int, spec.typ)) + for idx in 0..<(result.len div 2): + swap result[idx], result[result.len - idx - 1] + let adjustedWid = if negative: spec.minimumWidth - 1 else: spec.minimumWidth + if spec.padWithZero: + let toFill = spec.minimumWidth - result.len - xx.len - ord(negative) + if toFill > 0: + result = repeat('0', toFill) & result + + if spec.align == '<': + if negative: + result = "-" & xx & result + else: + result = xx & result + for i in result.len.. 0: + result = repeat(spec.fill, toFill) & result + +proc parseStandardFormatSpecifier*(s: string; start = 0; + ignoreUnknownSuffix = false): StandardFormatSpecifier = + ## An exported helper proc that parses the "standard format specifiers", + ## as specified by the grammar:: + ## + ## [[fill]align][#][0][minimumwidth][.precision][type] + ## + ## This is only of interest if you want to write a custom ``format`` proc that + ## should support the standard format specifiers. If ``ignoreUnknownSuffix`` is true, + ## an unknown suffix after the ``type`` field is not an error. + const alignChars = {'<', '>'} + result.fill = ' ' + result.align = '<' + var i = start + if i + 1 < s.len and s[i+1] in alignChars: + result.fill = s[i] + result.align = s[i+1] + inc i, 2 + elif i < s.len and s[i] in alignChars: + result.align = s[i] + inc i + + when false: + # XXX Python inspired 'sign' not yet supported! + if i < s.len and s[i] in {'-', '+', ' '}: + result.sign = s[i] + inc i + + if i < s.len and s[i] == '#': + result.alternateForm = true + inc i + + if i+1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}: + result.padWithZero = true + inc i + + let parsedLength = parseSaturatedNatural(s, result.minimumWidth, i) + inc i, parsedLength + if i < s.len and s[i] == '.': + inc i + let parsedLengthB = parseSaturatedNatural(s, result.precision, i) + inc i, parsedLengthB + else: + result.precision = -1 + + if i < s.len and s[i] in {'A'..'Z', 'a'..'z'}: + result.typ = s[i] + inc i + result.endPosition = i + if i != s.len and not ignoreUnknownSuffix: + raise newException(ValueError, + "invalid format string, cannot parse: " & s[i..^1]) + + +proc format*(value: SomeInteger; specifier: string; res: var string) = + ## Standard format implementation for ``SomeInteger``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + var radix = 10 + case spec.typ + of 'x', 'X': radix = 16 + of 'd', '\0': discard + of 'b': radix = 2 + of 'o': radix = 8 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) + res.add formatInt(value, radix, spec) + +proc format*(value: SomeReal; specifier: string; res: var string) = + ## Standard format implementation for ``SomeReal``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + + var fmode = ffDefault + case spec.typ + of 'e', 'E': + fmode = ffScientific + of 'f', 'F': + fmode = ffDecimal + of 'g', 'G': + fmode = ffDefault + of '\0': discard + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + + #let result = if spec.minimumWidth > 0 and spec.align == '<' and value < 0 and spec.padWithZero: + # "-" & alignString(formatBiggestFloat(-value, fmode, spec.precision), spec.minimumWidth-1, + # spec.align, '0') + #else: + let result = alignString(formatBiggestFloat(value, fmode, spec.precision), spec.minimumWidth, + spec.align, spec.fill) + if spec.typ in {'A'..'Z'}: + res.add toUpperAscii(result) + else: + res.add result + +proc format*(value: string; specifier: string; res: var string) = + ## Standard format implementation for ``string``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + var fmode = ffDefault + case spec.typ + of 's', '\0': discard + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + res.add alignString(value, spec.minimumWidth, spec.align, spec.fill) diff --git a/web/website.ini b/web/website.ini index a158e3b473..5560e67ea5 100644 --- a/web/website.ini +++ b/web/website.ini @@ -64,7 +64,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;impure/rdstdin" +srcdoc2: "pure/fenv;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile" From d244508b8aaad24877991891fe2cb3ee42057795 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 14:23:57 +0100 Subject: [PATCH 051/200] fixes #6932 --- compiler/semexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 28a97068bf..ea51929e2a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2229,7 +2229,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # XXX think about this more (``set`` procs) if n.len == 2: result = semConv(c, n) - elif contains(c.ambiguousSymbols, s.id): + elif contains(c.ambiguousSymbols, s.id) and n.len == 1: errorUseQualifier(c, n.info, s) elif n.len == 1: result = semObjConstr(c, n, flags) From 5c7493f833443f34f2fa3df36aa4648bc818678d Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 17:21:43 +0100 Subject: [PATCH 052/200] strformat: added '^' char for center alignment for Python compat --- lib/pure/strformat.nim | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index b2198aa406..97dff630e6 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -105,6 +105,9 @@ The optional align flag can be one of the following: '>' Forces the field to be right-aligned within the available space. +'^' + Forces the field to be centered within the available space. + Note that unless a minimum field width is defined, the field width will always be the same size as the data to fill it, so that the alignment option has no meaning in this case. @@ -167,7 +170,7 @@ The available floating point presentation types are: exponent notation. ``G`` General format. Same as 'g' except switches to 'E' if the number gets to large. -'' (None) similar to 'g', except that it prints at least one +(None) similar to 'g', except that it prints at least one digit after the decimal point. ================= ==================================================== @@ -229,6 +232,8 @@ macro fmt*(pattern: string): untyped = check fmt"""{"test":#>5}""", "#test" check fmt"""{"test":>5}""", " test" + check fmt"""{"test":#^7}""", "#test##" + check fmt"""{"test": <5}""", "test " check fmt"""{"test":<5}""", "test " check fmt"{1f:.3f}", "1.000" @@ -267,6 +272,7 @@ macro fmt*(pattern: string): untyped = check fmt"{-1:03}", "-01" check fmt"{10}", "10" check fmt"{16:#X}", "0x10" + check fmt"{16:^#7X}", " 0x10 " # Hex tests check fmt"{0:x}", "0" @@ -290,6 +296,7 @@ macro fmt*(pattern: string): untyped = check fmt"{123.456:1g}", "123.456" check fmt"{123.456:.1f}", "123.5" check fmt"{123.456:.0f}", "123." + #check fmt"{123.456:.0f}", "123." check fmt"{123.456:>9.3f}", " 123.456" check fmt"{123.456:9.3f}", "123.456 " check fmt"{123.456:>9.4f}", " 123.4560" @@ -401,6 +408,9 @@ proc alignString*(s: string, minimumWidth: int; align = '<'; fill = ' '): string result = s elif align == '<': result = s & repeat(fill, toFill) + elif align == '^': + let half = toFill div 2 + result = repeat(fill, half) & s & repeat(fill, toFill - half) else: result = repeat(fill, toFill) & s @@ -470,8 +480,12 @@ proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string else: result = xx & result let toFill = spec.minimumWidth - result.len - if toFill > 0: - result = repeat(spec.fill, toFill) & result + if spec.align == '^': + let half = toFill div 2 + result = repeat(spec.fill, half) & result & repeat(spec.fill, toFill - half) + else: + if toFill > 0: + result = repeat(spec.fill, toFill) & result proc parseStandardFormatSpecifier*(s: string; start = 0; ignoreUnknownSuffix = false): StandardFormatSpecifier = @@ -483,7 +497,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; ## This is only of interest if you want to write a custom ``format`` proc that ## should support the standard format specifiers. If ``ignoreUnknownSuffix`` is true, ## an unknown suffix after the ``type`` field is not an error. - const alignChars = {'<', '>'} + const alignChars = {'<', '>', '^'} result.fill = ' ' result.align = '<' var i = start From 2e61e6edf99fef44dcfaf23177714a064dbc05fd Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 17:58:04 +0100 Subject: [PATCH 053/200] strformat: support 'sign' as Python does --- lib/pure/strformat.nim | 61 +++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 97dff630e6..3feb046b7e 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -92,7 +92,7 @@ Standard format specifier The general form of a standard format specifier is:: - [[fill]align][#][0][minimumwidth][.precision][type] + [[fill]align][sign][#][0][minimumwidth][.precision][type] The brackets ([]) indicate an optional element. @@ -116,6 +116,18 @@ The optional 'fill' character defines the character to be used to pad the field to the minimum width. The fill character, if present, must be followed by an alignment flag. +The 'sign' option is only valid for numeric types, and can be one of the following: + +================= ==================================================== + Sign Meaning +================= ==================================================== +``+`` Indicates that a sign should be used for both + positive as well as negative numbers. +``-`` Indicates that a sign should be used only for + negative numbers (this is the default behavior). +`` `` (space) Indicates that a leading space should be used on + positive numbers. + If the '#' character is present, integers use the 'alternate form' for formatting. This means that binary, octal, and hexadecimal output will be prefixed with '0b', '0o', and '0x', respectively. @@ -273,6 +285,7 @@ macro fmt*(pattern: string): untyped = check fmt"{10}", "10" check fmt"{16:#X}", "0x10" check fmt"{16:^#7X}", " 0x10 " + check fmt"{16:^+#7X}", " +0x10 " # Hex tests check fmt"{0:x}", "0" @@ -292,6 +305,8 @@ macro fmt*(pattern: string): untyped = check fmt"{123.456}", "123.456" check fmt"{-123.456}", "-123.456" check fmt"{123.456:.3f}", "123.456" + check fmt"{123.456:+.3f}", "+123.456" + check fmt"{-123.456:+.3f}", "-123.456" check fmt"{-123.456:.3f}", "-123.456" check fmt"{123.456:1g}", "123.456" check fmt"{123.456:.1f}", "123.5" @@ -417,8 +432,7 @@ proc alignString*(s: string, minimumWidth: int; align = '<'; fill = ' '): string type StandardFormatSpecifier* = object ## Type that describes "standard format specifiers". fill*, align*: char ## Desired fill and alignment. - when false: - sign: char ## Desired sign. + sign*: char ## Desired sign. alternateForm*: bool ## Whether to prefix binary, octal and hex numbers ## with ``0b``, ``0o``, ``0x``. padWithZero*: bool ## Whether to pad with zeros rather than spaces. @@ -461,24 +475,23 @@ proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string result.add(mkDigit(d.int, spec.typ)) for idx in 0..<(result.len div 2): swap result[idx], result[result.len - idx - 1] - let adjustedWid = if negative: spec.minimumWidth - 1 else: spec.minimumWidth if spec.padWithZero: - let toFill = spec.minimumWidth - result.len - xx.len - ord(negative) + let sign = negative or spec.sign != '-' + let toFill = spec.minimumWidth - result.len - xx.len - ord(sign) if toFill > 0: result = repeat('0', toFill) & result + if negative: + result = "-" & xx & result + elif spec.sign != '-': + result = spec.sign & xx & result + else: + result = xx & result + if spec.align == '<': - if negative: - result = "-" & xx & result - else: - result = xx & result for i in result.len..', '^'} result.fill = ' ' result.align = '<' + result.sign = '-' var i = start if i + 1 < s.len and s[i+1] in alignChars: result.fill = s[i] @@ -509,11 +523,9 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; result.align = s[i] inc i - when false: - # XXX Python inspired 'sign' not yet supported! - if i < s.len and s[i] in {'-', '+', ' '}: - result.sign = s[i] - inc i + if i < s.len and s[i] in {'-', '+', ' '}: + result.sign = s[i] + inc i if i < s.len and s[i] == '#': result.alternateForm = true @@ -578,12 +590,11 @@ proc format*(value: SomeReal; specifier: string; res: var string) = "invalid type in format string for number, expected one " & " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) - #let result = if spec.minimumWidth > 0 and spec.align == '<' and value < 0 and spec.padWithZero: - # "-" & alignString(formatBiggestFloat(-value, fmode, spec.precision), spec.minimumWidth-1, - # spec.align, '0') - #else: - let result = alignString(formatBiggestFloat(value, fmode, spec.precision), spec.minimumWidth, - spec.align, spec.fill) + var f = formatBiggestFloat(value, fmode, spec.precision) + if value >= 0.0 and spec.sign != '-': + f = spec.sign & f + let result = alignString(f, spec.minimumWidth, + spec.align, spec.fill) if spec.typ in {'A'..'Z'}: res.add toUpperAscii(result) else: From 3fc708288758ca5a6630e4de9647a1987089f570 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 20:34:32 +0100 Subject: [PATCH 054/200] strformat: default for numbers is right alignment --- lib/pure/strformat.nim | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 3feb046b7e..db1bcf9f1a 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -100,10 +100,11 @@ The optional align flag can be one of the following: '<' Forces the field to be left-aligned within the available - space (This is the default.) + space. (This is the default for strings.) '>' Forces the field to be right-aligned within the available space. + (This is the default for numbers.) '^' Forces the field to be centered within the available space. @@ -273,14 +274,14 @@ macro fmt*(pattern: string): untyped = # Int tests check fmt"{12345}", "12345" check fmt"{ - 12345}", "-12345" - check fmt"{12345:6}", "12345 " + check fmt"{12345:6}", " 12345" check fmt"{12345:>6}", " 12345" check fmt"{12345:4}", "12345" check fmt"{12345:08}", "00012345" check fmt"{-12345:08}", "-0012345" check fmt"{0:0}", "0" check fmt"{0:02}", "00" - check fmt"{-1:3}", "-1 " + check fmt"{-1:3}", " -1" check fmt"{-1:03}", "-01" check fmt"{10}", "10" check fmt"{16:#X}", "0x10" @@ -296,9 +297,9 @@ macro fmt*(pattern: string): untyped = check fmt"{-255:X}", "-FF" check fmt"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" check fmt"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check fmt"{255:>4x}", " ff" + check fmt"{255:4x}", " ff" check fmt"{255:04x}", "00ff" - check fmt"{-255:>4x}", " -ff" + check fmt"{-255:4x}", " -ff" check fmt"{-255:04x}", "-0ff" # Float tests @@ -313,7 +314,7 @@ macro fmt*(pattern: string): untyped = check fmt"{123.456:.0f}", "123." #check fmt"{123.456:.0f}", "123." check fmt"{123.456:>9.3f}", " 123.456" - check fmt"{123.456:9.3f}", "123.456 " + check fmt"{123.456:9.3f}", " 123.456" check fmt"{123.456:>9.4f}", " 123.4560" check fmt"{123.456:>9.0f}", " 123." check fmt"{123.456:<9.4f}", "123.4560 " @@ -410,7 +411,7 @@ proc mkDigit(v: int, typ: char): string {.inline.} = else: result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10) -proc alignString*(s: string, minimumWidth: int; align = '<'; fill = ' '): string = +proc alignString*(s: string, minimumWidth: int; align = '\0'; fill = ' '): string = ## Aligns ``s`` using ``fill`` char. ## This is only of interest if you want to write a custom ``format`` proc that ## should support the standard format specifiers. @@ -421,7 +422,7 @@ proc alignString*(s: string, minimumWidth: int; align = '<'; fill = ' '): string let toFill = minimumWidth - sRuneLen if toFill <= 0: result = s - elif align == '<': + elif align == '<' or align == '\0': result = s & repeat(fill, toFill) elif align == '^': let half = toFill div 2 @@ -512,7 +513,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; ## an unknown suffix after the ``type`` field is not an error. const alignChars = {'<', '>', '^'} result.fill = ' ' - result.align = '<' + result.align = '\0' result.sign = '-' var i = start if i + 1 < s.len and s[i+1] in alignChars: @@ -593,8 +594,10 @@ proc format*(value: SomeReal; specifier: string; res: var string) = var f = formatBiggestFloat(value, fmode, spec.precision) if value >= 0.0 and spec.sign != '-': f = spec.sign & f + # the default for numbers is right-alignment: + let align = if spec.align == '\0': '>' else: spec.align let result = alignString(f, spec.minimumWidth, - spec.align, spec.fill) + align, spec.fill) if spec.typ in {'A'..'Z'}: res.add toUpperAscii(result) else: From 69aeb86f49ac39e6a305974840ea5e2abad12717 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 17 Dec 2017 20:40:43 +0100 Subject: [PATCH 055/200] strformat: fix the documentation examples --- lib/pure/strformat.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index db1bcf9f1a..c4044867da 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -19,20 +19,20 @@ Examples: doAssert fmt"""{"abc":<4}""" == "abc " doAssert fmt"{-12345:08}" == "-0012345" - doAssert fmt"{-1:3}" == "-1 " + doAssert fmt"{-1:3}" == " -1" doAssert fmt"{-1:03}" == "-01" doAssert fmt"{16:#X}" == "0x10" doAssert fmt"{123.456}" == "123.456" doAssert fmt"{123.456:>9.3f}" == " 123.456" - doAssert fmt"{123.456:9.3f}" == "123.456 " - doAssert fmt"{123.456:>9.4f}" == " 123.4560" + doAssert fmt"{123.456:9.3f}" == " 123.456" + doAssert fmt"{123.456:9.4f}" == " 123.4560" doAssert fmt"{123.456:>9.0f}" == " 123." doAssert fmt"{123.456:<9.4f}" == "123.4560 " doAssert fmt"{123.456:e}" == "1.234560e+02" doAssert fmt"{123.456:>13e}" == " 1.234560e+02" - doAssert fmt"{123.456:<13e}" == "1.234560e+02 " + doAssert fmt"{123.456:13e}" == " 1.234560e+02" An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: From 3659fec72551bd14f893b8eef6ea606a8b929a49 Mon Sep 17 00:00:00 2001 From: cooldome Date: Sun, 17 Dec 2017 22:56:21 +0000 Subject: [PATCH 056/200] Alternative fix for #4910 that covers #6892; fixes #6892 (#6938) --- compiler/semexprs.nim | 14 ++++++-------- tests/cpp/tget_subsystem.nim | 14 +++++++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ea51929e2a..7867c7e368 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1290,13 +1290,10 @@ proc takeImplicitAddr(c: PContext, n: PNode): PNode = proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le.sons[0] - if x.typ.kind == tyVar and x.kind == nkSym: - if x.sym.kind == skResult: - n.sons[0] = x # 'result[]' --> 'result' - n.sons[1] = takeImplicitAddr(c, ri) - if x.sym.kind != skParam: - # XXX This is hacky. See bug #4910. - x.typ.flags.incl tfVarIsPtr + if x.typ.kind == tyVar and x.kind == nkSym and x.sym.kind == skResult: + n.sons[0] = x # 'result[]' --> 'result' + n.sons[1] = takeImplicitAddr(c, ri) + x.typ.flags.incl tfVarIsPtr #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info template resultTypeIsInferrable(typ: PType): untyped = @@ -1449,14 +1446,15 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = var t = skipTypes(restype, {tyGenericInst, tyAlias}) case t.kind of tyVar: + t.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892 if n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv}: n.sons[0] = n.sons[0].sons[1] - n.sons[0] = takeImplicitAddr(c, n.sons[0]) of tyTuple: for i in 0.. Date: Mon, 18 Dec 2017 04:10:52 -0500 Subject: [PATCH 057/200] Prep for tester parallel: private nimcache for each test (#6937) * Compile tester with --opt:speed This makes "tester html" substantially faster * Use a private nimcache for each test This allows reusing the cache between test runs. It is also prep for parallel testing within a single category (#6913) --- koch.nim | 2 +- tests/testament/categories.nim | 34 ++++++++++++++-------------------- tests/testament/tester.nim | 27 ++++++++++++++++----------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/koch.nim b/koch.nim index cde74966bb..3ef9a340ab 100644 --- a/koch.nim +++ b/koch.nim @@ -441,7 +441,7 @@ template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = # we compile the tester with taintMode:on to have a basic # taint mode test :-) - nimexec "cc --taintMode:on tests/testament/tester" + nimexec "cc --taintMode:on --opt:speed tests/testament/tester" # Since tests take a long time (on my machine), and we want to defy Murhpys # law - lets make sure the compiler really is freshly compiled! nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim" diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 5468fc309d..33b93e3c4d 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -15,58 +15,52 @@ const rodfilesDir = "tests/rodfiles" - nimcacheDir = rodfilesDir / "nimcache" -proc delNimCache() = +proc delNimCache(filename, options: string) = + let dir = nimcacheDir(filename, options) try: - removeDir(nimcacheDir) + removeDir(dir) except OSError: - echo "[Warning] could not delete: ", nimcacheDir + echo "[Warning] could not delete: ", dir proc runRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = + template test(filename: string, clearCacheFirst=false) = + if clearCacheFirst: delNimCache(filename, options) testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) - delNimCache() # test basic recompilation scheme: - test "hallo" + test "hallo", true test "hallo" when false: # test incremental type information: test "hallo2" - delNimCache() # test type converters: - test "aconv" + test "aconv", true test "bconv" - delNimCache() # test G, A, B example from the documentation; test init sections: - test "deada" + test "deada", true test "deada2" - delNimCache() when false: # test method generation: - test "bmethods" + test "bmethods", true test "bmethods2" - delNimCache() # test generics: - test "tgeneric1" + test "tgeneric1", true test "tgeneric2" - delNimCache() proc compileRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = + template test(filename: untyped, clearCacheFirst=true) = + if clearCacheFirst: delNimCache(filename, options) testSpec r, makeTest(rodfilesDir / filename, options, cat) - delNimCache() # test DLL interfacing: - test "gtkex1" + test "gtkex1", true test "gtkex2" - delNimCache() # --------------------- DLL generation tests ---------------------------------- diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index ffd945d183..69b640fa28 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -12,7 +12,7 @@ import parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs, times, sets + algorithm, compiler/nodejs, times, sets, md5 const resultsFile = "testresults.html" @@ -71,8 +71,14 @@ proc getFileDir(filename: string): string = if not result.isAbsolute(): result = getCurrentDir() / result +proc nimcacheDir(filename, options: string): string = + ## Give each test a private nimcache dir so they don't clobber each other's. + return "nimcache" / (filename & '_' & options.getMD5) + proc callCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = + target: TTarget, extraOptions=""): TSpec = + let nimcache = nimcacheDir(filename, options) + let options = options & " --nimCache:" & nimcache.quoteShell & extraOptions let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, "filedir", filename.getFileDir()]) @@ -222,9 +228,10 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarg r.addResult(test, target, expected.msg, given.msg, reSuccess) inc(r.passed) -proc generatedFile(path, name: string, target: TTarget): string = +proc generatedFile(test: TTest, target: TTarget): string = + let (_, name, _) = test.name.splitFile let ext = targetToExt[target] - result = path / "nimcache" / + result = nimcacheDir(test.name, test.options) / (if target == targetJS: "" else: "compiler_") & name.changeFileExt(ext) @@ -234,8 +241,7 @@ proc needsCodegenCheck(spec: TSpec): bool = proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string, given: var TSpec) = try: - let (path, name, _) = test.name.splitFile - let genFile = generatedFile(path, name, target) + let genFile = generatedFile(test, target) let contents = readFile(genFile).string let check = spec.ccodeCheck if check.len > 0: @@ -325,9 +331,8 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = case expected.action of actionCompile: - var given = callCompiler(expected.cmd, test.name, - test.options & " --stdout --hint[Path]:off --hint[Processing]:off", - target) + var given = callCompiler(expected.cmd, test.name, test.options, target, + extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off") compilerOutputTests(test, target, given, expected, r) of actionRun, actionRunNoSpec: # In this branch of code "early return" pattern is clearer than deep @@ -342,8 +347,8 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = let isJsTarget = target == targetJS var exeFile: string if isJsTarget: - let (dir, file, _) = splitFile(tname) - exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache" + let (_, file, _) = splitFile(tname) + exeFile = nimcacheDir(test.name, test.options) / file & ".js" else: exeFile = changeFileExt(tname, ExeExt) From 2502f86d2fa3e07c993cef811536cff29b71a3f0 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 18 Dec 2017 13:12:10 +0100 Subject: [PATCH 058/200] next steps in giving Nim a decent DFA infrastructure --- compiler/dfa.nim | 88 ++++++++++++++++++++++++++++++++++++++++--- compiler/sempass2.nim | 6 +-- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index fbf71d95ca..66a71e8391 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -132,7 +132,7 @@ proc gen(c: var Con; n: PNode) # {.noSideEffect.} proc genWhile(c: var Con; n: PNode) = # L1: # cond, tmp - # fjmp tmp, L2 + # fork tmp, L2 # body # jmp L1 # L2: @@ -168,15 +168,13 @@ proc genIf(c: var Con, n: PNode) = var endings: seq[TPosition] = @[] for i in countup(0, len(n) - 1): var it = n.sons[i] + c.gen(it.sons[0]) if it.len == 2: - c.gen(it.sons[0].sons[1]) - var elsePos = c.forkI(it.sons[0].sons[1]) + let elsePos = c.forkI(it.sons[1]) c.gen(it.sons[1]) if i < sonsLen(n)-1: endings.add(c.gotoI(it.sons[1])) c.patch(elsePos) - else: - c.gen(it.sons[0]) for endPos in endings: c.patch(endPos) proc genAndOr(c: var Con; n: PNode) = @@ -337,6 +335,85 @@ proc gen(c: var Con; n: PNode) = else: discard proc dfa(code: seq[Instr]) = + var u = newSeq[IntSet](code.len) # usages + var d = newSeq[IntSet](code.len) # defs + var backrefs = initTable[int, int]() + for i in 0.. 0 and maxIters > 0 and someChange: + dec maxIters + var pc = w.pop() # w[^1] + var prevPc = -1 + # this simulates a single linear control flow execution: + while pc < code.len and someChange: + # according to the paper, it is better to shrink the working set here + # in this inner loop: + #let widx = w.find(pc) + #if widx >= 0: w.del(widx) + + if prevPc >= 0: + someChange = false + # merge step and test for changes (we compute the fixpoints here): + # 'u' needs to be the union of prevPc, pc + # 'd' needs to be the intersection of 'pc' + for id in u[prevPc]: + if not u[pc].containsOrIncl(id): + someChange = true + # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection + # is only interesting on merge points: + for id in d[prevPc]: + if not d[pc].containsOrIncl(id): + someChange = true + # if this is a merge point, we take the intersection of the 'd' sets: + if backrefs.hasKey(pc): + var intersect = initIntSet() + assign(intersect, d[pc]) + var first = true + for prevPc in backrefs.allValues(pc): + for def in d[pc]: + if def notin d[prevPc]: + excl(intersect, def) + someChange = true + assign d[pc], intersect + + # our interpretation ![I!]: + prevPc = pc + case code[pc].kind + of goto: + # we must leave endless loops eventually: + #if someChange: + pc = pc + code[pc].dest + #else: + # inc pc + of fork: + # we follow the next instruction but push the dest onto our "work" stack: + #if someChange: + w.add pc + code[pc].dest + inc pc + of use, useWithinCall, def: + inc pc + + # now check the condition we're interested in: + for i in 0.. Date: Mon, 18 Dec 2017 13:16:29 +0100 Subject: [PATCH 059/200] added new stdlib module 'cstrutils' for easier cstring handling --- lib/pure/cstrutils.nim | 79 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 lib/pure/cstrutils.nim diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim new file mode 100644 index 0000000000..4371408925 --- /dev/null +++ b/lib/pure/cstrutils.nim @@ -0,0 +1,79 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module supports helper routines for working with ``cstring`` +## without having to convert ``cstring`` to ``string`` in order to +## save allocations. + +include "system/inclrtl" + +proc toLowerAscii(c: char): char {.inline.} = + if c in {'A'..'Z'}: + result = chr(ord(c) + (ord('a') - ord('A'))) + else: + result = c + +proc startsWith*(s, prefix: cstring): bool {.noSideEffect, + rtl, extern: "csuStartsWith".} = + ## Returns true iff ``s`` starts with ``prefix``. + ## + ## If ``prefix == ""`` true is returned. + var i = 0 + while true: + if prefix[i] == '\0': return true + if s[i] != prefix[i]: return false + inc(i) + +proc endsWith*(s, suffix: cstring): bool {.noSideEffect, + rtl, extern: "csuEndsWith".} = + ## Returns true iff ``s`` ends with ``suffix``. + ## + ## If ``suffix == ""`` true is returned. + let slen = s.len + var i = 0 + var j = slen - len(suffix) + while i+j <% slen: + if s[i+j] != suffix[i]: return false + inc(i) + if suffix[i] == '\0': return true + +proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect, + rtl, extern: "csuCmpIgnoreStyle".} = + ## Compares two strings normalized (i.e. case and + ## underscores do not matter). Returns: + ## + ## | 0 iff a == b + ## | < 0 iff a < b + ## | > 0 iff a > b + var i = 0 + var j = 0 + while true: + while a[i] == '_': inc(i) + while b[j] == '_': inc(j) # BUGFIX: typo + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) + inc(j) + +proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect, + rtl, extern: "csuCmpIgnoreCase".} = + ## Compares two strings in a case insensitive manner. Returns: + ## + ## | 0 iff a == b + ## | < 0 iff a < b + ## | > 0 iff a > b + var i = 0 + while true: + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[i]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) From 07fe1aa655dc75eec1a4cf4c697615b5642e8a7c Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Mon, 18 Dec 2017 11:49:49 -0500 Subject: [PATCH 060/200] Use escape sequences rather than hex in string/char literals (#6941) This should makes documentation easier to read for people who haven't committed the ascii table to memory. --- compiler/renderer.nim | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 2092fc67c1..267ce7de79 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -175,8 +175,17 @@ proc put(g: var TSrcGen, kind: TTokType, s: string) = proc toNimChar(c: char): string = case c - of '\0': result = "\\0" - of '\x01'..'\x1F', '\x80'..'\xFF': result = "\\x" & strutils.toHex(ord(c), 2) + of '\0': result = "\\x00" # not "\\0" to avoid ambiguous cases like "\\012". + of '\a': result = "\\a" # \x07 + of '\b': result = "\\b" # \x08 + of '\t': result = "\\t" # \x09 + of '\L': result = "\\L" # \x0A + of '\v': result = "\\v" # \x0B + of '\f': result = "\\f" # \x0C + of '\c': result = "\\c" # \x0D + of '\e': result = "\\e" # \x1B + of '\x01'..'\x06', '\x0E'..'\x1A', '\x1C'..'\x1F', '\x80'..'\xFF': + result = "\\x" & strutils.toHex(ord(c), 2) of '\'', '\"', '\\': result = '\\' & c else: result = c & "" From a879973081e2c29d64e9fb9d8e539aa980533b10 Mon Sep 17 00:00:00 2001 From: GULPF Date: Mon, 18 Dec 2017 23:11:28 +0100 Subject: [PATCH 061/200] Better times module (#6552) * First work on better timezones * Update tests to new api. Removed tests for checking that `isDst` was included when formatting, since `isDst` no longer affects utc offset (the entire utc offset is stored directly in `utcOffset` instead). * Deprecate getLocaltime & getGmTime * Add `now()` as a shorthand for GetTIme().inZone(Local) * Add FedericoCeratto's timezone tests (#6548) * Run more tests in all timezones * Make month enum start at 1 instead of 0 * Deprecate getDayOfWeekJulian * Fix issues with gc safety * Rename TimeInfo => DateTime * Fixes #6465 * Improve isLeapYear * FIx handling negative adjTime * Cleanup: - deprecated toSeconds and fromSeconds, added fromUnix and toUnix instead (that returns int64 instead of float) - added missing doc comments - removed some unnecessary JS specific implementations * Fix misstake in JS `-` for Time * Update usage of TimeEffect * Removed unecessary use of `difftime` * JS fix for local tz * Fix subtraction of months * Fix `days` field in `toTimeInterval` * Style and docs * Fix getDayOfYear for real this time... * Fix handling of adding/subtracting time across dst transitions * Fix some bad usage of the times module in the stdlib * Revert to use proper time resoultion for seeding in random.nim * Move deprecated procs to bottom of file * Always use `epochTime` in `randomize` * Remove TimeInterval normalization * Fixes #6905 * Fix getDayOfWeek for year < 1 * Export toEpochDay/fromEpochDay and change year/month/monthday order * Add asserts for checking that the monthday is valid * Fix some remaining ambiguous references to `Time` * Fix ambiguous reference to Time --- lib/posix/posix.nim | 3 +- lib/posix/posix_linux_amd64.nim | 6 +- lib/posix/posix_other.nim | 36 +- lib/pure/cookies.nim | 6 +- lib/pure/ioselects/ioselectors_epoll.nim | 7 +- lib/pure/ioselects/ioselectors_kqueue.nim | 4 +- lib/pure/oids.nim | 2 +- lib/pure/os.nim | 35 +- lib/pure/osproc.nim | 16 +- lib/pure/random.nim | 8 +- lib/pure/times.nim | 1633 +++++++++++---------- tests/js/ttimes.nim | 47 +- tests/stdlib/ttimes.nim | 364 +++-- 13 files changed, 1153 insertions(+), 1014 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index b635c0b0b1..fba35868ca 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -609,11 +609,12 @@ proc clock_nanosleep*(a1: ClockId, a2: cint, a3: var Timespec, proc clock_settime*(a1: ClockId, a2: var Timespec): cint {. importc, header: "".} +proc `==`*(a, b: Time): bool {.borrow.} +proc `-`*(a, b: Time): Time {.borrow.} proc ctime*(a1: var Time): cstring {.importc, header: "".} proc ctime_r*(a1: var Time, a2: cstring): cstring {.importc, header: "".} proc difftime*(a1, a2: Time): cdouble {.importc, header: "".} proc getdate*(a1: cstring): ptr Tm {.importc, header: "".} - proc gmtime*(a1: var Time): ptr Tm {.importc, header: "".} proc gmtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "".} proc localtime*(a1: var Time): ptr Tm {.importc, header: "".} diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim index c44128b16c..9e6211b633 100644 --- a/lib/posix/posix_linux_amd64.nim +++ b/lib/posix/posix_linux_amd64.nim @@ -12,8 +12,6 @@ # To be included from posix.nim! -from times import Time - const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) @@ -40,13 +38,15 @@ type const SIG_HOLD* = cast[SigHandler](2) type + Time* {.importc: "time_t", header: "".} = distinct clong + Timespec* {.importc: "struct timespec", header: "", final, pure.} = object ## struct timespec tv_sec*: Time ## Seconds. tv_nsec*: clong ## Nanoseconds. Dirent* {.importc: "struct dirent", - header: "", final, pure.} = object ## dirent_t struct + header: "", final, pure.} = object ## dirent_t struct d_ino*: Ino d_off*: Off d_reclen*: cushort diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 7321889a8f..e552bf807e 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -9,8 +9,6 @@ {.deadCodeElim:on.} -from times import Time - const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) @@ -36,6 +34,8 @@ type {.deprecated: [TSocketHandle: SocketHandle].} type + Time* {.importc: "time_t", header: "".} = distinct int + Timespec* {.importc: "struct timespec", header: "", final, pure.} = object ## struct timespec tv_sec*: Time ## Seconds. @@ -209,24 +209,24 @@ type st_gid*: Gid ## Group ID of file. st_rdev*: Dev ## Device ID (if file is character or block special). st_size*: Off ## For regular files, the file size in bytes. - ## For symbolic links, the length in bytes of the - ## pathname contained in the symbolic link. - ## For a shared memory object, the length in bytes. - ## For a typed memory object, the length in bytes. - ## For other file types, the use of this field is - ## unspecified. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. when defined(macosx) or defined(android): - st_atime*: Time ## Time of last access. - st_mtime*: Time ## Time of last data modification. - st_ctime*: Time ## Time of last status change. + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. else: - st_atim*: Timespec ## Time of last access. - st_mtim*: Timespec ## Time of last data modification. - st_ctim*: Timespec ## Time of last status change. - st_blksize*: Blksize ## A file system-specific preferred I/O block size - ## for this object. In some file system types, this - ## may vary from file to file. - st_blocks*: Blkcnt ## Number of blocks allocated for this object. + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. Statvfs* {.importc: "struct statvfs", header: "", diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 07b37c7d48..8f16717ac1 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -51,7 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "", if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") -proc setCookie*(key, value: string, expires: TimeInfo, +proc setCookie*(key, value: string, expires: DateTime, domain = "", path = "", noName = false, secure = false, httpOnly = false): string = ## Creates a command in the format of @@ -63,9 +63,9 @@ proc setCookie*(key, value: string, expires: TimeInfo, noname, secure, httpOnly) when isMainModule: - var tim = Time(int(getTime()) + 76 * (60 * 60 * 24)) + var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24)) - let cookie = setCookie("test", "value", tim.getGMTime()) + let cookie = setCookie("test", "value", tim.utc) when not defined(testing): echo cookie let start = "Set-Cookie: test=value; Expires=" diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 35cdace09e..8827f239f2 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -277,15 +277,16 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, var events = {Event.Timer} var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint + if oneshot: - new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_sec = posix.Time(0) new_ts.it_interval.tv_nsec = 0 - new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_sec = posix.Time(timeout div 1_000) new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 incl(events, Event.Oneshot) epv.events = epv.events or EPOLLONESHOT else: - new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_sec = posix.Time(timeout div 1000) new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 3e2ec64a80..af5aa15dfa 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -452,10 +452,10 @@ proc selectInto*[T](s: Selector[T], timeout: int, if timeout != -1: if timeout >= 1000: - tv.tv_sec = (timeout div 1_000).Time + tv.tv_sec = posix.Time(timeout div 1_000) tv.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tv.tv_sec = 0.Time + tv.tv_sec = posix.Time(0) tv.tv_nsec = timeout * 1_000_000 else: ptv = nil diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 60b53dbe02..427a68964c 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -88,7 +88,7 @@ proc generatedTime*(oid: Oid): Time = var tmp: int32 var dummy = oid.time bigEndian32(addr(tmp), addr(dummy)) - result = Time(tmp) + result = fromUnix(tmp) when not defined(testing) and isMainModule: let xo = genOid() diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a59134007a..87f6def292 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -173,33 +173,33 @@ proc findExe*(exe: string, followSymlinks: bool = true; return x result = "" -proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s last modification time. when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_mtime + return fromUnix(res.st_mtime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)).int64) findClose(h) -proc getLastAccessTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s last read or write access time. when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_atime + return fromUnix(res.st_atime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)).int64) findClose(h) -proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s creation time. ## ## **Note:** Under POSIX OS's, the returned time may actually be the time at @@ -208,12 +208,12 @@ proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_ctime + return fromUnix(res.st_ctime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftCreationTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftCreationTime)).int64) findClose(h) proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = @@ -1443,7 +1443,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect].} = winlean.sleep(int32(milsecs)) else: var a, b: Timespec - a.tv_sec = Time(milsecs div 1000) + a.tv_sec = posix.Time(milsecs div 1000) a.tv_nsec = (milsecs mod 1000) * 1000 * 1000 discard posix.nanosleep(a, b) @@ -1481,16 +1481,17 @@ type size*: BiggestInt # Size of file. permissions*: set[FilePermission] # File permissions linkCount*: BiggestInt # Number of hard links the file object has. - lastAccessTime*: Time # Time file was last accessed. - lastWriteTime*: Time # Time file was last modified/written to. - creationTime*: Time # Time file was created. Not supported on all systems! + lastAccessTime*: times.Time # Time file was last accessed. + lastWriteTime*: times.Time # Time file was last modified/written to. + creationTime*: times.Time # Time file was created. Not supported on all systems! template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e: FILETIME): untyped {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics + template toTime(e: FILETIME): untyped {.gensym.} = + fromUnix(winTimeToUnixTime(rdFileTime(e)).int64) # local templates default to bind semantics template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) @@ -1522,9 +1523,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) formalInfo.size = rawInfo.st_size formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt - formalInfo.lastAccessTime = rawInfo.st_atime - formalInfo.lastWriteTime = rawInfo.st_mtime - formalInfo.creationTime = rawInfo.st_ctime + formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64) + formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64) + formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64) result.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index f0542ea980..1625845d18 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1060,10 +1060,10 @@ elif not defined(useNimRtl): var tmspec: Timespec if timeout >= 1000: - tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tmspec.tv_sec = 0.Time + tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: @@ -1109,20 +1109,20 @@ elif not defined(useNimRtl): var b: Timespec b.tv_sec = e.tv_sec b.tv_nsec = e.tv_nsec - e.tv_sec = (e.tv_sec - s.tv_sec).Time + e.tv_sec = e.tv_sec - s.tv_sec if e.tv_nsec >= s.tv_nsec: e.tv_nsec -= s.tv_nsec else: - if e.tv_sec == 0.Time: + if e.tv_sec == posix.Time(0): raise newException(ValueError, "System time was modified") else: diff = s.tv_nsec - e.tv_nsec e.tv_nsec = 1_000_000_000 - diff - t.tv_sec = (t.tv_sec - e.tv_sec).Time + t.tv_sec = t.tv_sec - e.tv_sec if t.tv_nsec >= e.tv_nsec: t.tv_nsec -= e.tv_nsec else: - t.tv_sec = (int(t.tv_sec) - 1).Time + t.tv_sec = t.tv_sec - posix.Time(1) diff = e.tv_nsec - t.tv_nsec t.tv_nsec = 1_000_000_000 - diff s.tv_sec = b.tv_sec @@ -1154,10 +1154,10 @@ elif not defined(useNimRtl): raiseOSError(osLastError()) if timeout >= 1000: - tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tmspec.tv_sec = 0.Time + tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 7edd93c088..de419b9fbf 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -190,12 +190,8 @@ when not defined(nimscript): proc randomize*() {.benign.} = ## Initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does not work for NimScript. - when defined(JS): - proc getMil(t: Time): int {.importcpp: "getTime", nodecl.} - randomize(getMil times.getTime()) - else: - let time = int64(times.epochTime() * 1_000_000_000) - randomize(time) + let time = int64(times.epochTime() * 1_000_000_000) + randomize(time) {.pop.} diff --git a/lib/pure/times.nim b/lib/pure/times.nim index c1d6c3e530..dcc817b7b2 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -10,27 +10,26 @@ ## This module contains routines and types for dealing with time. ## This module is available for the `JavaScript target -## `_. +## `_. The proleptic Gregorian calendar is the only calendar supported. ## ## Examples: ## ## .. code-block:: nim ## ## import times, os -## var -## t = cpuTime() +## let time = cpuTime() ## ## sleep(100) # replace this with something to be timed -## echo "Time taken: ",cpuTime() - t +## echo "Time taken: ",cpuTime() - time ## -## echo "My formatted time: ", format(getLocalTime(getTime()), "d MMMM yyyy HH:mm") +## echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") ## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() ## ## echo "epochTime() float value: ", epochTime() ## echo "getTime() float value: ", toSeconds(getTime()) ## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", getLocalTime(getTime()) + 1.hours -## echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1) +## echo "An hour from now : ", now() + 1.hours +## echo "An hour from (UTC) now: ", getTime().utc + initInterval(0,0,0,1) {.push debugger:off.} # the user does not want to trace a part # of the standard library! @@ -40,132 +39,85 @@ import include "system/inclrtl" -type - Month* = enum ## represents a month - mJan, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec - WeekDay* = enum ## represents a weekday - dMon, dTue, dWed, dThu, dFri, dSat, dSun +when defined(posix): + import posix -when defined(posix) and not defined(JS): - when defined(linux) and defined(amd64): - type - TimeImpl {.importc: "time_t", header: "".} = clong - Time* = distinct TimeImpl ## distinct type that represents a time - ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", - header: "".} = object ## struct timeval - tv_sec: clong ## Seconds. - tv_usec: clong ## Microseconds. - else: - type - TimeImpl {.importc: "time_t", header: "".} = int - Time* = distinct TimeImpl ## distinct type that represents a time - ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", - header: "".} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. - - # we cannot import posix.nim here, because posix.nim depends on times.nim. - # Ok, we could, but I don't want circular dependencies. - # And gettimeofday() is not defined in the posix module anyway. Sigh. + type CTime = posix.Time proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "".} when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "".}: int - proc tzset(): void {.importc, header: "".} tzset() elif defined(windows): import winlean # newest version of Visual C++ defines time_t to be of 64 bits - type TimeImpl {.importc: "time_t", header: "".} = int64 + type CTime {.importc: "time_t", header: "".} = distinct int64 # visual c's c runtime exposes these under a different name - var - timezone {.importc: "_timezone", header: "".}: int - - type - Time* = distinct TimeImpl - - -elif defined(JS): - type - TimeBase = float - Time* = distinct TimeBase - - proc getDay(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getHours(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getTime(t: Time): int {.tags: [], raises: [], noSideEffect, benign, importcpp.} - proc getTimezoneOffset(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getDate(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDate(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCHours(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDay(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc parse(t: Time; s: cstring): Time {.tags: [], raises: [], benign, importcpp.} - proc setDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setTime(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc toGMTString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.} - proc toLocaleString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.} + var timezone {.importc: "_timezone", header: "".}: int type - TimeInfo* = object of RootObj ## represents a time in different parts - second*: range[0..61] ## The number of seconds after the minute, - ## normally in the range 0 to 59, but can - ## be up to 61 to allow for leap seconds. - minute*: range[0..59] ## The number of minutes after the hour, - ## in the range 0 to 59. - hour*: range[0..23] ## The number of hours past midnight, - ## in the range 0 to 23. - monthday*: range[1..31] ## The day of the month, in the range 1 to 31. - month*: Month ## The current month. - year*: int ## The current year. - weekday*: WeekDay ## The current day of the week. - yearday*: range[0..365] ## The number of days since January 1, - ## in the range 0 to 365. - ## Always 0 if the target is JS. - isDST*: bool ## Determines whether DST is in effect. - ## Semantically, this adds another negative hour - ## offset to the time in addition to the timezone. - timezone*: int ## The offset of the (non-DST) timezone in seconds - ## west of UTC. Note that the sign of this number - ## is the opposite of the one in a formatted - ## timezone string like ``+01:00`` (which would be - ## parsed into the timezone ``-3600``). + Month* = enum ## Represents a month. Note that the enum starts at ``1``, so ``ord(month)`` will give + ## the month number in the range ``[1..12]``. + mJan = 1, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec - ## I make some assumptions about the data in here. Either - ## everything should be positive or everything negative. Zero is - ## fine too. Mixed signs will lead to unexpected results. - TimeInterval* = object ## a time interval + WeekDay* = enum ## Represents a weekday. + dMon, dTue, dWed, dThu, dFri, dSat, dSun + + MonthdayRange* = range[1..31] + HourRange* = range[0..23] + MinuteRange* = range[0..59] + SecondRange* = range[0..60] + YeardayRange* = range[0..365] + + TimeImpl = int64 + + Time* = distinct TimeImpl ## Represents a point in time. + ## This is currently implemented as a ``int64`` representing + ## seconds since ``1970-01-01T00:00:00Z``, but don't + ## rely on this knowledge because it might change + ## in the future to allow for higher precision. + ## Use the procs ``toUnix`` and ``fromUnix`` to + ## work with unix timestamps instead. + + DateTime* = object of RootObj ## Represents a time in different parts. + ## Although this type can represent leap + ## seconds, they are generally not supported + ## in this module. They are not ignored, + ## but the ``DateTime``'s returned by + ## procedures in this module will never have + ## a leap second. + second*: SecondRange ## The number of seconds after the minute, + ## normally in the range 0 to 59, but can + ## be up to 60 to allow for a leap second. + minute*: MinuteRange ## The number of minutes after the hour, + ## in the range 0 to 59. + hour*: HourRange ## The number of hours past midnight, + ## in the range 0 to 23. + monthday*: MonthdayRange ## The day of the month, in the range 1 to 31. + month*: Month ## The current month. + year*: int ## The current year, using astronomical year numbering + ## (meaning that before year 1 is year 0, then year -1 and so on). + weekday*: WeekDay ## The current day of the week. + yearday*: YeardayRange ## The number of days since January 1, + ## in the range 0 to 365. + isDst*: bool ## Determines whether DST is in effect. + ## Always false for the JavaScript backend. + timezone*: Timezone ## The timezone represented as an implementation of ``Timezone``. + utcOffset*: int ## The offset in seconds west of UTC, including any offset due to DST. + ## Note that the sign of this number is the opposite + ## of the one in a formatted offset string like ``+01:00`` + ## (which would be parsed into the UTC offset ``-3600``). + + TimeInterval* = object ## Represents a duration of time. Can be used to add and subtract + ## from a ``DateTime`` or ``Time``. + ## Note that a ``TimeInterval`` doesn't represent a fixed duration of time, + ## since the duration of some units depend on the context (e.g a year + ## can be either 365 or 366 days long). The non-fixed time units are years, + ## months and days. milliseconds*: int ## The number of milliseconds seconds*: int ## The number of seconds minutes*: int ## The number of minutes @@ -174,92 +126,394 @@ type months*: int ## The number of months years*: int ## The number of years + Timezone* = object ## Timezone interface for supporting ``DateTime``'s of arbritary timezones. + ## The ``times`` module only supplies implementations for the systems local time and UTC. + ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly + ## and are only exported so that ``Timezone`` can be implemented by other modules. + zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. + ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. + ## This type is only used for implementing timezones. + adjTime*: Time ## Time adjusted to a timezone. + utcOffset*: int + isDst*: bool + {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, - TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} + TTimeInterval: TimeInterval, TTimeInfo: DateTime, TimeInfo: DateTime].} -proc getTime*(): Time {.tags: [TimeEffect], benign.} - ## gets the current calendar time as a UNIX epoch value (number of seconds - ## elapsed since 1970) with integer precission. Use epochTime for higher - ## resolution. -proc getLocalTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-time representation, - ## expressed relative to the user's specified time zone. -proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). +const + secondsInMin = 60 + secondsInHour = 60*60 + secondsInDay = 60*60*24 + minutesInHour = 60 -proc timeInfoToTime*(timeInfo: TimeInfo): Time - {.tags: [TimeEffect], benign, deprecated.} - ## converts a broken-down time structure to - ## calendar time representation. The function ignores the specified - ## contents of the structure members `weekday` and `yearday` and recomputes - ## them from the other information in the broken-down time structure. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTime`` instead. +proc fromUnix*(unix: int64): Time {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) to a ``Time``. + Time(unix) -proc toTime*(timeInfo: TimeInfo): Time {.tags: [TimeEffect], benign.} - ## converts a broken-down time structure to - ## calendar time representation. The function ignores the specified - ## contents of the structure members `weekday` and `yearday` and recomputes - ## them from the other information in the broken-down time structure. +proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert ``t`` to a unix timestamp (seconds since ``1970-01-01T00:00:00Z``). + t.int64 -proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign.} - ## Takes a float which contains the number of seconds since the unix epoch and - ## returns a time object. +proc isLeapYear*(year: int): bool = + ## Returns true if ``year`` is a leap year. + year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) -proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign.} = - ## Takes an int which contains the number of seconds since the unix epoch and - ## returns a time object. - fromSeconds(float(since1970)) +proc getDaysInMonth*(month: Month, year: int): int = + ## Get the number of days in a ``month`` of a ``year``. + # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month + case month + of mFeb: result = if isLeapYear(year): 29 else: 28 + of mApr, mJun, mSep, mNov: result = 30 + else: result = 31 -proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.} - ## Returns the time in seconds since the unix epoch. +proc getDaysInYear*(year: int): int = + ## Get the number of days in a ``year`` + result = 365 + (if isLeapYear(year): 1 else: 0) + +proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = + assert monthday <= getDaysInMonth(month, year), + $year & "-" & $ord(month) & "-" & $monthday & " is not a valid date" + +proc toEpochDay*(monthday: MonthdayRange, month: Month, year: int): int64 = + ## Get the epoch day from a year/month/day date. + ## The epoch day is the number of days since 1970/01/01 (it might be negative). + assertValidDate monthday, month, year + # Based on http://howardhinnant.github.io/date_algorithms.html + var (y, m, d) = (year, ord(month), monthday.int) + if m <= 2: + y.dec + + let era = (if y >= 0: y else: y-399) div 400 + let yoe = y - era * 400 + let doy = (153 * (m + (if m > 2: -3 else: 9)) + 2) div 5 + d-1 + let doe = yoe * 365 + yoe div 4 - yoe div 100 + doy + return era * 146097 + doe - 719468 + +proc fromEpochDay*(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = + ## Get the year/month/day date from a epoch day. + ## The epoch day is the number of days since 1970/01/01 (it might be negative). + # Based on http://howardhinnant.github.io/date_algorithms.html + var z = epochday + z.inc 719468 + let era = (if z >= 0: z else: z - 146096) div 146097 + let doe = z - era * 146097 + let yoe = (doe - doe div 1460 + doe div 36524 - doe div 146096) div 365 + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe div 4 - yoe div 100) + let mp = (5 * doy + 2) div 153 + let d = doy - (153 * mp + 2) div 5 + 1 + let m = mp + (if mp < 10: 3 else: -9) + return (d.MonthdayRange, m.Month, (y + ord(m <= 2)).int) + +proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign .} = + ## Returns the day of the year. + ## Equivalent with ``initDateTime(day, month, year).yearday``. + assertValidDate monthday, month, year + const daysUntilMonth: array[Month, int] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] + const daysUntilMonthLeap: array[Month, int] = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] + + if isLeapYear(year): + result = daysUntilMonthLeap[month] + monthday - 1 + else: + result = daysUntilMonth[month] + monthday - 1 + +proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign .} = + ## Returns the day of the week enum from day, month and year. + ## Equivalent with ``initDateTime(day, month, year).weekday``. + assertValidDate monthday, month, year + # 1970-01-01 is a Thursday, we adjust to the previous Monday + let days = toEpochday(monthday, month, year) - 3 + let weeks = (if days >= 0: days else: days - 6) div 7 + let wd = days - weeks * 7 + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. + result = if wd == 0: dSun else: WeekDay(wd - 1) + +# Forward declarations +proc utcZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc utcZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign.} - ## computes the difference of two calendar times. Result is in seconds. + rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign, deprecated.} = + ## Computes the difference of two calendar times. Result is in seconds. + ## This is deprecated because it will need to change when sub second time resolution is implemented. + ## Use ``a.toUnix - b.toUnix`` instead. ## ## .. code-block:: nim ## let a = fromSeconds(1_000_000_000) ## let b = fromSeconds(1_500_000_000) ## echo initInterval(seconds=int(b - a)) ## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0) + a.toUnix - b.toUnix proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect.} = - ## returns true iff ``a < b``, that is iff a happened before b. - when defined(js): - result = TimeBase(a) < TimeBase(b) - else: - result = a - b < 0 + rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true iff ``a < b``, that is iff a happened before b. proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect.}= - ## returns true iff ``a <= b``. - when defined(js): - result = TimeBase(a) <= TimeBase(b) - else: - result = a - b <= 0 + rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true iff ``a <= b``. proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect.} = - ## returns true if ``a == b``, that is if both times represent the same value - when defined(js): - result = TimeBase(a) == TimeBase(b) + rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true if ``a == b``, that is if both times represent the same point in time. + +proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = + ## Converts a broken-down time structure to + ## calendar time representation. + let epochDay = toEpochday(dt.monthday, dt.month, dt.year) + result = Time(epochDay * secondsInDay) + result.inc dt.hour * secondsInHour + result.inc dt.minute * 60 + result.inc dt.second + # The code above ignores the UTC offset of `timeInfo`, + # so we need to compensate for that here. + result.inc dt.utcOffset + +proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = + let adjTime = zt.adjTime.int64 + let epochday = (if adjTime >= 0: adjTime else: adjTime - (secondsInDay - 1)) div secondsInDay + var rem = zt.adjTime.int64 - epochday * secondsInDay + let hour = rem div secondsInHour + rem = rem - hour * secondsInHour + let minute = rem div secondsInMin + rem = rem - minute * secondsInMin + let second = rem + + let (d, m, y) = fromEpochday(epochday) + + DateTime( + year: y, + month: m, + monthday: d, + hour: hour, + minute: minute, + second: second, + weekday: getDayOfWeek(d, m, y), + yearday: getDayOfYear(d, m, y), + isDst: zt.isDst, + timezone: zone, + utcOffset: zt.utcOffset + ) + +proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Break down ``time`` into a ``DateTime`` using ``zone`` as the timezone. + let zoneInfo = zone.zoneInfoFromUtc(time) + result = initDateTime(zoneInfo, zone) + +proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Convert ``dt`` into a ``DateTime`` using ``zone`` as the timezone. + dt.toTime.inZone(zone) + +proc `$`*(zone: Timezone): string = + ## Returns the name of the timezone. + zone.name + +proc `==`*(zone1, zone2: Timezone): bool = + ## Two ``Timezone``'s are considered equal if their name is equal. + zone1.name == zone2.name + +proc toAdjTime(dt: DateTime): Time = + let epochDay = toEpochday(dt.monthday, dt.month, dt.year) + result = Time(epochDay * secondsInDay) + result.inc dt.hour * secondsInHour + result.inc dt.minute * secondsInMin + result.inc dt.second + +when defined(JS): + type JsDate = object + proc newDate(year, month, date, hours, minutes, seconds, milliseconds: int): JsDate {.tags: [], raises: [], importc: "new Date".} + proc newDate(): JsDate {.importc: "new Date".} + proc newDate(value: float): JsDate {.importc: "new Date".} + proc getTimezoneOffset(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getTime(js: JsDate): int {.tags: [], raises: [], noSideEffect, benign, importcpp.} + proc getDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc setFullYear(js: JsDate, year: int): void {.tags: [], raises: [], benign, importcpp.} + + proc localZoneInfoFromUtc(time: Time): ZonedTime = + let jsDate = newDate(time.float * 1000) + let offset = jsDate.getTimezoneOffset() * secondsInMin + result.adjTime = Time(time.int64 - offset) + result.utcOffset = offset + result.isDst = false + + proc localZoneInfoFromTz(adjTime: Time): ZonedTime = + let utcDate = newDate(adjTime.float * 1000) + let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), + utcDate.getUTCHours(), utcDate.getUTCMinutes(), utcDate.getUTCSeconds(), 0) + + # This is as dumb as it looks - JS doesn't support years in the range 0-99 in the constructor + # because they are assumed to be 19xx... + # Because JS doesn't support timezone history, it doesn't really matter in practice. + if utcDate.getUTCFullYear() in 0 .. 99: + localDate.setFullYear(utcDate.getUTCFullYear()) + + result.adjTime = adjTime + result.utcOffset = localDate.getTimezoneOffset() * secondsInMin + result.isDst = false + +else: + when defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(macosx): + type + StructTm {.importc: "struct tm".} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + gmtoff {.importc: "tm_gmtoff".}: clong else: - result = a - b == 0 + type + StructTm {.importc: "struct tm".} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + when defined(linux) and defined(amd64): + gmtoff {.importc: "tm_gmtoff".}: clong + zone {.importc: "tm_zone".}: cstring + type + StructTmPtr = ptr StructTm -proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.} - ## returns the offset of the local (non-DST) timezone in seconds west of UTC. + proc localtime(timer: ptr CTime): StructTmPtr {. importc: "localtime", header: "", tags: [].} -proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} - ## get the milliseconds from the start of the program. **Deprecated since - ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. + proc toAdjTime(tm: StructTm): Time = + let epochDay = toEpochday(tm.monthday, (tm.month + 1).Month, tm.year.int + 1900) + result = Time(epochDay * secondsInDay) + result.inc tm.hour * secondsInHour + result.inc tm.minute * 60 + result.inc tm.second + + proc getStructTm(time: Time | int64): StructTm = + let timei64 = time.int64 + var a = + if timei64 < low(CTime): + CTime(low(CTime)) + elif timei64 > high(CTime): + CTime(high(CTime)) + else: + CTime(timei64) + result = localtime(addr(a))[] + + proc localZoneInfoFromUtc(time: Time): ZonedTime = + let tm = getStructTm(time) + let adjTime = tm.toAdjTime + result.adjTime = adjTime + result.utcOffset = (time.toUnix - adjTime.toUnix).int + result.isDst = tm.isdst > 0 + + proc localZoneInfoFromTz(adjTime: Time): ZonedTime = + var adjTimei64 = adjTime.int64 + let past = adjTimei64 - secondsInDay + var tm = getStructTm(past) + let pastOffset = past - tm.toAdjTime.int64 + + let future = adjTimei64 + secondsInDay + tm = getStructTm(future) + let futureOffset = future - tm.toAdjTime.int64 + + var utcOffset: int + if pastOffset == futureOffset: + utcOffset = pastOffset.int + else: + if pastOffset > futureOffset: + adjTimei64 -= secondsInHour + + adjTimei64 += pastOffset + utcOffset = (adjTimei64 - getStructTm(adjTimei64).toAdjTime.int64).int + + # This extra roundtrip is needed to normalize any impossible datetimes + # as a result of offset changes (normally due to dst) + let utcTime = adjTime.int64 + utcOffset + tm = getStructTm(utcTime) + result.adjTime = tm.toAdjTime + result.utcOffset = (utcTime - result.adjTime.int64).int + result.isDst = tm.isdst > 0 + +proc utcZoneInfoFromUtc(time: Time): ZonedTime = + result.adjTime = time + result.utcOffset = 0 + result.isDst = false + +proc utcZoneInfoFromTz(adjTime: Time): ZonedTime = + utcZoneInfoFromUtc(adjTime) # adjTime == time since we are in UTC + +proc utc*(): TimeZone = + ## Get the ``Timezone`` implementation for the UTC timezone. + ## + ## .. code-block:: nim + ## doAssert now().utc.timezone == utc() + ## doAssert utc().name == "Etc/UTC" + Timezone(zoneInfoFromUtc: utcZoneInfoFromUtc, zoneInfoFromTz: utcZoneInfoFromTz, name: "Etc/UTC") + +proc local*(): TimeZone = + ## Get the ``Timezone`` implementation for the local timezone. + ## + ## .. code-block:: nim + ## doAssert now().timezone == local() + ## doAssert local().name == "LOCAL" + Timezone(zoneInfoFromUtc: localZoneInfoFromUtc, zoneInfoFromTz: localZoneInfoFromTz, name: "LOCAL") + +proc utc*(dt: DateTime): DateTime = + ## Shorthand for ``dt.inZone(utc())``. + dt.inZone(utc()) + +proc local*(dt: DateTime): DateTime = + ## Shorthand for ``dt.inZone(local())``. + dt.inZone(local()) + +proc utc*(t: Time): DateTime = + ## Shorthand for ``t.inZone(utc())``. + t.inZone(utc()) + +proc local*(t: Time): DateTime = + ## Shorthand for ``t.inZone(local())``. + t.inZone(local()) + +proc getTime*(): Time {.tags: [TimeEffect], benign.} + ## Gets the current time as a ``Time`` with second resolution. Use epochTime for higher + ## resolution. + +proc now*(): DateTime {.tags: [TimeEffect], benign.} = + ## Get the current time as a ``DateTime`` in the local timezone. + ## + ## Shorthand for ``getTime().local``. + getTime().local proc initInterval*(milliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = - ## creates a new ``TimeInterval``. + ## Creates a new ``TimeInterval``. ## ## You can also use the convenience procedures called ``milliseconds``, ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. @@ -269,46 +523,33 @@ proc initInterval*(milliseconds, seconds, minutes, hours, days, months, ## .. code-block:: nim ## ## let day = initInterval(hours=24) - ## let tomorrow = getTime() + day - ## echo(tomorrow) - var carryO = 0 - result.milliseconds = `mod`(milliseconds, 1000) - carryO = `div`(milliseconds, 1000) - result.seconds = `mod`(carryO + seconds, 60) - carryO = `div`(carryO + seconds, 60) - result.minutes = `mod`(carryO + minutes, 60) - carryO = `div`(carryO + minutes, 60) - result.hours = `mod`(carryO + hours, 24) - carryO = `div`(carryO + hours, 24) - result.days = carryO + days - - result.months = `mod`(months, 12) - carryO = `div`(months, 12) - result.years = carryO + years + ## let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + ## doAssert $(dt + day) == "2000-01-02T12-00-00+00:00" + result.milliseconds = milliseconds + result.seconds = seconds + result.minutes = minutes + result.hours = hours + result.days = days + result.months = months + result.years = years proc `+`*(ti1, ti2: TimeInterval): TimeInterval = ## Adds two ``TimeInterval`` objects together. - var carryO = 0 - result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) - carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) - result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60) - carryO = `div`(carryO + ti1.seconds + ti2.seconds, 60) - result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60) - carryO = `div`(carryO + ti1.minutes + ti2.minutes, 60) - result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24) - carryO = `div`(carryO + ti1.hours + ti2.hours, 24) - result.days = carryO + ti1.days + ti2.days - - result.months = `mod`(ti1.months + ti2.months, 12) - carryO = `div`(ti1.months + ti2.months, 12) - result.years = carryO + ti1.years + ti2.years + result.milliseconds = ti1.milliseconds + ti2.milliseconds + result.seconds = ti1.seconds + ti2.seconds + result.minutes = ti1.minutes + ti2.minutes + result.hours = ti1.hours + ti2.hours + result.days = ti1.days + ti2.days + result.months = ti1.months + ti2.months + result.years = ti1.years + ti2.years proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval + ## ## .. code-block:: nim ## ## let day = -initInterval(hours=24) - ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: 0, days: -1, months: 0, years: 0) + ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: -24, days: 0, months: 0, years: 0) result = TimeInterval( milliseconds: -ti.milliseconds, seconds: -ti.seconds, @@ -325,123 +566,100 @@ proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## Time components are compared one-by-one, see output: ## ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) + ## let a = fromUnix(1_000_000_000) + ## let b = fromUnix(1_500_000_000) ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: -2, months: -2, years: 16) + ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) result = ti1 + (-ti2) -proc isLeapYear*(year: int): bool = - ## returns true if ``year`` is a leap year +proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDiff, absDiff: int64] = + ## Evaluates how many seconds the interval is worth + ## in the context of ``dt``. + ## The result in split into an adjusted diff and an absolute diff. - if year mod 400 == 0: - return true - elif year mod 100 == 0: - return false - elif year mod 4 == 0: - return true - else: - return false - -proc getDaysInMonth*(month: Month, year: int): int = - ## Get the number of days in a ``month`` of a ``year`` - - # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month - case month - of mFeb: result = if isLeapYear(year): 29 else: 28 - of mApr, mJun, mSep, mNov: result = 30 - else: result = 31 - -proc getDaysInYear*(year: int): int = - ## Get the number of days in a ``year`` - result = 365 + (if isLeapYear(year): 1 else: 0) - -proc toSeconds(a: TimeInfo, interval: TimeInterval): float = - ## Calculates how many seconds the interval is worth by adding up - ## all the fields - - var anew = a + var anew = dt var newinterv = interval - result = 0 newinterv.months += interval.years * 12 var curMonth = anew.month - if newinterv.months < 0: # subtracting + # Subtracting + if newinterv.months < 0: for mth in countDown(-1 * newinterv.months, 1): - result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) if curMonth == mJan: curMonth = mDec anew.year.dec() else: curMonth.dec() - else: # adding + result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay + # Adding + else: for mth in 1 .. newinterv.months: - result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + result.adjDiff += getDaysInMonth(curMonth, anew.year) * secondsInDay if curMonth == mDec: curMonth = mJan anew.year.inc() else: curMonth.inc() - result += float(newinterv.days * 24 * 60 * 60) - result += float(newinterv.hours * 60 * 60) - result += float(newinterv.minutes * 60) - result += float(newinterv.seconds) - result += newinterv.milliseconds / 1000 + result.adjDiff += newinterv.days * secondsInDay + result.absDiff += newinterv.hours * secondsInHour + result.absDiff += newinterv.minutes * secondsInMin + result.absDiff += newinterv.seconds + result.absDiff += newinterv.milliseconds div 1000 -proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## adds ``interval`` time from TimeInfo ``a``. +proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Adds ``interval`` to ``dt``. Components from ``interval`` are added + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + ## + ## Note that when adding months, monthday overflow is allowed. This means that if the resulting + ## month doesn't have enough days it, the month will be incremented and the monthday will be + ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, + ## which will overflow and result in `1 December`. ## - ## **Note:** This has been only briefly tested and it may not be - ## very accurate. - let t = toSeconds(toTime(a)) - let secs = toSeconds(a, interval) - if a.timezone == 0: - result = getGMTime(fromSeconds(t + secs)) + ## .. code-block:: nim + ## let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + ## doAssert $(dt + 1.months) == "2017-04-30T00:00:00+00:00" + ## # This is correct and happens due to monthday overflow. + ## doAssert $(dt - 1.months) == "2017-03-02T00:00:00+00:00" + let (adjDiff, absDiff) = evaluateInterval(dt, interval) + + if adjDiff.int64 != 0: + let zInfo = dt.timezone.zoneInfoFromTz(Time(dt.toAdjTime.int64 + adjDiff)) + + if absDiff != 0: + let time = Time(zInfo.adjTime.int64 + zInfo.utcOffset + absDiff) + result = initDateTime(dt.timezone.zoneInfoFromUtc(time), dt.timezone) + else: + result = initDateTime(zInfo, dt.timezone) else: - result = getLocalTime(fromSeconds(t + secs)) + result = initDateTime(dt.timezone.zoneInfoFromUtc(Time(dt.toTime.int64 + absDiff)), dt.timezone) -proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## subtracts ``interval`` time from TimeInfo ``a``. - ## - ## **Note:** This has been only briefly tested, it is inaccurate especially - ## when you subtract so much that you reach the Julian calendar. - let - t = toSeconds(toTime(a)) - secs = toSeconds(a, -interval) - if a.timezone == 0: - result = getGMTime(fromSeconds(t + secs)) - else: - result = getLocalTime(fromSeconds(t + secs)) - -proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds - -proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} = - ## An alias for a misspelled field in ``TimeInterval``. - ## - ## **Warning:** This should not be used! It will be removed in the next - ## version. - t.milliseconds = milliseconds +proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + dt + (-interval) proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current date as a string of the format ``YYYY-MM-DD``. - var ti = getLocalTime(getTime()) - result = $ti.year & '-' & intToStr(ord(ti.month)+1, 2) & + ## Gets the current date as a string of the format ``YYYY-MM-DD``. + var ti = now() + result = $ti.year & '-' & intToStr(ord(ti.month), 2) & '-' & intToStr(ti.monthday, 2) proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current clock time as a string of the format ``HH:MM:SS``. - var ti = getLocalTime(getTime()) + ## Gets the current clock time as a string of the format ``HH:MM:SS``. + var ti = now() result = intToStr(ti.hour, 2) & ':' & intToStr(ti.minute, 2) & ':' & intToStr(ti.second, 2) proc `$`*(day: WeekDay): string = - ## stingify operator for ``WeekDay``. + ## Stringify operator for ``WeekDay``. const lookup: array[WeekDay, string] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] return lookup[day] proc `$`*(m: Month): string = - ## stingify operator for ``Month``. + ## Stringify operator for ``Month``. const lookup: array[Month, string] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"] @@ -450,74 +668,68 @@ proc `$`*(m: Month): string = proc milliseconds*(ms: int): TimeInterval {.inline.} = ## TimeInterval of `ms` milliseconds ## - ## Note: not all time functions have millisecond resolution - initInterval(`mod`(ms,1000), `div`(ms,1000)) + ## Note: not all time procedures have millisecond resolution + initInterval(milliseconds = ms) proc seconds*(s: int): TimeInterval {.inline.} = ## TimeInterval of `s` seconds ## ## ``echo getTime() + 5.second`` - initInterval(0,`mod`(s,60), `div`(s,60)) + initInterval(seconds = s) proc minutes*(m: int): TimeInterval {.inline.} = ## TimeInterval of `m` minutes ## ## ``echo getTime() + 5.minutes`` - initInterval(0,0,`mod`(m,60), `div`(m,60)) + initInterval(minutes = m) proc hours*(h: int): TimeInterval {.inline.} = ## TimeInterval of `h` hours ## ## ``echo getTime() + 2.hours`` - initInterval(0,0,0,`mod`(h,24),`div`(h,24)) + initInterval(hours = h) proc days*(d: int): TimeInterval {.inline.} = ## TimeInterval of `d` days ## ## ``echo getTime() + 2.days`` - initInterval(0,0,0,0,d) + initInterval(days = d) proc months*(m: int): TimeInterval {.inline.} = ## TimeInterval of `m` months ## ## ``echo getTime() + 2.months`` - initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12)) + initInterval(months = m) proc years*(y: int): TimeInterval {.inline.} = ## TimeInterval of `y` years ## ## ``echo getTime() + 2.years`` - initInterval(0,0,0,0,0,0,y) + initInterval(years = y) -proc `+=`*(t: var Time, ti: TimeInterval) = - ## modifies `t` by adding the interval `ti` - t = toTime(getLocalTime(t) + ti) +proc `+=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by adding `interval`. + time = toTime(time.local + interval) -proc `+`*(t: Time, ti: TimeInterval): Time = - ## adds the interval `ti` to Time `t` - ## by converting to localTime, adding the interval, and converting back +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time` + ## by converting to a ``DateTime`` in the local timezone, + ## adding the interval, and converting back to ``Time``. ## ## ``echo getTime() + 1.day`` - result = toTime(getLocalTime(t) + ti) + result = toTime(time.local + interval) -proc `-=`*(t: var Time, ti: TimeInterval) = - ## modifies `t` by subtracting the interval `ti` - t = toTime(getLocalTime(t) - ti) +proc `-=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by subtracting `interval`. + time = toTime(time.local - interval) -proc `-`*(t: Time, ti: TimeInterval): Time = - ## subtracts the interval `ti` from Time `t` +proc `-`*(time: Time, interval: TimeInterval): Time = + ## Subtracts `interval` from Time `time`. ## ## ``echo getTime() - 1.day`` - result = toTime(getLocalTime(t) - ti) + result = toTime(time.local - interval) -const - secondsInMin = 60 - secondsInHour = 60*60 - secondsInDay = 60*60*24 - minutesInHour = 60 - epochStartYear = 1970 - -proc formatToken(info: TimeInfo, token: string, buf: var string) = +proc formatToken(dt: DateTime, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. ## ## Pass the found token in the user input string, and the buffer where the @@ -525,96 +737,96 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = ## formatting tokens require modifying the previous characters. case token of "d": - buf.add($info.monthday) + buf.add($dt.monthday) of "dd": - if info.monthday < 10: + if dt.monthday < 10: buf.add("0") - buf.add($info.monthday) + buf.add($dt.monthday) of "ddd": - buf.add(($info.weekday)[0 .. 2]) + buf.add(($dt.weekday)[0 .. 2]) of "dddd": - buf.add($info.weekday) + buf.add($dt.weekday) of "h": - buf.add($(if info.hour > 12: info.hour - 12 else: info.hour)) + buf.add($(if dt.hour > 12: dt.hour - 12 else: dt.hour)) of "hh": - let amerHour = if info.hour > 12: info.hour - 12 else: info.hour + let amerHour = if dt.hour > 12: dt.hour - 12 else: dt.hour if amerHour < 10: buf.add('0') buf.add($amerHour) of "H": - buf.add($info.hour) + buf.add($dt.hour) of "HH": - if info.hour < 10: + if dt.hour < 10: buf.add('0') - buf.add($info.hour) + buf.add($dt.hour) of "m": - buf.add($info.minute) + buf.add($dt.minute) of "mm": - if info.minute < 10: + if dt.minute < 10: buf.add('0') - buf.add($info.minute) + buf.add($dt.minute) of "M": - buf.add($(int(info.month)+1)) + buf.add($ord(dt.month)) of "MM": - if info.month < mOct: + if dt.month < mOct: buf.add('0') - buf.add($(int(info.month)+1)) + buf.add($ord(dt.month)) of "MMM": - buf.add(($info.month)[0..2]) + buf.add(($dt.month)[0..2]) of "MMMM": - buf.add($info.month) + buf.add($dt.month) of "s": - buf.add($info.second) + buf.add($dt.second) of "ss": - if info.second < 10: + if dt.second < 10: buf.add('0') - buf.add($info.second) + buf.add($dt.second) of "t": - if info.hour >= 12: + if dt.hour >= 12: buf.add('P') else: buf.add('A') of "tt": - if info.hour >= 12: + if dt.hour >= 12: buf.add("PM") else: buf.add("AM") of "y": - var fr = ($info.year).len()-1 + var fr = ($dt.year).len()-1 if fr < 0: fr = 0 - buf.add(($info.year)[fr .. ($info.year).len()-1]) + buf.add(($dt.year)[fr .. ($dt.year).len()-1]) of "yy": - var fr = ($info.year).len()-2 + var fr = ($dt.year).len()-2 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 2: fyear = repeat('0', 2-fyear.len()) & fyear buf.add(fyear) of "yyy": - var fr = ($info.year).len()-3 + var fr = ($dt.year).len()-3 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 3: fyear = repeat('0', 3-fyear.len()) & fyear buf.add(fyear) of "yyyy": - var fr = ($info.year).len()-4 + var fr = ($dt.year).len()-4 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 4: fyear = repeat('0', 4-fyear.len()) & fyear buf.add(fyear) of "yyyyy": - var fr = ($info.year).len()-5 + var fr = ($dt.year).len()-5 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear buf.add(fyear) of "z": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour if nonDstTz <= 0: buf.add('+') else: buf.add('-') buf.add($hours) of "zz": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour if nonDstTz <= 0: buf.add('+') else: buf.add('-') @@ -622,7 +834,7 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = buf.add($hours) of "zzz": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour if nonDstTz <= 0: buf.add('+') @@ -638,8 +850,8 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = raise newException(ValueError, "Invalid format string: " & token) -proc format*(info: TimeInfo, f: string): string = - ## This function formats `info` as specified by `f`. The following format +proc format*(dt: DateTime, f: string): string {.tags: [].}= + ## This procedure formats `dt` as specified by `f`. The following format ## specifiers are available: ## ## ========== ================================================================================= ================================================ @@ -683,7 +895,7 @@ proc format*(info: TimeInfo, f: string): string = while true: case f[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - formatToken(info, currentF, result) + formatToken(dt, currentF, result) currentF = "" if f[i] == '\0': break @@ -700,187 +912,187 @@ proc format*(info: TimeInfo, f: string): string = if currentF.len < 1 or currentF[high(currentF)] == f[i]: currentF.add(f[i]) else: - formatToken(info, currentF, result) + formatToken(dt, currentF, result) dec(i) # Move position back to re-process the character separately. currentF = "" inc(i) -proc `$`*(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} = - ## converts a `TimeInfo` object to a string representation. +proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = + ## Converts a `DateTime` object to a string representation. ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. try: - result = format(timeInfo, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this + result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this except ValueError: assert false # cannot happen because format string is valid -proc `$`*(time: Time): string {.tags: [TimeEffect], raises: [], benign.} = +proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## converts a `Time` value to a string representation. It will use the local ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. - $getLocalTime(time) + $time.local {.pop.} -proc parseToken(info: var TimeInfo; token, value: string; j: var int) = +proc parseToken(dt: var DateTime; token, value: string; j: var int) = ## Helper of the parse proc to parse individual tokens. var sv: int case token of "d": var pd = parseInt(value[j..j+1], sv) - info.monthday = sv + dt.monthday = sv j += pd of "dd": - info.monthday = value[j..j+1].parseInt() + dt.monthday = value[j..j+1].parseInt() j += 2 of "ddd": case value[j..j+2].toLowerAscii() - of "sun": info.weekday = dSun - of "mon": info.weekday = dMon - of "tue": info.weekday = dTue - of "wed": info.weekday = dWed - of "thu": info.weekday = dThu - of "fri": info.weekday = dFri - of "sat": info.weekday = dSat + of "sun": dt.weekday = dSun + of "mon": dt.weekday = dMon + of "tue": dt.weekday = dTue + of "wed": dt.weekday = dWed + of "thu": dt.weekday = dThu + of "fri": dt.weekday = dFri + of "sat": dt.weekday = dSat else: raise newException(ValueError, "Couldn't parse day of week (ddd), got: " & value[j..j+2]) j += 3 of "dddd": if value.len >= j+6 and value[j..j+5].cmpIgnoreCase("sunday") == 0: - info.weekday = dSun + dt.weekday = dSun j += 6 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("monday") == 0: - info.weekday = dMon + dt.weekday = dMon j += 6 elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("tuesday") == 0: - info.weekday = dTue + dt.weekday = dTue j += 7 elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("wednesday") == 0: - info.weekday = dWed + dt.weekday = dWed j += 9 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("thursday") == 0: - info.weekday = dThu + dt.weekday = dThu j += 8 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("friday") == 0: - info.weekday = dFri + dt.weekday = dFri j += 6 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("saturday") == 0: - info.weekday = dSat + dt.weekday = dSat j += 8 else: raise newException(ValueError, "Couldn't parse day of week (dddd), got: " & value) of "h", "H": var pd = parseInt(value[j..j+1], sv) - info.hour = sv + dt.hour = sv j += pd of "hh", "HH": - info.hour = value[j..j+1].parseInt() + dt.hour = value[j..j+1].parseInt() j += 2 of "m": var pd = parseInt(value[j..j+1], sv) - info.minute = sv + dt.minute = sv j += pd of "mm": - info.minute = value[j..j+1].parseInt() + dt.minute = value[j..j+1].parseInt() j += 2 of "M": var pd = parseInt(value[j..j+1], sv) - info.month = Month(sv-1) + dt.month = sv.Month j += pd of "MM": var month = value[j..j+1].parseInt() j += 2 - info.month = Month(month-1) + dt.month = month.Month of "MMM": case value[j..j+2].toLowerAscii(): - of "jan": info.month = mJan - of "feb": info.month = mFeb - of "mar": info.month = mMar - of "apr": info.month = mApr - of "may": info.month = mMay - of "jun": info.month = mJun - of "jul": info.month = mJul - of "aug": info.month = mAug - of "sep": info.month = mSep - of "oct": info.month = mOct - of "nov": info.month = mNov - of "dec": info.month = mDec + of "jan": dt.month = mJan + of "feb": dt.month = mFeb + of "mar": dt.month = mMar + of "apr": dt.month = mApr + of "may": dt.month = mMay + of "jun": dt.month = mJun + of "jul": dt.month = mJul + of "aug": dt.month = mAug + of "sep": dt.month = mSep + of "oct": dt.month = mOct + of "nov": dt.month = mNov + of "dec": dt.month = mDec else: raise newException(ValueError, "Couldn't parse month (MMM), got: " & value) j += 3 of "MMMM": if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: - info.month = mJan + dt.month = mJan j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: - info.month = mFeb + dt.month = mFeb j += 8 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: - info.month = mMar + dt.month = mMar j += 5 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: - info.month = mApr + dt.month = mApr j += 5 elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: - info.month = mMay + dt.month = mMay j += 3 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: - info.month = mJun + dt.month = mJun j += 4 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: - info.month = mJul + dt.month = mJul j += 4 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: - info.month = mAug + dt.month = mAug j += 6 elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: - info.month = mSep + dt.month = mSep j += 9 elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: - info.month = mOct + dt.month = mOct j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: - info.month = mNov + dt.month = mNov j += 8 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: - info.month = mDec + dt.month = mDec j += 8 else: raise newException(ValueError, "Couldn't parse month (MMMM), got: " & value) of "s": var pd = parseInt(value[j..j+1], sv) - info.second = sv + dt.second = sv j += pd of "ss": - info.second = value[j..j+1].parseInt() + dt.second = value[j..j+1].parseInt() j += 2 of "t": - if value[j] == 'P' and info.hour > 0 and info.hour < 12: - info.hour += 12 + if value[j] == 'P' and dt.hour > 0 and dt.hour < 12: + dt.hour += 12 j += 1 of "tt": - if value[j..j+1] == "PM" and info.hour > 0 and info.hour < 12: - info.hour += 12 + if value[j..j+1] == "PM" and dt.hour > 0 and dt.hour < 12: + dt.hour += 12 j += 2 of "yy": # Assumes current century var year = value[j..j+1].parseInt() - var thisCen = getLocalTime(getTime()).year div 100 - info.year = thisCen*100 + year + var thisCen = now().year div 100 + dt.year = thisCen*100 + year j += 2 of "yyyy": - info.year = value[j..j+3].parseInt() + dt.year = value[j..j+3].parseInt() j += 4 of "z": - info.isDST = false + dt.isDst = false if value[j] == '+': - info.timezone = 0 - parseInt($value[j+1]) * secondsInHour + dt.utcOffset = 0 - parseInt($value[j+1]) * secondsInHour elif value[j] == '-': - info.timezone = parseInt($value[j+1]) * secondsInHour + dt.utcOffset = parseInt($value[j+1]) * secondsInHour elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: @@ -888,13 +1100,13 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (z), got: " & value[j]) j += 2 of "zz": - info.isDST = false + dt.isDst = false if value[j] == '+': - info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = 0 - value[j+1..j+2].parseInt() * secondsInHour elif value[j] == '-': - info.timezone = value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = value[j+1..j+2].parseInt() * secondsInHour elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: @@ -902,31 +1114,33 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (zz), got: " & value[j]) j += 3 of "zzz": - info.isDST = false + dt.isDst = false var factor = 0 if value[j] == '+': factor = -1 elif value[j] == '-': factor = 1 elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: raise newException(ValueError, "Couldn't parse timezone offset (zzz), got: " & value[j]) - info.timezone = factor * value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = factor * value[j+1..j+2].parseInt() * secondsInHour j += 4 - info.timezone += factor * value[j..j+1].parseInt() * 60 + dt.utcOffset += factor * value[j..j+1].parseInt() * 60 j += 2 else: # Ignore the token and move forward in the value string by the same length j += token.len -proc parse*(value, layout: string): TimeInfo = - ## This function parses a date/time string using the standard format - ## identifiers as listed below. The function defaults information not provided - ## in the format string from the running program (timezone, month, year, etc). - ## Daylight saving time is only set if no timezone is given and the given date - ## lies within the DST period of the current locale. +proc parse*(value, layout: string, zone: Timezone = local()): DateTime = + ## This procedure parses a date/time string using the standard format + ## identifiers as listed below. The procedure defaults information not provided + ## in the format string from the running program (month, year, etc). + ## + ## The return value will always be in the `zone` timezone. If no UTC offset was + ## parsed, then the input will be assumed to be specified in the `zone` timezone + ## already, so no timezone conversion will be done in that case. ## ## ========== ================================================================================= ================================================ ## Specifier Description Example @@ -965,17 +1179,17 @@ proc parse*(value, layout: string): TimeInfo = var j = 0 # pointer for value string var token = "" # Assumes current day of month, month and year, but time is reset to 00:00:00. Weekday will be reset after parsing. - var info = getLocalTime(getTime()) - info.hour = 0 - info.minute = 0 - info.second = 0 - info.isDST = true # using this is flag for checking whether a timezone has \ + var dt = now() + dt.hour = 0 + dt.minute = 0 + dt.second = 0 + dt.isDst = true # using this is flag for checking whether a timezone has \ # been read (because DST is always false when a tz is parsed) while true: case layout[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': if token.len > 0: - parseToken(info, token, value, j) + parseToken(dt, token, value, j) # Reset token token = "" # Break if at end of line @@ -997,26 +1211,15 @@ proc parse*(value, layout: string): TimeInfo = token.add(layout[i]) inc(i) else: - parseToken(info, token, value, j) + parseToken(dt, token, value, j) token = "" - if info.isDST: - # means that no timezone has been parsed. In this case, we need to check - # whether the date is within DST of the local time. - let tmp = getLocalTime(toTime(info)) - # correctly set isDST so that the following step works on the correct time - info.isDST = tmp.isDST - - # Correct weekday and yearday; transform timestamp to local time. - # There currently is no way of returning this with the original (parsed) - # timezone while also setting weekday and yearday (we are depending on stdlib - # to provide this calculation). - return getLocalTime(toTime(info)) - -# Leap year calculations are adapted from: -# http://www.codeproject.com/Articles/7358/Ultra-fast-Algorithms-for-Working-with-Leap-Years -# The dayOfTheWeek procs are adapated from: -# http://stason.org/TULARC/society/calendars/2-5-What-day-of-the-week-was-2-August-1953.html + if dt.isDst: + # No timezone parsed - assume timezone is `zone` + result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) + else: + # Otherwise convert to `zone` + result = dt.toTime.inZone(zone) proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. @@ -1042,35 +1245,188 @@ proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = result.years = days div 365 result.days = days mod 365 -proc getDayOfWeek*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. - # so we must correct for the WeekDay type. - if d == 0: return dSun - result = (d-1).WeekDay +proc toTimeInterval*(time: Time): TimeInterval = + ## Converts a Time to a TimeInterval. + ## + ## To be used when diffing times. + ## + ## .. code-block:: nim + ## let a = fromSeconds(1_000_000_000) + ## let b = fromSeconds(1_500_000_000) + ## echo a, " ", b # real dates + ## echo a.toTimeInterval # meaningless value, don't use it by itself + ## echo b.toTimeInterval - a.toTimeInterval + ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) + # Milliseconds not available from Time + var dt = time.local + initInterval(0, dt.second, dt.minute, dt.hour, dt.monthday, dt.month.ord - 1, dt.year) -proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, - ## according to the Julian calendar. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 - result = d.WeekDay +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + assertValidDate monthday, month, year + doAssert monthday <= getDaysInMonth(month, year), "Invalid date: " & $month & " " & $monthday & ", " & $year + let dt = DateTime( + monthday: monthday, + year: year, + month: month, + hour: hour, + minute: minute, + second: second + ) + result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) -proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = - ## Converts a Time to TimeInfo. +when not defined(JS): + proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} + ## gets time after the UNIX epoch (1970) in seconds. It is a float + ## because sub-second resolution is likely to be supported (depending + ## on the hardware/OS). + + proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} + ## gets time spent that the CPU spent to run the current process in + ## seconds. This may be more useful for benchmarking than ``epochTime``. + ## However, it may measure the real time instead (depending on the OS). + ## The value of the result has no meaning. + ## To generate useful timing values, take the difference between + ## the results of two ``cpuTime`` calls: + ## + ## .. code-block:: nim + ## var t0 = cpuTime() + ## doWork() + ## echo "CPU time [s] ", cpuTime() - t0 + +when defined(JS): + proc getTime(): Time = + (newDate().getTime() div 1000).Time + + proc epochTime*(): float {.tags: [TimeEffect].} = + newDate().getTime() / 1000 + +else: + type + Clock {.importc: "clock_t".} = distinct int + + proc timec(timer: ptr CTime): CTime {. + importc: "time", header: "", tags: [].} + + proc getClock(): Clock {.importc: "clock", header: "", tags: [TimeEffect].} + + var + clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int + + proc getTime(): Time = + timec(nil).Time + + const + epochDiff = 116444736000000000'i64 + rateDiff = 10000000'i64 # 100 nsecs + + proc unixTimeToWinTime*(time: CTime): int64 = + ## converts a UNIX `Time` (``time_t``) to a Windows file time + result = int64(time) * rateDiff + epochDiff + + proc winTimeToUnixTime*(time: int64): CTime = + ## converts a Windows time to a UNIX `Time` (``time_t``) + result = CTime((time - epochDiff) div rateDiff) + + when not defined(useNimRtl): + proc epochTime(): float = + when defined(posix): + var a: Timeval + posix_gettimeofday(a) + result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 + elif defined(windows): + var f: winlean.FILETIME + getSystemTimeAsFileTime(f) + var i64 = rdFileTime(f) - epochDiff + var secs = i64 div rateDiff + var subsecs = i64 mod rateDiff + result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 + else: + {.error: "unknown OS".} + + proc cpuTime(): float = + result = toFloat(int(getClock())) / toFloat(clocksPerSec) + +# Deprecated procs + +proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = + ## Takes a float which contains the number of seconds since the unix epoch and + ## returns a time object. + Time(since1970) + +proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = + ## Takes an int which contains the number of seconds since the unix epoch and + ## returns a time object. + Time(since1970) + +proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = + ## Returns the time in seconds since the unix epoch. + float(time) + +proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = + ## Converts the calendar time `time` to broken-time representation, + ## expressed relative to the user's specified time zone. + time.local + +proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = + ## Converts the calendar time `time` to broken-down time representation, + ## expressed in Coordinated Universal Time (UTC). + time.utc + +proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} = + ## Returns the offset of the local (non-DST) timezone in seconds west of UTC. + when defined(JS): + return newDate().getTimezoneOffset() * 60 + elif defined(freebsd) or defined(netbsd) or defined(openbsd): + var a = timec(nil) + let lt = localtime(addr(a)) + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + return -(lt.gmtoff) + else: + return timezone + +proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = + ## Converts a broken-down time structure to calendar time representation. ## ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``getLocalTime`` or ``getGMTime`` instead. + ## Use ``toTime`` instead. + dt.toTime + +when defined(JS): + var startMilsecs = getTime() + proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = + ## get the milliseconds from the start of the program. **Deprecated since + ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. + when defined(JS): + ## get the milliseconds from the start of the program + return int(getTime() - startMilsecs) +else: + proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = + when defined(macosx): + result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) + else: + result = int(getClock()) div (clocksPerSec div 1000) + +proc miliseconds*(t: TimeInterval): int {.deprecated.} = + t.milliseconds + +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = + ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + t.toTimeInterval() + +proc timeToTimeInfo*(t: Time): DateTime {.deprecated.} = + ## Converts a Time to DateTime. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``inZone`` instead. + const epochStartYear = 1970 + let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1095,307 +1451,22 @@ proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = m = mon # month is zero indexed enum md = days # NB: month is zero indexed but dayOfWeek expects 1 indexed. - wd = getDayOfWeek(days, mon.int + 1, y).Weekday + wd = getDayOfWeek(days, mon, y).Weekday h = daySeconds div secondsInHour + 1 mi = (daySeconds mod secondsInHour) div secondsInMin s = daySeconds mod secondsInMin - result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) + result = DateTime(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = - ## Converts a Time to a TimeInterval. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTimeInterval`` instead. - # Milliseconds not available from Time - var tInfo = t.getLocalTime() - initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) +proc getDayOfWeek*(day, month, year: int): WeekDay {.tags: [], raises: [], benign, deprecated.} = + getDayOfWeek(day, month.Month, year) -proc toTimeInterval*(t: Time): TimeInterval = - ## Converts a Time to a TimeInterval. - ## - ## To be used when diffing times. - ## - ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) - ## echo a, " ", b # real dates - ## echo a.toTimeInterval # meaningless value, don't use it by itself - ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: -2, months: -2, years: 16) - # Milliseconds not available from Time - var tInfo = t.getLocalTime() - initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - -when not defined(JS): - proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending - ## on the hardware/OS). - - proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time spent that the CPU spent to run the current process in - ## seconds. This may be more useful for benchmarking than ``epochTime``. - ## However, it may measure the real time instead (depending on the OS). - ## The value of the result has no meaning. - ## To generate useful timing values, take the difference between - ## the results of two ``cpuTime`` calls: - ## - ## .. code-block:: nim - ## var t0 = cpuTime() - ## doWork() - ## echo "CPU time [s] ", cpuTime() - t0 - -when not defined(JS): - # C wrapper: - when defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(macosx): - type - StructTM {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - gmtoff {.importc: "tm_gmtoff".}: clong - else: - type - StructTM {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - when defined(linux) and defined(amd64): - gmtoff {.importc: "tm_gmtoff".}: clong - zone {.importc: "tm_zone".}: cstring - type - TimeInfoPtr = ptr StructTM - Clock {.importc: "clock_t".} = distinct int - - when not defined(windows): - # This is not ANSI C, but common enough - proc timegm(t: StructTM): Time {. - importc: "timegm", header: "", tags: [].} - - proc localtime(timer: ptr Time): TimeInfoPtr {. - importc: "localtime", header: "", tags: [].} - proc gmtime(timer: ptr Time): TimeInfoPtr {. - importc: "gmtime", header: "", tags: [].} - proc timec(timer: ptr Time): Time {. - importc: "time", header: "", tags: [].} - proc mktime(t: StructTM): Time {. - importc: "mktime", header: "", tags: [].} - proc getClock(): Clock {.importc: "clock", header: "", tags: [TimeEffect].} - proc difftime(a, b: Time): float {.importc: "difftime", header: "", - tags: [].} - - var - clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - - # our own procs on top of that: - proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = - const - weekDays: array[0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - timezone: if local: getTimezone() else: 0 - ) - - - proc timeInfoToTM(t: TimeInfo): StructTM = - const - weekDays: array[WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] - result.second = t.second - result.minute = t.minute - result.hour = t.hour - result.monthday = t.monthday - result.month = cint(t.month) - result.year = cint(t.year - 1900) - result.weekday = weekDays[t.weekday] - result.yearday = t.yearday - result.isdst = if t.isDST: 1 else: 0 - - when not defined(useNimRtl): - proc `-` (a, b: Time): int64 = - return toBiggestInt(difftime(a, b)) - - proc getStartMilsecs(): int = - #echo "clocks per sec: ", clocksPerSec, "clock: ", int(getClock()) - #return getClock() div (clocksPerSec div 1000) - when defined(macosx): - result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) - else: - result = int(getClock()) div (clocksPerSec div 1000) - when false: - var a: Timeval - posix_gettimeofday(a) - result = a.tv_sec * 1000'i64 + a.tv_usec div 1000'i64 - #echo "result: ", result - - proc getTime(): Time = return timec(nil) - proc getLocalTime(t: Time): TimeInfo = - var a = t - let lt = localtime(addr(a)) - assert(not lt.isNil) - result = tmToTimeInfo(lt[], true) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc getGMTime(t: Time): TimeInfo = - var a = t - result = tmToTimeInfo(gmtime(addr(a))[], false) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc toTime(timeInfo: TimeInfo): Time = - var cTimeInfo = timeInfo # for C++ we have to make a copy - # because the header of mktime is broken in my version of libc - - result = mktime(timeInfoToTM(cTimeInfo)) - # mktime is defined to interpret the input as local time. As timeInfoToTM - # does ignore the timezone, we need to adjust this here. - result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) - - proc timeInfoToTime(timeInfo: TimeInfo): Time = toTime(timeInfo) - - const - epochDiff = 116444736000000000'i64 - rateDiff = 10000000'i64 # 100 nsecs - - proc unixTimeToWinTime*(t: Time): int64 = - ## converts a UNIX `Time` (``time_t``) to a Windows file time - result = int64(t) * rateDiff + epochDiff - - proc winTimeToUnixTime*(t: int64): Time = - ## converts a Windows time to a UNIX `Time` (``time_t``) - result = Time((t - epochDiff) div rateDiff) - - proc getTimezone(): int = - when defined(freebsd) or defined(netbsd) or defined(openbsd): - var a = timec(nil) - let lt = localtime(addr(a)) - # BSD stores in `gmtoff` offset east of UTC in seconds, - # but posix systems using west of UTC in seconds - return -(lt.gmtoff) - else: - return timezone - - proc fromSeconds(since1970: float): Time = Time(since1970) - - proc toSeconds(time: Time): float = float(time) - - when not defined(useNimRtl): - proc epochTime(): float = - when defined(posix): - var a: Timeval - posix_gettimeofday(a) - result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 - elif defined(windows): - var f: winlean.FILETIME - getSystemTimeAsFileTime(f) - var i64 = rdFileTime(f) - epochDiff - var secs = i64 div rateDiff - var subsecs = i64 mod rateDiff - result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - else: - {.error: "unknown OS".} - - proc cpuTime(): float = - result = toFloat(int(getClock())) / toFloat(clocksPerSec) - -elif defined(JS): - proc newDate(): Time {.importc: "new Date".} - proc internGetTime(): Time {.importc: "new Date", tags: [].} - - proc newDate(value: float): Time {.importc: "new Date".} - proc newDate(value: cstring): Time {.importc: "new Date".} - proc getTime(): Time = - # Warning: This is something different in JS. - return newDate() - - const - weekDays: array[0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - - proc getLocalTime(t: Time): TimeInfo = - result.second = t.getSeconds() - result.minute = t.getMinutes() - result.hour = t.getHours() - result.monthday = t.getDate() - result.month = Month(t.getMonth()) - result.year = t.getFullYear() - result.weekday = weekDays[t.getDay()] - result.timezone = getTimezone() - - result.yearday = result.monthday - 1 - for month in mJan.. 10 + + test "dst handling": + putEnv("TZ", "Europe/Stockholm") + # In case of an impossible time, the time is moved to after the impossible time period + check initDateTime(26, mMar, 2017, 02, 30, 00).format(f) == "2017-03-26 03:30 +02:00" + # In case of an ambiguous time, the earlier time is choosen + check initDateTime(29, mOct, 2017, 02, 00, 00).format(f) == "2017-10-29 02:00 +02:00" + # These are just dates on either side of the dst switch + check initDateTime(29, mOct, 2017, 01, 00, 00).format(f) == "2017-10-29 01:00 +02:00" + check initDateTime(29, mOct, 2017, 01, 00, 00).isDst + check initDateTime(29, mOct, 2017, 03, 01, 00).format(f) == "2017-10-29 03:01 +01:00" + check (not initDateTime(29, mOct, 2017, 03, 01, 00).isDst) + + check initDateTime(21, mOct, 2017, 01, 00, 00).format(f) == "2017-10-21 01:00 +02:00" + + test "issue #6520": + putEnv("TZ", "Europe/Stockholm") + var local = fromUnix(1469275200).local + var utc = fromUnix(1469275200).utc + + let claimedOffset = local.utcOffset + local.utcOffset = 0 + check claimedOffset == utc.toTime - local.toTime + + test "issue #5704": + putEnv("TZ", "Asia/Seoul") + let diff = parse("19700101-000000", "yyyyMMdd-hhmmss").toTime - parse("19000101-000000", "yyyyMMdd-hhmmss").toTime + check diff == 2208986872 + + test "issue #6465": + putEnv("TZ", "Europe/Stockholm") + let dt = parse("2017-03-25 12:00", "yyyy-MM-dd hh:mm") + check $(dt + 1.days) == "2017-03-26T12:00:00+02:00" + + test "datetime before epoch": + check $fromUnix(-2147483648).utc == "1901-12-13T20:45:52+00:00" + + test "adding/subtracting time across dst": + putenv("TZ", "Europe/Stockholm") + + let dt1 = initDateTime(26, mMar, 2017, 03, 00, 00) + check $(dt1 - 1.seconds) == "2017-03-26T01:59:59+01:00" + + var dt2 = initDateTime(29, mOct, 2017, 02, 59, 59) + check $(dt2 + 1.seconds) == "2017-10-29T02:00:00+01:00" + + putEnv("TZ", orig_tz) + + else: + # not on Linux or macosx: run one parseTest only + test "parseTest": + runTimezoneTests() + + test "isLeapYear": + check isLeapYear(2016) + check (not isLeapYear(2015)) + check isLeapYear(2000) + check (not isLeapYear(1900)) + + test "subtract months": + var dt = initDateTime(1, mFeb, 2017, 00, 00, 00, utc()) + check $(dt - 1.months) == "2017-01-01T00:00:00+00:00" + dt = initDateTime(15, mMar, 2017, 00, 00, 00, utc()) + check $(dt - 1.months) == "2017-02-15T00:00:00+00:00" + dt = initDateTime(31, mMar, 2017, 00, 00, 00, utc()) + # This happens due to monthday overflow. It's consistent with Phobos. + check $(dt - 1.months) == "2017-03-03T00:00:00+00:00" \ No newline at end of file From a9ba02e8c90942ac3bc38c4fbab152968a8147dc Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Tue, 19 Dec 2017 01:34:55 +0200 Subject: [PATCH 062/200] added asyncjs standard library module (#6841) --- changelog.md | 18 +++++--- doc/lib.rst | 2 + lib/js/asyncjs.nim | 110 ++++++++++++++++++++++++++++++++++++++++++++ tests/js/tasync.nim | 26 +++++++++++ web/website.ini | 2 +- 5 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 lib/js/asyncjs.nim create mode 100644 tests/js/tasync.nim diff --git a/changelog.md b/changelog.md index b216b0f17f..d519ecfcf6 100644 --- a/changelog.md +++ b/changelog.md @@ -39,36 +39,37 @@ what to return if the environment variable does not exist. - Bodies of ``for`` loops now get their own scope: -.. code-block:: nim +```nim # now compiles: for i in 0..4: let i = i + 1 echo i +``` - The parsing rules of ``if`` expressions were changed so that multiple statements are allowed in the branches. We found few code examples that now fail because of this change, but here is one: -.. code-block:: nim - +```nim t[ti] = if exp_negative: '-' else: '+'; inc(ti) +``` This now needs to be written as: -.. code-block:: nim - +```nim t[ti] = (if exp_negative: '-' else: '+'); inc(ti) +``` - To make Nim even more robust the system iterators ``..`` and ``countup`` now only accept a single generic type ``T``. This means the following code doesn't die with an "out of range" error anymore: -.. code-block:: nim - +```nim var b = 5.Natural var a = -5 for i in a..b: echo i +``` - ``formatFloat``/``formatBiggestFloat`` now support formatting floats with zero precision digits. The previous ``precision = 0`` behavior (default formatting) @@ -139,3 +140,6 @@ This now needs to be written as: - For string formatting / interpolation a new module called [strformat](https://nim-lang.org/docs/strformat.html) has been added to the stdlib. +- codegenDecl pragma now works for the JavaScript backend. It returns an empty string for + function return type placeholders. +- Asynchronous programming for the JavaScript backend using the `asyncjs` module. diff --git a/doc/lib.rst b/doc/lib.rst index 6eaf6c788a..2719472fef 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -433,6 +433,8 @@ Modules for JS backend * `jsffi `_ Types and macros for easier interaction with JavaScript. +* `asyncjs `_ + Types and macros for writing asynchronous procedures in JavaScript. Deprecated modules ------------------ diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim new file mode 100644 index 0000000000..bde3d787fb --- /dev/null +++ b/lib/js/asyncjs.nim @@ -0,0 +1,110 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim Authors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## This module implements types and macros for writing asynchronous code +## for the JS backend. It provides tools for interaction with JavaScript async API-s +## and libraries, writing async procedures in Nim and converting callback-based code +## to promises. +## +## A Nim procedure is asynchronous when it includes the ``{.async.}`` pragma. It +## should always have a ``Future[T]`` return type or not have a return type at all. +## A ``Future[void]`` return type is assumed by default. +## +## This is roughly equivalent to the ``async`` keyword in JavaScript code. +## +## .. code-block:: nim +## proc loadGame(name: string): Future[Game] {.async.} = +## # code +## +## should be equivalent to +## +## .. code-block:: javascript +## async function loadGame(name) { +## // code +## } +## +## A call to an asynchronous procedure usually needs ``await`` to wait for +## the completion of the ``Future``. +## +## .. code-block:: nim +## var game = await loadGame(name) +## +## Often, you might work with callback-based API-s. You can wrap them with +## asynchronous procedures using promises and ``newPromise``: +## +## .. code-block:: nim +## proc loadGame(name: string): Future[Game] = +## var promise = newPromise() do (resolve: proc(response: Game)): +## cbBasedLoadGame(name) do (game: Game): +## resolve(game) +## return promise +## +## Forward definitions work properly, you just don't need to add the ``{.async.}`` pragma: +## +## .. code-block:: nim +## proc loadGame(name: string): Future[Game] +## +## JavaScript compatibility +## ~~~~~~~~~~~~~~~~~~~~~~~~~ +## +## Nim currently generates `async/await` JavaScript code which is supported in modern +## EcmaScript and most modern versions of browsers, Node.js and Electron. +## If you need to use this module with older versions of JavaScript, you can +## use a tool that backports the resulting JavaScript code, as babel. + +import jsffi +import macros + +when not defined(js) and not defined(nimdoc) and not defined(nimsuggest): + {.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".} + +type + Future*[T] = ref object + future*: T + ## Wraps the return type of an asynchronous procedure. + + PromiseJs* {.importcpp: "Promise".} = ref object + ## A JavaScript Promise + +proc replaceReturn(node: var NimNode) = + var z = 0 + for s in node: + var son = node[z] + if son.kind == nnkReturnStmt: + node[z] = nnkReturnStmt.newTree(nnkCall.newTree(ident("jsResolve"), son[0])) + elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result": + node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(ident("jsResolve"), son[1])) + else: + replaceReturn(son) + inc z + +proc generateJsasync(arg: NimNode): NimNode = + assert arg.kind == nnkProcDef + result = arg + if arg.params[0].kind == nnkEmpty: + result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void")) + var code = result.body + replaceReturn(code) + result.body = nnkStmtList.newTree() + var q = quote: + proc await[T](f: Future[T]): T {.importcpp: "(await #)".} + proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} + result.body.add(q) + for child in code: + result.body.add(child) + result.pragma = quote: + {.codegenDecl: "async function $2($3)".} + +macro async*(arg: untyped): untyped = + ## Macro which converts normal procedures into + ## javascript-compatible async procedures + generateJsasync(arg) + +proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".} + ## A helper for wrapping callback-based functions + ## into promises and async procedures diff --git a/tests/js/tasync.nim b/tests/js/tasync.nim new file mode 100644 index 0000000000..a164827d20 --- /dev/null +++ b/tests/js/tasync.nim @@ -0,0 +1,26 @@ +discard """ + disabled: true + output: ''' +0 +x +''' +""" + +import asyncjs + +# demonstrate forward definition +# for js +proc y(e: int): Future[string] + +proc x(e: int) {.async.} = + var s = await y(e) + echo s + +proc y(e: int): Future[string] {.async.} = + echo 0 + return "x" + + + +discard x(2) + diff --git a/web/website.ini b/web/website.ini index 5560e67ea5..d8deb2d70e 100644 --- a/web/website.ini +++ b/web/website.ini @@ -67,7 +67,7 @@ srcdoc2: "pure/collections/heapqueue" srcdoc2: "pure/fenv;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" -srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile" +srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile;js/asyncjs" ; Note: everything under 'webdoc' doesn't get listed in the index, so wrappers ; should live here From 3de81af44d68b910c62afc99fefb3ecacab433ec Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 19 Dec 2017 12:04:42 +0200 Subject: [PATCH 063/200] Added a couple of procs for RSA verification (#6942) --- lib/wrappers/openssl.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 431ea59122..55b0bc3f8e 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -64,6 +64,8 @@ type des_key_schedule* = array[1..16, des_ks_struct] + pem_password_cb* = proc(buf: cstring, size, rwflag: cint, userdata: pointer): cint {.cdecl.} + {.deprecated: [PSSL: SslPtr, PSSL_CTX: SslCtx, PBIO: BIO].} const @@ -432,6 +434,12 @@ proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_clear_error".} proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_free_strings".} proc ErrRemoveState*(pid: cInt){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".} +proc PEM_read_bio_RSA_PUBKEY*(bp: BIO, x: ptr PRSA, pw: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLSSLName, importc.} + +proc RSA_verify*(kind: cint, origMsg: pointer, origMsgLen: cuint, signature: pointer, + signatureLen: cuint, rsa: PRSA): cint {.cdecl, dynlib: DLLSSLName, importc.} + when true: discard else: From 85ac3130aa80e784fcd84c5a38122c61f4a8860d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 19 Dec 2017 12:39:50 +0100 Subject: [PATCH 064/200] =?UTF-8?q?make=20asyncdispatch.poll=20completing?= =?UTF-8?q?=20all=20opterations=20that=20can=20be=20comple=E2=80=A6=20(#69?= =?UTF-8?q?11)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit introduce asyncdispatch.drain that completes all operations that can be completed immediately; implements #6523 --- lib/pure/asyncdispatch.nim | 43 +++++++++++++++++++++++++++--------- tests/async/tioselectors.nim | 6 ++--- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index a71d30ab9f..675e8fc5eb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -168,18 +168,20 @@ type timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]] callbacks*: Deque[proc ()] -proc processTimers(p: PDispatcherBase) {.inline.} = +proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} = #Process just part if timers at a step var count = p.timers.len let t = epochTime() while count > 0 and t >= p.timers[0].finishAt: p.timers.pop().fut.complete() dec count + didSomeWork = true -proc processPendingCallbacks(p: PDispatcherBase) = +proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) = while p.callbacks.len > 0: var cb = p.callbacks.popFirst() cb() + didSomeWork = true proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = # If dispatcher has active timers this proc returns the timeout @@ -284,14 +286,13 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. + proc runOnce(timeout = 500): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") + result = false if p.handles.len != 0: let at = p.adjustedTimeout(timeout) var llTimeout = @@ -304,6 +305,7 @@ when defined(windows) or defined(nimdoc): let res = getQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, addr lpCompletionKey, cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + result = true # http://stackoverflow.com/a/12277264/492186 # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html @@ -333,13 +335,14 @@ when defined(windows) or defined(nimdoc): else: if errCode.int32 == WAIT_TIMEOUT: # Timed out - discard + result = false else: raiseOSError(errCode) # Timer processing. - processTimers(p) + processTimers(p, result) # Callback queue processing - processPendingCallbacks(p) + processPendingCallbacks(p, result) + var acceptEx: WSAPROC_ACCEPTEX var connectEx: WSAPROC_CONNECTEX @@ -1202,7 +1205,7 @@ else: # descriptor was unregistered in callback via `unregister()`. discard - proc poll*(timeout = 500) = + proc runOnce(timeout = 500): bool = let p = getGlobalDispatcher() when ioselSupportedPlatform: let customSet = {Event.Timer, Event.Signal, Event.Process, @@ -1212,6 +1215,7 @@ else: raise newException(ValueError, "No handles or timers registered in dispatcher.") + result = false if not p.selector.isEmpty(): var keys: array[64, ReadyKey] var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) @@ -1224,20 +1228,24 @@ else: if Event.Read in events or events == {Event.Error}: processBasicCallbacks(fd, readList) + result = true if Event.Write in events or events == {Event.Error}: processBasicCallbacks(fd, writeList) + result = true if Event.User in events or events == {Event.Error}: processBasicCallbacks(fd, readList) custom = true if rLength == 0: p.selector.unregister(fd) + result = true when ioselSupportedPlatform: if (customSet * events) != {}: custom = true processCustomCallbacks(fd) + result = true # because state `data` can be modified in callback we need to update # descriptor events with currently registered callbacks. @@ -1249,9 +1257,9 @@ else: p.selector.updateHandle(SocketHandle(fd), newEvents) # Timer processing. - processTimers(p) + processTimers(p, result) # Callback queue processing - processPendingCallbacks(p) + processPendingCallbacks(p, result) proc recv*(socket: AsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] = @@ -1474,6 +1482,19 @@ else: data.readList.add(cb) p.selector.registerEvent(SelectEvent(ev), data) +proc drain*(timeout = 500) = + ## Waits for completion events and processes them. Raises ``ValueError`` + ## if there are no pending operations. In contrast to ``poll`` this + ## processes as many events as are available. + if runOnce(timeout): + while runOnce(0): discard + +proc poll*(timeout = 500) = + ## Waits for completion events and processes them. Raises ``ValueError`` + ## if there are no pending operations. This runs the underlying OS + ## `epoll`:idx: or `kqueue`:idx: primitive only once. + discard runOnce() + # Common procedures between current and upcoming asyncdispatch include includes.asynccommon diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index 48043b4b56..d2e4cfec11 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -579,9 +579,9 @@ else: var event = newSelectEvent() selector.registerEvent(event, 1) discard selector.select(0) - event.setEvent() + event.trigger() var rc1 = selector.select(0) - event.setEvent() + event.trigger() var rc2 = selector.select(0) var rc3 = selector.select(0) assert(len(rc1) == 1 and len(rc2) == 1 and len(rc3) == 0) @@ -611,7 +611,7 @@ else: var event = newSelectEvent() for i in 0..high(thr): createThread(thr[i], event_wait_thread, event) - event.setEvent() + event.trigger() joinThreads(thr) assert(counter == 1) result = true From 7f6afa9e9b554799cf9a39d0f8cc7d35e47a2cb4 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Tue, 19 Dec 2017 13:57:37 +0200 Subject: [PATCH 065/200] Make asyncjs Future[void] play nicely with last line discardable calls --- lib/js/asyncjs.nim | 32 +++++++++++++++++++++++++++++--- tests/js/tasync.nim | 9 +++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index bde3d787fb..73dbfb1e7d 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -83,20 +83,46 @@ proc replaceReturn(node: var NimNode) = replaceReturn(son) inc z +proc isFutureVoid(node: NimNode): bool = + result = node.kind == nnkBracketExpr and + node[0].kind == nnkIdent and $node[0] == "Future" and + node[1].kind == nnkIdent and $node[1] == "void" + proc generateJsasync(arg: NimNode): NimNode = assert arg.kind == nnkProcDef result = arg + var isVoid = false if arg.params[0].kind == nnkEmpty: result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void")) + isVoid = true + elif isFutureVoid(arg.params[0]): + isVoid = true var code = result.body replaceReturn(code) result.body = nnkStmtList.newTree() - var q = quote: + + var awaitFunction = quote: proc await[T](f: Future[T]): T {.importcpp: "(await #)".} - proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} - result.body.add(q) + result.body.add(awaitFunction) + + var resolve: NimNode + var jsResolveNode = ident("jsResolve") + if isVoid: + resolve = quote: + var `jsResolveNode` {.importcpp: "undefined".}: Future[void] + else: + resolve = quote: + proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} + result.body.add(resolve) + for child in code: result.body.add(child) + + if isVoid: + var voidFix = quote: + return `jsResolveNode` + result.body.add(voidFix) + result.pragma = quote: {.codegenDecl: "async function $2($3)".} diff --git a/tests/js/tasync.nim b/tests/js/tasync.nim index a164827d20..8cc972a626 100644 --- a/tests/js/tasync.nim +++ b/tests/js/tasync.nim @@ -3,6 +3,7 @@ discard """ output: ''' 0 x +e ''' """ @@ -12,15 +13,19 @@ import asyncjs # for js proc y(e: int): Future[string] -proc x(e: int) {.async.} = +proc e: int {.discardable.} = + echo "e" + return 2 + +proc x(e: int): Future[void] {.async.} = var s = await y(e) echo s + e() proc y(e: int): Future[string] {.async.} = echo 0 return "x" - discard x(2) From b3dfc93beee2ac47a907ea77c1ed2da84ba4b672 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Tue, 19 Dec 2017 20:50:37 +0200 Subject: [PATCH 066/200] Fix forward --- lib/js/asyncjs.nim | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 73dbfb1e7d..c99170a49d 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -92,33 +92,37 @@ proc generateJsasync(arg: NimNode): NimNode = assert arg.kind == nnkProcDef result = arg var isVoid = false + var jsResolveNode = ident("jsResolve") + if arg.params[0].kind == nnkEmpty: result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void")) isVoid = true elif isFutureVoid(arg.params[0]): isVoid = true + var code = result.body replaceReturn(code) result.body = nnkStmtList.newTree() - var awaitFunction = quote: - proc await[T](f: Future[T]): T {.importcpp: "(await #)".} - result.body.add(awaitFunction) + if len(code) > 0: + var awaitFunction = quote: + proc await[T](f: Future[T]): T {.importcpp: "(await #)".} + result.body.add(awaitFunction) - var resolve: NimNode - var jsResolveNode = ident("jsResolve") - if isVoid: - resolve = quote: - var `jsResolveNode` {.importcpp: "undefined".}: Future[void] + var resolve: NimNode + if isVoid: + resolve = quote: + var `jsResolveNode` {.importcpp: "undefined".}: Future[void] + else: + resolve = quote: + proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} + result.body.add(resolve) else: - resolve = quote: - proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} - result.body.add(resolve) - + result.body = newEmptyNode() for child in code: result.body.add(child) - if isVoid: + if len(code) > 0 and isVoid: var voidFix = quote: return `jsResolveNode` result.body.add(voidFix) @@ -126,6 +130,7 @@ proc generateJsasync(arg: NimNode): NimNode = result.pragma = quote: {.codegenDecl: "async function $2($3)".} + macro async*(arg: untyped): untyped = ## Macro which converts normal procedures into ## javascript-compatible async procedures From 7b495e23d453c6528c9eee3ac8ae6982744ef403 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Wed, 20 Dec 2017 14:09:02 +0200 Subject: [PATCH 067/200] Fix the forward test --- tests/js/tasync.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/js/tasync.nim b/tests/js/tasync.nim index 8cc972a626..34ef97b8bf 100644 --- a/tests/js/tasync.nim +++ b/tests/js/tasync.nim @@ -1,7 +1,6 @@ discard """ disabled: true output: ''' -0 x e ''' @@ -11,7 +10,7 @@ import asyncjs # demonstrate forward definition # for js -proc y(e: int): Future[string] +proc y(e: int): Future[string] {.async.} proc e: int {.discardable.} = echo "e" @@ -23,8 +22,10 @@ proc x(e: int): Future[void] {.async.} = e() proc y(e: int): Future[string] {.async.} = - echo 0 - return "x" + if e > 0: + return await y(0) + else: + return "x" discard x(2) From eba544996d5629943dbf84c0eeedaf5b958d6363 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Wed, 20 Dec 2017 14:11:22 +0200 Subject: [PATCH 068/200] Fix docs! --- lib/js/asyncjs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index c99170a49d..ec410ee39a 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -44,10 +44,10 @@ ## resolve(game) ## return promise ## -## Forward definitions work properly, you just don't need to add the ``{.async.}`` pragma: +## Forward definitions work properly, you just need to always add the ``{.async.}`` pragma: ## ## .. code-block:: nim -## proc loadGame(name: string): Future[Game] +## proc loadGame(name: string): Future[Game] {.async.} ## ## JavaScript compatibility ## ~~~~~~~~~~~~~~~~~~~~~~~~~ From 47d00eb397231a72c246b1c97cd7c1048be6bee9 Mon Sep 17 00:00:00 2001 From: skilchen Date: Wed, 20 Dec 2017 22:26:22 +0100 Subject: [PATCH 069/200] add missing math.trunc for js backend (#6950) --- lib/pure/math.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 7fd8bbcefd..a9dabfa486 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -291,6 +291,8 @@ when not defined(JS): ## echo fmod(-2.5, 0.3) ## -0.1 else: + proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} From af1404c85daff26acd119c354c21aa5bd156420c Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Thu, 21 Dec 2017 03:58:24 -0500 Subject: [PATCH 070/200] Minor fixes to strformat docs (#6953) --- lib/pure/strformat.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index c4044867da..180cbcbecc 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -42,7 +42,7 @@ An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: format(key, temp) format(" is ", temp) format(value, arg, temp) - format("{z}", temp) + format(" {z}", temp) temp Parts of the string that are enclosed in the curly braces are interpreted @@ -94,7 +94,7 @@ The general form of a standard format specifier is:: [[fill]align][sign][#][0][minimumwidth][.precision][type] -The brackets ([]) indicate an optional element. +The square brackets ``[]`` indicate an optional element. The optional align flag can be one of the following: @@ -126,8 +126,9 @@ The 'sign' option is only valid for numeric types, and can be one of the followi positive as well as negative numbers. ``-`` Indicates that a sign should be used only for negative numbers (this is the default behavior). -`` `` (space) Indicates that a leading space should be used on +(space) Indicates that a leading space should be used on positive numbers. +================= ==================================================== If the '#' character is present, integers use the 'alternate form' for formatting. This means that binary, octal, and hexadecimal output will be prefixed From ee67a67ac450968fc3d753005fe7362ac0414a10 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 21 Dec 2017 10:03:33 +0100 Subject: [PATCH 071/200] first steps in adding template/macro calls to stack traces --- compiler/ast.nim | 4 ++-- compiler/ccgexprs.nim | 34 +++++++++++++++++++++++++----- compiler/ccgstmts.nim | 6 ++---- compiler/cgen.nim | 42 ++++++++++++++++++++++++-------------- compiler/evaltempl.nim | 10 ++++++--- compiler/jsgen.nim | 2 ++ compiler/lambdalifting.nim | 2 +- compiler/renderer.nim | 4 ++-- compiler/sem.nim | 2 +- compiler/semexprs.nim | 1 + compiler/semfold.nim | 3 +++ compiler/semgnrc.nim | 2 +- compiler/semtempl.nim | 4 ++-- compiler/transf.nim | 2 +- compiler/vmgen.nim | 2 ++ lib/core/macros.nim | 2 +- lib/system/excpt.nim | 12 +++++++++++ 17 files changed, 96 insertions(+), 38 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 5b923acb25..27a44c6c2b 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -62,8 +62,8 @@ type nkTripleStrLit, # a triple string literal """ nkNilLit, # the nil literal # end of atoms - nkMetaNode_Obsolete, # difficult to explain; represents itself - # (used for macros) + nkComesFrom, # "comes from" template/macro information for + # better stack trace generation nkDotCall, # used to temporarily flag a nkCall node; # this is used # for transforming ``s.len`` to ``len(s)`` diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1505052cc6..dd526c5fba 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1982,10 +1982,35 @@ proc genComplexConst(p: BProc, sym: PSym, d: var TLoc) = assert((sym.loc.r != nil) and (sym.loc.t != nil)) putLocIntoDest(p, d, sym.loc) +template genStmtListExprImpl(exprOrStmt) {.dirty.} = + #let hasNimFrame = magicsys.getCompilerProc("nimFrame") != nil + let hasNimFrame = p.prc != nil and + sfSystemModule notin p.module.module.flags and + optStackTrace in p.prc.options + var frameName: Rope = nil + for i in 0 .. n.len - 2: + let it = n[i] + if it.kind == nkComesFrom: + if hasNimFrame and frameName == nil: + inc p.labels + frameName = "FR" & rope(p.labels) & "_" + let theMacro = it[0].sym + add p.s(cpsStmts), initFrameNoDebug(p, frameName, + makeCString theMacro.name.s, + theMacro.info.quotedFilename, it.info.line) + else: + genStmts(p, it) + if n.len > 0: exprOrStmt + if frameName != nil: + add p.s(cpsStmts), deinitFrameNoDebug(p, frameName) + proc genStmtListExpr(p: BProc, n: PNode, d: var TLoc) = - var length = sonsLen(n) - for i in countup(0, length - 2): genStmts(p, n.sons[i]) - if length > 0: expr(p, n.sons[length - 1], d) + genStmtListExprImpl: + expr(p, n[n.len - 1], d) + +proc genStmtList(p: BProc, n: PNode) = + genStmtListExprImpl: + genStmts(p, n[n.len - 1]) proc upConv(p: BProc, n: PNode, d: var TLoc) = var a: TLoc @@ -2184,8 +2209,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkCheckedFieldExpr: genCheckedRecordField(p, n, d) of nkBlockExpr, nkBlockStmt: genBlock(p, n, d) of nkStmtListExpr: genStmtListExpr(p, n, d) - of nkStmtList: - for i in countup(0, sonsLen(n) - 1): genStmts(p, n.sons[i]) + of nkStmtList: genStmtList(p, n) of nkIfExpr, nkIfStmt: genIf(p, n, d) of nkWhen: # This should be a "when nimvm" node. diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 24a376dece..36816cc2c0 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -564,9 +564,6 @@ proc genBreakStmt(p: BProc, t: PNode) = genLineDir(p, t) lineF(p, cpsStmts, "goto $1;$n", [label]) -proc getRaiseFrmt(p: BProc): string = - result = "#raiseException((#Exception*)$1, $2);$n" - proc genRaiseStmt(p: BProc, t: PNode) = if p.inExceptBlock > 0: # if the current try stmt have a finally block, @@ -580,7 +577,8 @@ proc genRaiseStmt(p: BProc, t: PNode) = var e = rdLoc(a) var typ = skipTypes(t.sons[0].typ, abstractPtrs) genLineDir(p, t) - lineCg(p, cpsStmts, getRaiseFrmt(p), [e, makeCString(typ.sym.name.s)]) + lineCg(p, cpsStmts, "#raiseException((#Exception*)$1, $2);$n", + [e, makeCString(typ.sym.name.s)]) else: genLineDir(p, t) # reraise the last exception: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 07c2824d0f..217138dd04 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -493,7 +493,32 @@ proc initLocExprSingleUse(p: BProc, e: PNode, result: var TLoc) = proc lenField(p: BProc): Rope = result = rope(if p.module.compileToCpp: "len" else: "Sup.len") -include ccgcalls, "ccgstmts.nim", "ccgexprs.nim" +include ccgcalls, "ccgstmts.nim" + +proc initFrame(p: BProc, procname, filename: Rope): Rope = + discard cgsym(p.module, "nimFrame") + if p.maxFrameLen > 0: + discard cgsym(p.module, "VarSlot") + result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n", + procname, filename, p.maxFrameLen.rope, + p.blocks[0].frameLen.rope) + else: + result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename) + +proc initFrameNoDebug(p: BProc; frame, procname, filename: Rope; line: int): Rope = + discard cgsym(p.module, "nimFrame") + addf(p.blocks[0].sections[cpsLocals], "TFrame $1;$n", [frame]) + result = rfmt(nil, "\t$1.procname = $2; $1.filename = $3; " & + " $1.line = $4; $1.len = -1; nimFrame(&$1);$n", + frame, procname, filename, rope(line)) + +proc deinitFrameNoDebug(p: BProc; frame: Rope): Rope = + result = rfmt(p.module, "\t#popFrameOfAddr(&$1);$n", frame) + +proc deinitFrame(p: BProc): Rope = + result = rfmt(p.module, "\t#popFrame();$n") + +include ccgexprs # ----------------------------- dynamic library handling ----------------- # We don't finalize dynamic libs as the OS does this for us. @@ -600,7 +625,7 @@ proc symInDynamicLibPartial(m: BModule, sym: PSym) = sym.typ.sym = nil # generate a new name proc cgsym(m: BModule, name: string): Rope = - var sym = magicsys.getCompilerProc(name) + let sym = magicsys.getCompilerProc(name) if sym != nil: case sym.kind of skProc, skFunc, skMethod, skConverter, skIterator: genProc(m, sym) @@ -637,19 +662,6 @@ proc generateHeaders(m: BModule) = add(m.s[cfsHeaders], "#undef powerpc" & tnl) add(m.s[cfsHeaders], "#undef unix" & tnl) -proc initFrame(p: BProc, procname, filename: Rope): Rope = - discard cgsym(p.module, "nimFrame") - if p.maxFrameLen > 0: - discard cgsym(p.module, "VarSlot") - result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n", - procname, filename, p.maxFrameLen.rope, - p.blocks[0].frameLen.rope) - else: - result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename) - -proc deinitFrame(p: BProc): Rope = - result = rfmt(p.module, "\t#popFrame();$n") - proc closureSetup(p: BProc, prc: PSym) = if tfCapturesEnv notin prc.typ.flags: return # prc.ast[paramsPos].last contains the type we're after: diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 7fa6df3da4..704ff819c9 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -109,7 +109,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode = var evalTemplateCounter* = 0 # to prevent endless recursion in templates instantiation -proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode = +proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode = when true: result = res result.info = info @@ -124,8 +124,12 @@ proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode = if x[i].kind in nkCallKinds: x.sons[i].info = info else: - result = newNodeI(nkPar, info) + result = newNodeI(nkStmtListExpr, info) + var d = newNodeI(nkComesFrom, info) + d.add newSymNode(sym, info) + result.add d result.add res + result.typ = res.typ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode = inc(evalTemplateCounter) @@ -156,6 +160,6 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode = for i in countup(0, safeLen(body) - 1): evalTemplateAux(body.sons[i], args, ctx, result) result.flags.incl nfFromTemplate - result = wrapInComesFrom(n.info, result) + result = wrapInComesFrom(n.info, tmpl, result) dec(evalTemplateCounter) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 50dfa22f89..65a6a5dae9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2369,6 +2369,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkGotoState, nkState: internalError(n.info, "first class iterators not implemented") of nkPragmaBlock: gen(p, n.lastSon, r) + of nkComesFrom: + discard "XXX to implement for better stack traces" else: internalError(n.info, "gen: unknown node type: " & $n.kind) var globals: PGlobals diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 8204395242..cf43ba15d3 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -723,7 +723,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; result = accessViaEnvParam(n, owner) else: result = accessViaEnvVar(n, owner, d, c) - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom, nkTemplateDef, nkTypeSection: discard of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 2092fc67c1..a514bf6b7d 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1397,8 +1397,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = put(g, tkBracketRi, "]") of nkTupleClassTy: put(g, tkTuple, "tuple") - of nkMetaNode_Obsolete: - put(g, tkParLe, "(META|") + of nkComesFrom: + put(g, tkParLe, "(ComesFrom|") gsub(g, n, 0) put(g, tkParRi, ")") of nkGotoState, nkState: diff --git a/compiler/sem.nim b/compiler/sem.nim index 495321de45..bc994201d1 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -423,7 +423,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, result = evalMacroCall(c.module, c.cache, n, nOrig, sym) if efNoSemCheck notin flags: result = semAfterMacroCall(c, n, result, sym, flags) - result = wrapInComesFrom(nOrig.info, result) + result = wrapInComesFrom(nOrig.info, sym, result) popInfoContext() proc forceBool(c: PContext, n: PNode): PNode = diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7867c7e368..51a088a9e1 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2380,6 +2380,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = if n.len != 1 and n.len != 2: illFormedAst(n) for i in 0 ..< n.len: n.sons[i] = semExpr(c, n.sons[i]) + of nkComesFrom: discard "ignore the comes from information for now" else: localError(n.info, errInvalidExpressionX, renderTree(n, {renderNoComments})) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 1e7c0aa9e4..d2d36140d8 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -655,5 +655,8 @@ proc getConstExpr(m: PSym, n: PNode): PNode = result.typ = n.typ of nkBracketExpr: result = foldArrayAccess(m, n) of nkDotExpr: result = foldFieldAccess(m, n) + of nkStmtListExpr: + if n.len == 2 and n[0].kind == nkComesFrom: + result = getConstExpr(m, n[1]) else: discard diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index da2c6fe7f3..16da06952a 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -186,7 +186,7 @@ proc semGenericStmt(c: PContext, n: PNode, let a = n.sym let b = getGenSym(c, a) if b != a: n.sym = b - of nkEmpty, succ(nkSym)..nkNilLit: + of nkEmpty, succ(nkSym)..nkNilLit, nkComesFrom: # see tests/compile/tgensymgeneric.nim: # We need to open the gensym'ed symbol again so that the instantiation # creates a fresh copy; but this is wrong the very first reason for gensym diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 1c9d8271af..f90dff8f1a 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -331,7 +331,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = of nkMixinStmt: if c.scopeN > 0: result = semTemplBodySons(c, n) else: result = semMixinStmt(c.c, n, c.toMixin) - of nkEmpty, nkSym..nkNilLit: + of nkEmpty, nkSym..nkNilLit, nkComesFrom: discard of nkIfStmt: for i in countup(0, sonsLen(n)-1): @@ -528,7 +528,7 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode = result = semTemplBodyDirty(c, n.sons[0]) of nkBindStmt: result = semBindStmt(c.c, n, c.toBind) - of nkEmpty, nkSym..nkNilLit: + of nkEmpty, nkSym..nkNilLit, nkComesFrom: discard else: # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam', diff --git a/compiler/transf.nim b/compiler/transf.nim index 8e4bb935b0..6bc809fd20 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -791,7 +791,7 @@ proc transform(c: PTransf, n: PNode): PTransNode = case n.kind of nkSym: result = transformSym(c, n) - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom: # nothing to be done for leaves: result = PTransNode(n) of nkBracketExpr: result = transformArrayAccess(c, n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 252b7c788b..3790a8392f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1847,6 +1847,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = globalError(n.info, errGenerated, "VM is not allowed to 'cast'") of nkTypeOfExpr: genTypeLit(c, n.typ, dest) + of nkComesFrom: + discard "XXX to implement for better stack traces" else: globalError(n.info, errGenerated, "cannot generate VM code for " & $n) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ee6c1a09ff..b08a2198e1 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -21,7 +21,7 @@ type nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkUIntLit, nnkUInt8Lit, nnkUInt16Lit, nnkUInt32Lit, nnkUInt64Lit, nnkFloatLit, nnkFloat32Lit, nnkFloat64Lit, nnkFloat128Lit, nnkStrLit, nnkRStrLit, - nnkTripleStrLit, nnkNilLit, nnkMetaNode, nnkDotCall, + nnkTripleStrLit, nnkNilLit, nnkComesFrom, nnkDotCall, nnkCommand, nnkCall, nnkCallStrLit, nnkInfix, nnkPrefix, nnkPostfix, nnkHiddenCallConv, nnkExprEqExpr, diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 70c18ae21e..8e42ea4683 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -70,6 +70,18 @@ proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr proc popFrame {.compilerRtl, inl.} = framePtr = framePtr.prev +when false: + proc popFrameOfAddr(s: PFrame) {.compilerRtl.} = + var it = framePtr + if it == s: + framePtr = framePtr.prev + else: + while it != nil: + if it == s: + framePtr = it.prev + break + it = it.prev + proc setFrame*(s: PFrame) {.compilerRtl, inl.} = framePtr = s From 77d56aaff68464933d904757bb2ef5b57c739b00 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 21 Dec 2017 10:11:45 +0100 Subject: [PATCH 072/200] cleanup times.nim --- compiler/scriptconfig.nim | 3 ++- lib/pure/times.nim | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index dac2672636..8eb76457cc 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -73,7 +73,8 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; cbos copyFile: os.copyFile(getString(a, 0), getString(a, 1)) cbos getLastModificationTime: - setResult(a, toSeconds(getLastModificationTime(getString(a, 0)))) + # depends on Time's implementation! + setResult(a, int64(getLastModificationTime(getString(a, 0)))) cbos rawExec: setResult(a, osproc.execCmd getString(a, 0)) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index dcc817b7b2..606acbc1c4 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -26,7 +26,6 @@ ## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() ## ## echo "epochTime() float value: ", epochTime() -## echo "getTime() float value: ", toSeconds(getTime()) ## echo "cpuTime() float value: ", cpuTime() ## echo "An hour from now : ", now() + 1.hours ## echo "An hour from (UTC) now: ", getTime().utc + initInterval(0,0,0,1) @@ -180,7 +179,7 @@ proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline. proc toEpochDay*(monthday: MonthdayRange, month: Month, year: int): int64 = ## Get the epoch day from a year/month/day date. ## The epoch day is the number of days since 1970/01/01 (it might be negative). - assertValidDate monthday, month, year + assertValidDate monthday, month, year # Based on http://howardhinnant.github.io/date_algorithms.html var (y, m, d) = (year, ord(month), monthday.int) if m <= 2: @@ -194,7 +193,7 @@ proc toEpochDay*(monthday: MonthdayRange, month: Month, year: int): int64 = proc fromEpochDay*(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = ## Get the year/month/day date from a epoch day. - ## The epoch day is the number of days since 1970/01/01 (it might be negative). + ## The epoch day is the number of days since 1970/01/01 (it might be negative). # Based on http://howardhinnant.github.io/date_algorithms.html var z = epochday z.inc 719468 @@ -494,11 +493,11 @@ proc local*(dt: DateTime): DateTime = dt.inZone(local()) proc utc*(t: Time): DateTime = - ## Shorthand for ``t.inZone(utc())``. + ## Shorthand for ``t.inZone(utc())``. t.inZone(utc()) proc local*(t: Time): DateTime = - ## Shorthand for ``t.inZone(local())``. + ## Shorthand for ``t.inZone(local())``. t.inZone(local()) proc getTime*(): Time {.tags: [TimeEffect], benign.} @@ -590,7 +589,7 @@ proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDiff, absD anew.year.dec() else: curMonth.dec() - result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay + result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay # Adding else: for mth in 1 .. newinterv.months: @@ -610,7 +609,7 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## Adds ``interval`` to ``dt``. Components from ``interval`` are added ## in the order of their size, i.e first the ``years`` component, then the ``months`` ## component and so on. The returned ``DateTime`` will have the same timezone as the input. - ## + ## ## Note that when adding months, monthday overflow is allowed. This means that if the resulting ## month doesn't have enough days it, the month will be incremented and the monthday will be ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, @@ -1392,7 +1391,7 @@ proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = ## ## **Warning:** This procedure is deprecated since version 0.14.0. ## Use ``toTime`` instead. - dt.toTime + dt.toTime when defined(JS): var startMilsecs = getTime() From 18508cdcc3311c606250b2c1eee120a551baf4cd Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 21 Dec 2017 10:23:21 +0100 Subject: [PATCH 073/200] testament: use splitWhitespace instead of split --- tests/testament/specs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index e8513ab24c..ac79e39429 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -116,7 +116,7 @@ proc specDefaults*(result: var TSpec) = result.maxCodeSize = 0 proc parseTargets*(value: string): set[TTarget] = - for v in value.normalize.split: + for v in value.normalize.splitWhitespace: case v of "c": result.incl(targetC) of "cpp", "c++": result.incl(targetCpp) @@ -192,7 +192,7 @@ proc parseSpec*(filename: string): TSpec = of "ccodecheck": result.ccodeCheck = e.value of "maxcodesize": discard parseInt(e.value, result.maxCodeSize) of "target", "targets": - for v in e.value.normalize.split: + for v in e.value.normalize.splitWhitespace: case v of "c": result.targets.incl(targetC) of "cpp", "c++": result.targets.incl(targetCpp) From 765116d54736c4d0ad50e1cf222c325f7d8a4d1c Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 21 Dec 2017 10:49:09 +0100 Subject: [PATCH 074/200] testament html generation improvements; merged #6667 manually --- tests/testament/htmlgen.nim | 142 ++++++------ tests/testament/testamenthtml.templ | 331 +++++++++++++--------------- 2 files changed, 217 insertions(+), 256 deletions(-) diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim index 05c24b2b58..bf26a956d2 100644 --- a/tests/testament/htmlgen.nim +++ b/tests/testament/htmlgen.nim @@ -9,54 +9,33 @@ ## HTML generator for the tester. -import cgi, backend, strutils, json, os +import cgi, backend, strutils, json, os, tables, times import "testamenthtml.templ" -proc generateTestRunTabListItemPartial(outfile: File, testRunRow: JsonNode, firstRow = false) = +proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = let - # The first tab gets the bootstrap class for a selected tab - firstTabActiveClass = if firstRow: "active" - else: "" - commitId = htmlQuote testRunRow["commit"].str - hash = htmlQuote(testRunRow["commit"].str) - branch = htmlQuote(testRunRow["branch"].str) - machineId = htmlQuote testRunRow["machine"].str - machineName = htmlQuote(testRunRow["machine"].str) - - outfile.generateHtmlTabListItem( - firstTabActiveClass, - commitId, - machineId, - branch, - hash, - machineName - ) - -proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode, onlyFailing = false) = - let - trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str) + trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str). + multiReplace({".": "_", " ": "_", ":": "_"}) name = testResultRow["name"].str.htmlQuote() category = testResultRow["category"].str.htmlQuote() target = testResultRow["target"].str.htmlQuote() action = testResultRow["action"].str.htmlQuote() result = htmlQuote testResultRow["result"].str - expected = htmlQuote testResultRow["expected"].str - gotten = htmlQuote testResultRow["given"].str + expected = testResultRow["expected"].str + gotten = testResultRow["given"].str timestamp = "unknown" - var panelCtxClass, textCtxClass, bgCtxClass, resultSign, resultDescription: string + var + panelCtxClass, textCtxClass, bgCtxClass: string + resultSign, resultDescription: string case result of "reSuccess": - if onlyFailing: - return panelCtxClass = "success" textCtxClass = "success" bgCtxClass = "success" resultSign = "ok" resultDescription = "PASS" of "reIgnored": - if onlyFailing: - return panelCtxClass = "info" textCtxClass = "info" bgCtxClass = "info" @@ -71,9 +50,7 @@ proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode, only outfile.generateHtmlTestresultPanelBegin( trId, name, target, category, action, resultDescription, - timestamp, - result, resultSign, - panelCtxClass, textCtxClass, bgCtxClass + timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass ) if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace(): outfile.generateHtmlTestresultOutputNone() @@ -90,7 +67,7 @@ type totalCount, successCount, ignoredCount, failedCount: int successPercentage, ignoredPercentage, failedPercentage: BiggestFloat -proc allTestResults(): AllTests = +proc allTestResults(onlyFailing = false): AllTests = result.data = newJArray() for file in os.walkFiles("testresults/*.json"): let data = parseFile(file) @@ -98,69 +75,74 @@ proc allTestResults(): AllTests = echo "[ERROR] ignoring json file that is not an array: ", file else: for elem in data: - result.data.add elem let state = elem["result"].str + inc result.totalCount if state.contains("reSuccess"): inc result.successCount elif state.contains("reIgnored"): inc result.ignoredCount + if not onlyFailing or not(state.contains("reSuccess")): + result.data.add elem + result.successPercentage = 100 * + (result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat) + result.ignoredPercentage = 100 * + (result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat) + result.failedCount = result.totalCount - + result.successCount - result.ignoredCount + result.failedPercentage = 100 * + (result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat) - result.totalCount = result.data.len - result.successPercentage = 100 * (result.successCount.toBiggestFloat() / result.totalCount.toBiggestFloat()) - result.ignoredPercentage = 100 * (result.ignoredCount.toBiggestFloat() / result.totalCount.toBiggestFloat()) - result.failedCount = result.totalCount - result.successCount - result.ignoredCount - result.failedPercentage = 100 * (result.failedCount.toBiggestFloat() / result.totalCount.toBiggestFloat()) - - -proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode, onlyFailing = false) = +proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) = for testresultRow in allResults: - generateTestResultPanelPartial(outfile, testresultRow, onlyFailing) + generateTestResultPanelPartial(outfile, testresultRow) -proc generateTestRunTabContentPartial(outfile: File, allResults: AllTests, testRunRow: JsonNode, onlyFailing = false, firstRow = false) = +proc generateAllTestsContent(outfile: File, allResults: AllTests, + onlyFailing = false) = + if allResults.data.len < 1: return # Nothing to do if there is no data. + # Only results from one test run means that test run environment info is the + # same for all tests let - # The first tab gets the bootstrap classes for a selected and displaying tab content - firstTabActiveClass = if firstRow: " in active" - else: "" - commitId = htmlQuote testRunRow["commit"].str - hash = htmlQuote(testRunRow["commit"].str) - branch = htmlQuote(testRunRow["branch"].str) - machineId = htmlQuote testRunRow["machine"].str - machineName = htmlQuote(testRunRow["machine"].str) - os = htmlQuote("unknown_os") - cpu = htmlQuote("unknown_cpu") + firstRow = allResults.data[0] + commit = htmlQuote firstRow["commit"].str + branch = htmlQuote firstRow["branch"].str + machine = htmlQuote firstRow["machine"].str - outfile.generateHtmlTabPageBegin( - firstTabActiveClass, commitId, - machineId, branch, hash, machineName, os, cpu, + outfile.generateHtmlAllTestsBegin( + machine, commit, branch, allResults.totalCount, - allResults.successCount, formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%", - allResults.ignoredCount, formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%", - allResults.failedCount, formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%" + allResults.successCount, + formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%", + allResults.ignoredCount, + formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%", + allResults.failedCount, + formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%", + onlyFailing ) - generateTestResultsPanelGroupPartial(outfile, allResults.data, onlyFailing) - outfile.generateHtmlTabPageEnd() - -proc generateTestRunsHtmlPartial(outfile: File, allResults: AllTests, onlyFailing = false) = - # Iterating the results twice, get entire result set in one go - outfile.generateHtmlTabListBegin() - if allResults.data.len > 0: - generateTestRunTabListItemPartial(outfile, allResults.data[0], true) - outfile.generateHtmlTabListEnd() - - outfile.generateHtmlTabContentsBegin() - var firstRow = true - for testRunRow in allResults.data: - generateTestRunTabContentPartial(outfile, allResults, testRunRow, onlyFailing, firstRow) - if firstRow: - firstRow = false - outfile.generateHtmlTabContentsEnd() + generateTestResultsPanelGroupPartial(outfile, allResults.data) + outfile.generateHtmlAllTestsEnd() proc generateHtml*(filename: string, onlyFailing: bool) = + let + currentTime = getTime().getLocalTime() + timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz") var outfile = open(filename, fmWrite) outfile.generateHtmlBegin() - generateTestRunsHtmlPartial(outfile, allTestResults(), onlyFailing) + generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing) - outfile.generateHtmlEnd() + outfile.generateHtmlEnd(timestring) outfile.flushFile() close(outfile) + +proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) = + var + outfile = stdout + jsonString: string + + let results = allTestResults(onlyFailing) + if prettyPrint: + jsonString = results.data.pretty() + else: + jsonString = $ results.data + + outfile.writeLine(jsonString) diff --git a/tests/testament/testamenthtml.templ b/tests/testament/testamenthtml.templ index f7477f3aa5..9190f370eb 100644 --- a/tests/testament/testamenthtml.templ +++ b/tests/testament/testamenthtml.templ @@ -29,7 +29,7 @@ */ /** - * + * * @param {number} index * @param {Element[]} elemArray * @param {executeForElement} executeOnItem @@ -69,17 +69,16 @@ } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) * @param {executeForElement} executeOnEachPanel */ - function wholePanelAll(tabId, category, executeOnEachPanel) { + function wholePanelAll(category, executeOnEachPanel) { var selector = "div.panel"; if (typeof category === "string" && category) { selector += "-" + category; } - var jqPanels = $(selector, $("#" + tabId)); + var jqPanels = $(selector); /** @type {Element[]} */ var elemArray = jqPanels.toArray(); @@ -87,17 +86,16 @@ } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) * @param {executeForElement} executeOnEachPanel */ - function panelBodyAll(tabId, category, executeOnEachPanelBody) { + function panelBodyAll(category, executeOnEachPanelBody) { var selector = "div.panel"; if (typeof category === "string" && category) { selector += "-" + category; } - var jqPanels = $(selector, $("#" + tabId)); + var jqPanels = $(selector); var jqPanelBodies = $("div.panel-body", jqPanels); /** @type {Element[]} */ @@ -107,35 +105,31 @@ } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) */ - function showAll(tabId, category) { - wholePanelAll(tabId, category, executeShowOnElement); + function showAll(category) { + wholePanelAll(category, executeShowOnElement); } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) */ - function hideAll(tabId, category) { - wholePanelAll(tabId, category, executeHideOnElement); + function hideAll(category) { + wholePanelAll(category, executeHideOnElement); } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) */ - function expandAll(tabId, category) { - panelBodyAll(tabId, category, executeExpandOnElement); + function expandAll(category) { + panelBodyAll(category, executeExpandOnElement); } /** - * @param {string} tabId The id of the tabpanel div to search. * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) */ - function collapseAll(tabId, category) { - panelBodyAll(tabId, category, executeCollapseOnElement); + function collapseAll(category) { + panelBodyAll(category, executeCollapseOnElement); } @@ -143,176 +137,161 @@

Testament Test Results Nim Tester

#end proc -#proc generateHtmlTabListBegin*(outfile: File) = -
    -#end proc -#proc generateHtmlTabListItem*(outfile: File, firstTabActiveClass, commitId, -# machineId, branch, hash, machineName: string) = - -#end proc -#proc generateHtmlTabListEnd*(outfile: File) = -
-#end proc -#proc generateHtmlTabContentsBegin*(outfile: File) = -
-#end proc -#proc generateHtmlTabPageBegin*(outfile: File, firstTabActiveClass, commitId, -# machineId, branch, hash, machineName, os, cpu: string, totalCount: BiggestInt, +#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string, +# totalCount: BiggestInt, # successCount: BiggestInt, successPercentage: string, # ignoredCount: BiggestInt, ignoredPercentage: string, -# failedCount: BiggestInt, failedPercentage: string) = -
-

%branch#%hash@%machineName

-
-
Branch
-
%branch
-
Commit Hash
-
%hash
-
Machine Name
-
%machineName
-
OS
-
%os
-
CPU
-
%cpu
-
All Tests
-
- - %totalCount -
-
Successful Tests
-
- - %successCount (%successPercentage) -
-
Skipped Tests
-
- - %ignoredCount (%ignoredPercentage) -
-
Failed Tests
-
- - %failedCount (%failedPercentage) -
-
-
- - - - - - - - - - - - - - - - - -
All Tests -
- - - - -
-
Successful Tests -
- - - - -
-
Skipped Tests -
- - - - -
-
Failed Tests -
- - - - -
-
-
-
+# failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) = +
+
Hostname
+
%machine
+
Git Commit
+
%commit
+
Branch ref.
+
%branch
+
+
+
All Tests
+
+ + %totalCount +
+
Successful Tests
+
+ + %successCount (%successPercentage) +
+
Skipped Tests
+
+ + %ignoredCount (%ignoredPercentage) +
+
Failed Tests
+
+ + %failedCount (%failedPercentage) +
+
+
+ +# if not onlyFailing: + + + + + + + + +# end if + + + + + + + + +
All Tests +
+ + + + +
+
Successful Tests +
+ + + + +
+
Skipped Tests +
+ + + + +
+
Failed Tests +
+ + + + +
+
+
+
#end proc #proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category, -# action, resultDescription, timestamp, result, resultSign, +# action, resultDescription, timestamp, result, resultSign, # panelCtxClass, textCtxClass, bgCtxClass: string) = -
- -
-
-
Name
-
%name
-
Category
-
%category
-
Timestamp
-
%timestamp
-
Nim Action
-
%action
-
Nim Backend Target
-
%target
-
Code
-
%result
-
+
+ +
+
+
Name
+
%name
+
Category
+
%category
+
Timestamp
+
%timestamp
+
Nim Action
+
%action
+
Nim Backend Target
+
%target
+
Code
+
%result
+
#end proc #proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) = -
- - - - - - - - - - - - - -
ExpectedActual
%expected
%gotten
-
-#end proc -#proc generateHtmlTestresultOutputNone*(outfile: File) = -

No output details

-#end proc -#proc generateHtmlTestresultPanelEnd*(outfile: File) = -
+
+ + + + + + + + + + + + + +
ExpectedActual
%expected
%gotten
#end proc -#proc generateHtmlTabPageEnd*(outfile: File) = +#proc generateHtmlTestresultOutputNone*(outfile: File) = +

No output details

+#end proc +#proc generateHtmlTestresultPanelEnd*(outfile: File) =
#end proc -#proc generateHtmlTabContentsEnd*(outfile: File) = +#proc generateHtmlAllTestsEnd*(outfile: File) =
#end proc -#proc generateHtmlEnd*(outfile: File) = +#proc generateHtmlEnd*(outfile: File, timestamp: string) = +
+
+

+ Report generated by: testament – Nim Tester +
+ Made with Nim. Generated on: %timestamp +

+
- \ No newline at end of file + From 70380882c54a6eea08c715b015399673b55e9bc0 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sat, 7 Oct 2017 21:49:11 +0300 Subject: [PATCH 075/200] fix #6108 --- compiler/semexprs.nim | 13 ++++++++----- tests/concepts/tcomparable.nim | 13 +++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 tests/concepts/tcomparable.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 51a088a9e1..55c43ed09e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1365,13 +1365,16 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = if lhsIsResult: n.typ = enforceVoidContext if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ): - if cmpTypes(c, lhs.typ, rhs.typ) == isGeneric: + var rhsTyp = rhs.typ + if rhsTyp.kind in tyUserTypeClasses and rhsTyp.isResolvedUserTypeClass: + rhsTyp = rhsTyp.lastSon + if cmpTypes(c, lhs.typ, rhsTyp) in {isGeneric, isEqual}: internalAssert c.p.resultSym != nil - lhs.typ = rhs.typ - c.p.resultSym.typ = rhs.typ - c.p.owner.typ.sons[0] = rhs.typ + lhs.typ = rhsTyp + c.p.resultSym.typ = rhsTyp + c.p.owner.typ.sons[0] = rhsTyp else: - typeMismatch(n.info, lhs.typ, rhs.typ) + typeMismatch(n.info, lhs.typ, rhsTyp) n.sons[1] = fitNode(c, le, rhs, n.info) if not newDestructors: diff --git a/tests/concepts/tcomparable.nim b/tests/concepts/tcomparable.nim new file mode 100644 index 0000000000..06612a47e7 --- /dev/null +++ b/tests/concepts/tcomparable.nim @@ -0,0 +1,13 @@ +type + Comparable = concept a + (a < a) is bool + +proc myMax(a, b: Comparable): Comparable = + if a < b: + return b + else: + return a + +doAssert myMax(5, 10) == 10 +doAssert myMax(31.3, 1.23124) == 31.3 + From 057d5789ba6cf7e1f0cef681ffba77b0c4788110 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sat, 7 Oct 2017 22:17:16 +0300 Subject: [PATCH 076/200] fix #6277 --- compiler/ccgexprs.nim | 2 +- tests/concepts/titerable.nim | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/concepts/titerable.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index dd526c5fba..da92b281e9 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -849,7 +849,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = var a, b: TLoc initLocExpr(p, x, a) initLocExpr(p, y, b) - var ty = skipTypes(skipTypes(a.t, abstractVarRange), abstractPtrs) + var ty = skipTypes(a.t, abstractVarRange + abstractPtrs + tyUserTypeClasses) var first = intLiteral(firstOrd(ty)) # emit range check: if optBoundsCheck in p.options and tfUncheckedArray notin ty.flags: diff --git a/tests/concepts/titerable.nim b/tests/concepts/titerable.nim new file mode 100644 index 0000000000..b18658b2a1 --- /dev/null +++ b/tests/concepts/titerable.nim @@ -0,0 +1,20 @@ +discard """ + nimout: "int\nint" + output: 15 +""" + +import typetraits + +type + Iterable[T] = concept x + for value in x: + type(value) is T + +proc sum*[T](iter: Iterable[T]): T = + static: echo T.name + for element in iter: + static: echo element.type.name + result += element + +echo sum([1, 2, 3, 4, 5]) + From 2ceee884fe4d87e1540bf1b1dbfbf2e4d661c7b2 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 6 Oct 2017 16:36:26 +0300 Subject: [PATCH 077/200] fix #6462 --- compiler/ccgexprs.nim | 3 ++- compiler/semstmts.nim | 3 ++- tests/concepts/t6462.nim | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tests/concepts/t6462.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index da92b281e9..5a25a98530 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -683,9 +683,10 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = d.storage = OnHeap else: var a: TLoc - var typ = skipTypes(e.sons[0].typ, abstractInst) + var typ = e.sons[0].typ if typ.kind in {tyUserTypeClass, tyUserTypeClassInst} and typ.isResolvedUserTypeClass: typ = typ.lastSon + typ = typ.skipTypes(abstractInst) if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr: initLocExprSingleUse(p, e[0][0], d) return diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c85de35cd2..8ed120c98f 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -441,9 +441,10 @@ proc hasEmpty(typ: PType): bool = result = result or hasEmpty(s) proc makeDeref(n: PNode): PNode = - var t = skipTypes(n.typ, {tyGenericInst, tyAlias}) + var t = n.typ if t.kind in tyUserTypeClasses and t.isResolvedUserTypeClass: t = t.lastSon + t = skipTypes(t, {tyGenericInst, tyAlias}) result = n if t.kind == tyVar: result = newNodeIT(nkHiddenDeref, n.info, t.sons[0]) diff --git a/tests/concepts/t6462.nim b/tests/concepts/t6462.nim new file mode 100644 index 0000000000..2fa2268f83 --- /dev/null +++ b/tests/concepts/t6462.nim @@ -0,0 +1,23 @@ +discard """ + output: "true" +""" + +import future + +type + FilterMixin*[T] = ref object + test*: (T) -> bool + trans*: (T) -> T + + SeqGen*[T] = ref object + fil*: FilterMixin[T] + + WithFilter[T] = concept a + a.fil is FilterMixin[T] + +proc test*[T](a: WithFilter[T]): (T) -> bool = + a.fil.test + +var s = SeqGen[int](fil: FilterMixin[int](test: nil, trans: nil)) +echo s.test() == nil + From 63403b31ad2f6dcbfcfa95251e8200bb4603dad7 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Thu, 21 Dec 2017 13:15:19 +0300 Subject: [PATCH 078/200] Modified behavior of walkDirRec (#6952) --- lib/pure/os.nim | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 87f6def292..c18d032891 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -816,32 +816,40 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: k = getSymlinkFileKind(y) yield (k, y) -iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. - tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file in `dir`. - ## The full path for each file is returned. Directories are not returned. +iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, + followFilter = {pcDir}): string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## The full path for each file or directory is returned. ## **Warning**: ## Modifying the directory structure while the iterator ## is traversing may result in undefined behavior! ## - ## Walking is recursive. `filter` controls the behaviour of the iterator: + ## Walking is recursive. `filters` controls the behaviour of the iterator: ## ## --------------------- --------------------------------------------- - ## filter meaning + ## yieldFilter meaning ## --------------------- --------------------------------------------- ## ``pcFile`` yield real files ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## --------------------- --------------------------------------------- + ## + ## --------------------- --------------------------------------------- + ## followFilter meaning + ## --------------------- --------------------------------------------- ## ``pcDir`` follow real directories ## ``pcLinkToDir`` follow symbolic links to directories ## --------------------- --------------------------------------------- ## var stack = @[dir] while stack.len > 0: - for k,p in walkDir(stack.pop()): - if k in filter: - case k - of pcFile, pcLinkToFile: yield p - of pcDir, pcLinkToDir: stack.add(p) + for k, p in walkDir(stack.pop()): + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add(p) + if k in yieldFilter: + yield p proc rawRemoveDir(dir: string) = when defined(windows): From 3495c0a46ddb8c2763df7a2e57d91be4a7717eb8 Mon Sep 17 00:00:00 2001 From: konqoro Date: Thu, 21 Dec 2017 12:26:02 +0200 Subject: [PATCH 079/200] Fix json generation logic (#6909) --- compiler/extccomp.nim | 54 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 7a473ea434..5299b2dbf6 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -794,42 +794,40 @@ proc writeJsonBuildInstructions*(projectfile: string) = else: f.write escapeJson(x) - proc cfiles(f: File; buf: var string; list: CfileList, isExternal: bool) = - var i = 0 - for it in list: + proc cfiles(f: File; buf: var string; clist: CfileList, isExternal: bool) = + var pastStart = false + for it in clist: if CfileFlag.Cached in it.flags: continue let compileCmd = getCompileCFileCmd(it) + if pastStart: lit "],\L" lit "[" str it.cname lit ", " str compileCmd - inc i - if i == list.len: - lit "]\L" - else: - lit "],\L" + pastStart = true + lit "]\L" - proc linkfiles(f: File; buf, objfiles: var string) = - for i, it in externalToLink: - let - objFile = if noAbsolutePaths(): it.extractFilename else: it - objStr = addFileExt(objFile, CC[cCompiler].objExt) + proc linkfiles(f: File; buf, objfiles: var string; clist: CfileList; + llist: seq[string]) = + var pastStart = false + for it in llist: + let objfile = if noAbsolutePaths(): it.extractFilename + else: it + let objstr = addFileExt(objfile, CC[cCompiler].objExt) add(objfiles, ' ') - add(objfiles, objStr) - str objStr - if toCompile.len == 0 and i == externalToLink.high: - lit "\L" - else: - lit ",\L" - for i, x in toCompile: - let objStr = quoteShell(x.obj) + add(objfiles, objstr) + if pastStart: lit ",\L" + str objstr + pastStart = true + + for it in clist: + let objstr = quoteShell(it.obj) add(objfiles, ' ') - add(objfiles, objStr) - str objStr - if i == toCompile.high: - lit "\L" - else: - lit ",\L" + add(objfiles, objstr) + if pastStart: lit ",\L" + str objstr + pastStart = true + lit "\L" var buf = newStringOfCap(50) @@ -843,7 +841,7 @@ proc writeJsonBuildInstructions*(projectfile: string) = lit "],\L\"link\":[\L" var objfiles = "" # XXX add every file here that is to link - linkfiles(f, buf, objfiles) + linkfiles(f, buf, objfiles, toCompile, externalToLink) lit "],\L\"linkcmd\": " str getLinkCmd(projectfile, objfiles) From 0181253eea4ed5a47c37140c0533bc62d7f09e68 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 21 Dec 2017 12:28:05 +0100 Subject: [PATCH 080/200] fixes #6949 --- compiler/nimblecmd.nim | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim index 39c3a17e75..0f9e03352b 100644 --- a/compiler/nimblecmd.nim +++ b/compiler/nimblecmd.nim @@ -28,6 +28,10 @@ proc newVersion*(ver: string): Version = proc isSpecial(ver: Version): bool = return ($ver).len > 0 and ($ver)[0] == '#' +proc isValidVersion(v: string): bool = + if v.len > 0: + if v[0] in {'#'} + Digits: return true + proc `<`*(ver: Version, ver2: Version): bool = ## This is synced from Nimble's version module. @@ -72,15 +76,23 @@ proc getPathVersion*(p: string): tuple[name, version: string] = result.name = p return + for i in sepIdx.. Date: Thu, 21 Dec 2017 12:34:16 +0100 Subject: [PATCH 081/200] move securehash back into the stdlib --- changelog.md | 2 -- doc/lib.rst | 3 +++ {compiler => lib/pure}/securehash.nim | 0 tools/niminst/niminst.nim | 2 +- web/website.ini | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename {compiler => lib/pure}/securehash.nim (100%) diff --git a/changelog.md b/changelog.md index d519ecfcf6..4d205faf89 100644 --- a/changelog.md +++ b/changelog.md @@ -127,8 +127,6 @@ This now needs to be written as: - The behavior of ``$`` has been changed for all standard library collections. The collection-to-string implementations now perform proper quoting and escaping of strings and chars. -- Removed ``securehash`` stdlib module as it is not secure anymore. The module - is still available via ``compiler/securehash``. - The ``random`` procs in ``random.nim`` have all been deprecated. Instead use the new ``rand`` procs. The module now exports the state of the random number generator as type ``Rand`` so multiple threads can easily use their diff --git a/doc/lib.rst b/doc/lib.rst index 2719472fef..58dedc49c3 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -380,6 +380,9 @@ Cryptography and Hashing * `base64 `_ This module implements a base64 encoder and decoder. +* `securehash `_ + This module implements a sha1 encoder and decoder. + Multimedia support ------------------ diff --git a/compiler/securehash.nim b/lib/pure/securehash.nim similarity index 100% rename from compiler/securehash.nim rename to lib/pure/securehash.nim diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index ab0ce6a5bb..9c15326b0a 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -15,7 +15,7 @@ when haveZipLib: import os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - "../../compiler/securehash" + securehash const maxOS = 20 # max number of OSes diff --git a/web/website.ini b/web/website.ini index d8deb2d70e..32b1936d56 100644 --- a/web/website.ini +++ b/web/website.ini @@ -64,7 +64,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;impure/rdstdin;pure/strformat" +srcdoc2: "pure/fenv;pure/securehash;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile;js/asyncjs" From b22d9e4339f06e6af02f8da50e68a98707987612 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 21 Dec 2017 16:45:42 +0200 Subject: [PATCH 082/200] Fix #6906 --- lib/pure/asyncdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 675e8fc5eb..23eb80b37b 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1234,7 +1234,7 @@ else: processBasicCallbacks(fd, writeList) result = true - if Event.User in events or events == {Event.Error}: + if Event.User in events: processBasicCallbacks(fd, readList) custom = true if rLength == 0: From a89b81eb9679d1351aceae766e5eaf1bf8faeffe Mon Sep 17 00:00:00 2001 From: skilchen Date: Thu, 21 Dec 2017 16:32:26 +0100 Subject: [PATCH 083/200] fixes #6353 (#6951) --- lib/pure/math.nim | 17 ++++++++++++---- tests/stdlib/tfrexp1.nim | 44 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 tests/stdlib/tfrexp1.nim diff --git a/lib/pure/math.nim b/lib/pure/math.nim index a9dabfa486..cbd04a145b 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -351,15 +351,19 @@ proc round*[T: float32|float64](x: T, places: int = 0): T = result = round0(x*mult)/mult when not defined(JS): - proc frexp*(x: float32, exponent: var int): float32 {. + proc c_frexp*(x: float32, exponent: var int32): float32 {. importc: "frexp", header: "".} - proc frexp*(x: float64, exponent: var int): float64 {. + proc c_frexp*(x: float64, exponent: var int32): float64 {. importc: "frexp", header: "".} + proc frexp*[T, U](x: T, exponent: var U): T = ## Split a number into mantissa and exponent. ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 ## and less than 1) and the integer value n such that `x` (the original ## float value) equals m * 2**n. frexp stores n in `exponent` and returns ## m. + var exp: int32 + result = c_frexp(x, exp) + exponent = exp else: proc frexp*[T: float32|float64](x: T, exponent: var int): T = if x == 0.0: @@ -368,9 +372,14 @@ else: elif x < 0.0: result = -frexp(-x, exponent) else: - var ex = floor(log2(x)) - exponent = round(ex) + var ex = trunc(log2(x)) + exponent = int(ex) result = x / pow(2.0, ex) + if abs(result) >= 1: + inc(exponent) + result = result / 2 + if exponent == 1024 and result == 0.0: + result = 0.99999999999999988898 proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = ## Breaks `x` into an integral and a fractional part. diff --git a/tests/stdlib/tfrexp1.nim b/tests/stdlib/tfrexp1.nim new file mode 100644 index 0000000000..c6bb2b38cc --- /dev/null +++ b/tests/stdlib/tfrexp1.nim @@ -0,0 +1,44 @@ +discard """ + targets: "js c c++" + output: '''ok''' +""" + +import math +import strformat + +const manualTest = false + +proc frexp_test(lo, hi, step: float64) = + var exp: int + var frac: float64 + + var eps = 1e-15.float64 + + var x:float64 = lo + while x <= hi: + frac = frexp(x.float, exp) + let rslt = pow(2.0, float(exp)) * frac + + doAssert(abs(rslt - x) < eps) + + when manualTest: + echo fmt("x: {x:10.3f} exp: {exp:4d} frac: {frac:24.20f} check: {$(abs(rslt - x) < eps):-5s} {rslt: 9.3f}") + x += step + +when manualTest: + var exp: int + var frac: float64 + + for flval in [1.7976931348623157e+308, -1.7976931348623157e+308, # max, min float64 + 3.4028234663852886e+38, -3.4028234663852886e+38, # max, min float32 + 4.9406564584124654e-324, -4.9406564584124654e-324, # smallest/largest positive/negative float64 + 1.4012984643248171e-45, -1.4012984643248171e-45, # smallest/largest positive/negative float32 + 2.2250738585072014e-308, 1.1754943508222875e-38]: # smallest normal float64/float32 + frac = frexp(flval, exp) + echo fmt("{flval:25.16e}, {exp: 6d}, {frac: .20f} {frac * pow(2.0, float(exp)): .20e}") + + frexp_test(-1000.0, 1000.0, 0.0125) +else: + frexp_test(-1000000.0, 1000000.0, 0.125) + +echo "ok" From da90657317e8a57bae80ebd2d637a972d3b438ab Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Dec 2017 17:14:31 +0100 Subject: [PATCH 084/200] make the new --genDeps feature optional since it makes compilations slower --- compiler/commands.nim | 2 +- compiler/main.nim | 3 ++- doc/advopt.txt | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index de474c6e68..386d7bda86 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -611,7 +611,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "skipparentcfg": expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optSkipParentConfigFiles) - of "genscript": + of "genscript", "gendeps": expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optGenScript) of "colors": processOnOffSwitchG({optUseColors}, arg, pass, info) diff --git a/compiler/main.nim b/compiler/main.nim index db03f0e4db..08fc4b138a 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -80,7 +80,8 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = let proj = changeFileExt(gProjectFull, "") extccomp.callCCompiler(proj) extccomp.writeJsonBuildInstructions(proj) - writeDepsFile(graph, toGeneratedFile(proj, "")) + if optGenScript in gGlobalOptions: + writeDepsFile(graph, toGeneratedFile(proj, "")) proc commandJsonScript(graph: ModuleGraph; cache: IdentCache) = let proj = changeFileExt(gProjectFull, "") diff --git a/doc/advopt.txt b/doc/advopt.txt index ab10d65ba7..a1210118e3 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -37,6 +37,7 @@ Advanced options: --noMain do not generate a main procedure --genScript generate a compile script (in the 'nimcache' subdirectory named 'compile_$project$scriptext') + --genDeps generate a '.deps' file containing the dependencies --os:SYMBOL set the target operating system (cross-compilation) --cpu:SYMBOL set the target processor (cross-compilation) --debuginfo enables debug information From c2d91771bc1593fc8432392a75223dc1106bcfa3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Dec 2017 19:05:23 +0100 Subject: [PATCH 085/200] DFA works for simple examples --- compiler/dfa.nim | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 66a71e8391..6bb7a03a9e 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -344,20 +344,20 @@ proc dfa(code: seq[Instr]) = case code[i].kind of use, useWithinCall: u[i].incl(code[i].sym.id) of def: d[i].incl(code[i].sym.id) - of fork: + of fork, goto: let d = i+code[i].dest backrefs.add(d, i) - of goto: discard var w = @[0] var maxIters = 50 var someChange = true - while w.len > 0 and maxIters > 0 and someChange: + var takenGotos = initIntSet() + while w.len > 0 and maxIters > 0: # and someChange: dec maxIters var pc = w.pop() # w[^1] var prevPc = -1 # this simulates a single linear control flow execution: - while pc < code.len and someChange: + while pc < code.len: # according to the paper, it is better to shrink the working set here # in this inner loop: #let widx = w.find(pc) @@ -386,17 +386,21 @@ proc dfa(code: seq[Instr]) = if def notin d[prevPc]: excl(intersect, def) someChange = true + when defined(debugDfa): + echo "Excluding ", pc, " prev ", prevPc assign d[pc], intersect # our interpretation ![I!]: prevPc = pc + when defined(debugDfa): + echo "looking at ", pc case code[pc].kind of goto: # we must leave endless loops eventually: - #if someChange: - pc = pc + code[pc].dest - #else: - # inc pc + if not takenGotos.containsOrIncl(pc) or someChange: + pc = pc + code[pc].dest + else: + inc pc of fork: # we follow the next instruction but push the dest onto our "work" stack: #if someChange: @@ -405,6 +409,10 @@ proc dfa(code: seq[Instr]) = of use, useWithinCall, def: inc pc + when defined(useDfa) and defined(debugDfa): + for i in 0.. Date: Sat, 23 Dec 2017 14:08:47 +0000 Subject: [PATCH 086/200] Add link to #6934 in changelog.md --- changelog.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 4d205faf89..efdd1f5207 100644 --- a/changelog.md +++ b/changelog.md @@ -102,10 +102,12 @@ This now needs to be written as: - Nim's ``rst2html`` command now supports the testing of code snippets via an RST extension that we called ``:test:``:: + ```rst .. code-block:: nim :test: # shows how the 'if' statement works if true: echo "yes" + ``` - The ``[]`` proc for strings now raises an ``IndexError`` exception when the specified slice is out of bounds. See issue [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details. @@ -130,7 +132,8 @@ This now needs to be written as: - The ``random`` procs in ``random.nim`` have all been deprecated. Instead use the new ``rand`` procs. The module now exports the state of the random number generator as type ``Rand`` so multiple threads can easily use their - own random number generators that do not require locking. + own random number generators that do not require locking. For more information + about this rename see issue [#6934](https://github.com/nim-lang/Nim/issues/6934) - The compiler is now more consistent in its treatment of ambiguous symbols: Types that shadow procs and vice versa are marked as ambiguous (bug #6693). - ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably From 8e7829ff8266859f313b3dd7e192dcd1560b8d5f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Dec 2017 19:50:15 +0100 Subject: [PATCH 087/200] DFA attempt to capture the essence of linear types --- compiler/dfa.nim | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 6bb7a03a9e..2e80a1b263 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -337,10 +337,12 @@ proc gen(c: var Con; n: PNode) = proc dfa(code: seq[Instr]) = var u = newSeq[IntSet](code.len) # usages var d = newSeq[IntSet](code.len) # defs + var c = newSeq[IntSet](code.len) # consumed var backrefs = initTable[int, int]() for i in 0.. 0 and maxIters > 0: # and someChange: dec maxIters var pc = w.pop() # w[^1] @@ -389,6 +392,10 @@ proc dfa(code: seq[Instr]) = when defined(debugDfa): echo "Excluding ", pc, " prev ", prevPc assign d[pc], intersect + if consuming >= 0: + if not c[pc].containsOrIncl(consuming): + someChange = true + consuming = -1 # our interpretation ![I!]: prevPc = pc @@ -406,12 +413,21 @@ proc dfa(code: seq[Instr]) = #if someChange: w.add pc + code[pc].dest inc pc - of use, useWithinCall, def: + of use, useWithinCall: + #if not d[prevPc].missingOrExcl(): + # someChange = true + consuming = code[pc].sym.id + when defined(debugDfa): + echo "consumed: ", consuming + inc pc + of def: + if not d[pc].containsOrIncl(code[pc].sym.id): + someChange = true inc pc when defined(useDfa) and defined(debugDfa): for i in 0.. Date: Sun, 24 Dec 2017 09:23:17 -0500 Subject: [PATCH 088/200] cmp(x, y: string) now uses memcmp rather than strcmp (#6869) (#6968) --- lib/system.nim | 5 ++++- lib/system/sysstr.nim | 8 ++++---- tests/stdlib/tstring.nim | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 83e87683ab..85643891ba 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2916,7 +2916,10 @@ when not defined(JS): #and not defined(nimscript): elif x > y: result = 1 else: result = 0 else: - result = int(c_strcmp(x, y)) + let minlen = min(x.len, y.len) + result = int(c_memcmp(x.cstring, y.cstring, minlen.csize)) + if result == 0: + result = x.len - y.len when defined(nimscript): proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 56b8ade97a..4c5f3d9a15 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -24,10 +24,10 @@ proc cmpStrings(a, b: NimString): int {.inline, compilerProc.} = if a == b: return 0 if a == nil: return -1 if b == nil: return 1 - when defined(nimNoArrayToCstringConversion): - return c_strcmp(addr a.data, addr b.data) - else: - return c_strcmp(a.data, b.data) + let minlen = min(a.len, b.len) + result = c_memcmp(addr a.data, addr b.data, minlen.csize) + if result == 0: + result = a.len - b.len proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} = if a == b: return true diff --git a/tests/stdlib/tstring.nim b/tests/stdlib/tstring.nim index 904bc462a2..6607461500 100644 --- a/tests/stdlib/tstring.nim +++ b/tests/stdlib/tstring.nim @@ -56,4 +56,24 @@ proc test_string_slice() = echo("OK") +proc test_string_cmp() = + let world = "hello\0world" + let earth = "hello\0earth" + let short = "hello\0" + let hello = "hello" + let goodbye = "goodbye" + + doAssert world == world + doAssert world != earth + doAssert world != short + doAssert world != hello + doAssert world != goodbye + + doAssert cmp(world, world) == 0 + doAssert cmp(world, earth) > 0 + doAssert cmp(world, short) > 0 + doAssert cmp(world, hello) > 0 + doAssert cmp(world, goodbye) > 0 + test_string_slice() +test_string_cmp() From 2b3ec0a7c66d2246371ed51348aaa87d4c3cf0f9 Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 25 Dec 2017 00:22:03 +0300 Subject: [PATCH 089/200] Implement language feature #6885 (#6954) --- changelog.md | 22 ++++++++++++++ compiler/msgs.nim | 6 ++-- compiler/pragmas.nim | 2 ++ compiler/sem.nim | 13 ++++++++ compiler/semstmts.nim | 24 ++++++++------- lib/system/chcks.nim | 1 - tests/casestmt/tcasestm.nim | 59 +++++++++++++++++++++++++++++++++++++ tests/pragmas/tnoreturn.nim | 18 +++++++++++ 8 files changed, 131 insertions(+), 14 deletions(-) create mode 100644 tests/pragmas/tnoreturn.nim diff --git a/changelog.md b/changelog.md index efdd1f5207..5734a4cb12 100644 --- a/changelog.md +++ b/changelog.md @@ -144,3 +144,25 @@ This now needs to be written as: - codegenDecl pragma now works for the JavaScript backend. It returns an empty string for function return type placeholders. - Asynchronous programming for the JavaScript backend using the `asyncjs` module. +- Extra semantic checks for procs with noreturn pragma: return type is not allowed, + statements after call to noreturn procs are no longer allowed. +- Noreturn proc calls and raising exceptions branches are now skipped during common type + deduction in if and case expressions. The following code snippets now compile: +```nim +import strutils +let str = "Y" +let a = case str: + of "Y": true + of "N": false + else: raise newException(ValueError, "Invalid boolean") +let b = case str: + of nil, "": raise newException(ValueError, "Invalid boolean") + elif str.startsWith("Y"): true + elif str.startsWith("N"): false + else: false +let c = if str == "Y": true + elif str == "N": false + else: + echo "invalid bool" + quit("this is the end") +``` diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 2668c72ae0..4e6226122e 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -26,7 +26,8 @@ type errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, errExceptionExpected, errExceptionAlreadyHandled, errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, + errInvalidNumberOfYieldExpr, errCannotReturnExpr, + errNoReturnWithReturnTypeNotAllowed, errAttemptToRedefine, errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler, @@ -179,8 +180,9 @@ const errYieldNotAllowedInTryStmt: "'yield' cannot be used within 'try' in a non-inlined iterator", errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expressions", errCannotReturnExpr: "current routine cannot return an expression", + errNoReturnWithReturnTypeNotAllowed: "routines with NoReturn pragma are not allowed to have return type", errAttemptToRedefine: "redefinition of \'$1\'", - errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\' or \'continue'", + errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\', \'continue\' or proc call with noreturn pragma", errStmtExpected: "statement expected", errInvalidLabel: "\'$1\' is no label", errInvalidCmdLineOption: "invalid command line option: \'$1\'", diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index b598cadb20..35fedf4ea1 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -771,6 +771,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wNoreturn: noVal(it) incl(sym.flags, sfNoReturn) + if sym.ast[paramsPos][0].kind != nkEmpty: + localError(sym.ast[paramsPos][0].info, errNoReturnWithReturnTypeNotAllowed) of wDynlib: processDynLib(c, it, sym) of wCompilerproc: diff --git a/compiler/sem.nim b/compiler/sem.nim index bc994201d1..ababbd303c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -165,6 +165,19 @@ proc commonType*(x, y: PType): PType = result = newType(k, r.owner) result.addSonSkipIntLit(r) +proc endsInNoReturn(n: PNode): bool = + # check if expr ends in raise exception or call of noreturn proc + var it = n + while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: + it = it.lastSon + result = it.kind == nkRaiseStmt or + it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags + +proc commonType*(x: PType, y: PNode): PType = + # ignore exception raising branches in case/if expressions + if endsInNoReturn(y): return x + commonType(x, y.typ) + proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = result = newSym(kind, considerQuotedIdent(n), getCurrOwner(c), n.info) when defined(nimsuggest): diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 8ed120c98f..b1fa8c19b8 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -165,14 +165,14 @@ proc semIf(c: PContext, n: PNode): PNode = it.sons[0] = forceBool(c, semExprWithType(c, it.sons[0])) when not newScopeForIf: openScope(c) it.sons[1] = semExprBranch(c, it.sons[1]) - typ = commonType(typ, it.sons[1].typ) + typ = commonType(typ, it.sons[1]) closeScope(c) elif it.len == 1: hasElse = true it.sons[0] = semExprBranchScope(c, it.sons[0]) - typ = commonType(typ, it.sons[0].typ) + typ = commonType(typ, it.sons[0]) else: illFormedAst(it) - if isEmptyType(typ) or typ.kind == tyNil or not hasElse: + if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse: for it in n: discardCheck(c, it.lastSon) result.kind = nkIfStmt # propagate any enforced VoidContext: @@ -180,7 +180,8 @@ proc semIf(c: PContext, n: PNode): PNode = else: for it in n: let j = it.len-1 - it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) + if not endsInNoReturn(it.sons[j]): + it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) result.kind = nkIfExpr result.typ = typ @@ -213,7 +214,7 @@ proc semCase(c: PContext, n: PNode): PNode = semCaseBranch(c, n, x, i, covered) var last = sonsLen(x)-1 x.sons[last] = semExprBranchScope(c, x.sons[last]) - typ = commonType(typ, x.sons[last].typ) + typ = commonType(typ, x.sons[last]) of nkElifBranch: chckCovered = false checkSonsLen(x, 2) @@ -221,13 +222,13 @@ proc semCase(c: PContext, n: PNode): PNode = x.sons[0] = forceBool(c, semExprWithType(c, x.sons[0])) when not newScopeForIf: openScope(c) x.sons[1] = semExprBranch(c, x.sons[1]) - typ = commonType(typ, x.sons[1].typ) + typ = commonType(typ, x.sons[1]) closeScope(c) of nkElse: chckCovered = false checkSonsLen(x, 1) x.sons[0] = semExprBranchScope(c, x.sons[0]) - typ = commonType(typ, x.sons[0].typ) + typ = commonType(typ, x.sons[0]) hasElse = true else: illFormedAst(x) @@ -237,7 +238,7 @@ proc semCase(c: PContext, n: PNode): PNode = else: localError(n.info, errNotAllCasesCovered) closeScope(c) - if isEmptyType(typ) or typ.kind == tyNil or not hasElse: + if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse: for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon) # propagate any enforced VoidContext: if typ == enforceVoidContext: @@ -246,7 +247,8 @@ proc semCase(c: PContext, n: PNode): PNode = for i in 1..n.len-1: var it = n.sons[i] let j = it.len-1 - it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) + if not endsInNoReturn(it.sons[j]): + it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) result.typ = typ proc semTry(c: PContext, n: PNode): PNode = @@ -1851,8 +1853,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = else: n.typ = n.sons[i].typ if not isEmptyType(n.typ): n.kind = nkStmtListExpr - case n.sons[i].kind - of LastBlockStmts: + if n.sons[i].kind in LastBlockStmts or + n.sons[i].kind in nkCallKinds and n.sons[i][0].kind == nkSym and sfNoReturn in n.sons[i][0].sym.flags: for j in countup(i + 1, length - 1): case n.sons[j].kind of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkBlockExpr, diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 1520f231e1..69b680dbdc 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -63,7 +63,6 @@ proc chckObj(obj, subclass: PNimType) {.compilerproc.} = while x != subclass: if x == nil: sysFatal(ObjectConversionError, "invalid object conversion") - break x = x.base proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} = diff --git a/tests/casestmt/tcasestm.nim b/tests/casestmt/tcasestm.nim index 7ac20bf2f6..b005d8120e 100644 --- a/tests/casestmt/tcasestm.nim +++ b/tests/casestmt/tcasestm.nim @@ -36,5 +36,64 @@ var z = case i echo z #OUT ayyy +let str1 = "Y" +let str2 = "NN" +let a = case str1: + of "Y": true + of "N": false + else: + echo "no good" + quit("quiting") + +let b = case str2: + of nil, "": raise newException(ValueError, "Invalid boolean") + elif str2[0] == 'Y': true + elif str2[0] == 'N': false + else: "error".quit(2) + +doAssert(a == true) +doAssert(b == false) + +var bb: bool +doassert(not compiles( + bb = case str2: + of nil, "": raise newException(ValueError, "Invalid boolean") + elif str.startsWith("Y"): true + elif str.startsWith("N"): false +)) + +doassert(not compiles( + bb = case str2: + of "Y": true + of "N": false +)) + +doassert(not compiles( + bb = case str2: + of "Y": true + of "N": raise newException(ValueError, "N not allowed") +)) + +doassert(not compiles( + bb = case str2: + of "Y": raise newException(ValueError, "Invalid Y") + else: raise newException(ValueError, "Invalid N") +)) +doassert(not compiles( + bb = case str2: + of "Y": + raise newException(ValueError, "Invalid Y") + true + else: raise newException(ValueError, "Invalid") +)) + + +doassert(not compiles( + bb = case str2: + of "Y": + "invalid Y".quit(3) + true + else: raise newException(ValueError, "Invalid") +)) \ No newline at end of file diff --git a/tests/pragmas/tnoreturn.nim b/tests/pragmas/tnoreturn.nim new file mode 100644 index 0000000000..2075b352e6 --- /dev/null +++ b/tests/pragmas/tnoreturn.nim @@ -0,0 +1,18 @@ +discard """ +ccodeCheck: "\\i @'__attribute__((noreturn))' .*" +""" + +proc noret1*(i: int) {.noreturn.} = + echo i + +var p {.used.}: proc(i: int): int +doAssert(not compiles( + p = proc(i: int): int {.noreturn.} = i # noreturn lambda returns int +)) + + +doAssert(not compiles( + block: + noret1(5) + echo 1 # statement after noreturn +)) From 53cf0b2c24e5adc4fa99e49ddf1834991d663846 Mon Sep 17 00:00:00 2001 From: cooldome Date: Wed, 27 Dec 2017 12:09:24 +0300 Subject: [PATCH 090/200] Allow noreturn procs with void type (#6973) --- compiler/pragmas.nim | 2 +- tests/pragmas/tnoreturn.nim | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 35fedf4ea1..02b57d5a30 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -771,7 +771,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wNoreturn: noVal(it) incl(sym.flags, sfNoReturn) - if sym.ast[paramsPos][0].kind != nkEmpty: + if sym.typ[0] != nil: localError(sym.ast[paramsPos][0].info, errNoReturnWithReturnTypeNotAllowed) of wDynlib: processDynLib(c, it, sym) diff --git a/tests/pragmas/tnoreturn.nim b/tests/pragmas/tnoreturn.nim index 2075b352e6..4d00c60346 100644 --- a/tests/pragmas/tnoreturn.nim +++ b/tests/pragmas/tnoreturn.nim @@ -5,6 +5,10 @@ ccodeCheck: "\\i @'__attribute__((noreturn))' .*" proc noret1*(i: int) {.noreturn.} = echo i + +proc noret2*(i: int): void {.noreturn.} = + echo i + var p {.used.}: proc(i: int): int doAssert(not compiles( p = proc(i: int): int {.noreturn.} = i # noreturn lambda returns int From b103b4d3f2c8ac16e8f30f72cf733ace21bbc3a6 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 27 Dec 2017 10:23:57 +0100 Subject: [PATCH 091/200] manual: clarify the rules for integer literals --- doc/manual/types.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/types.txt b/doc/manual/types.txt index 56584f2570..1477995dda 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -41,7 +41,8 @@ These integer types are pre-defined: ``int`` the generic signed integer type; its size is platform dependent and has the same size as a pointer. This type should be used in general. An integer - literal that has no type suffix is of this type. + literal that has no type suffix is of this type if it is in the range + ``low(int32)..high(int32)`` otherwise the literal's type is ``int64``. intXX additional signed integer types of XX bits use this naming scheme From 5c08092b88c0b6399c52804fd6f2c1fc92c58a86 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 27 Dec 2017 10:25:45 +0100 Subject: [PATCH 092/200] minor todo.txt update --- todo.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/todo.txt b/todo.txt index 1fae1ddacb..e06ddf5554 100644 --- a/todo.txt +++ b/todo.txt @@ -1,8 +1,6 @@ version 1.0 battle plan ======================= -- introduce ``nkStmtListExpr`` for template/macro invokations to produce - better stack traces - let 'doAssert' analyse the expressions and produce more helpful output - fix "high priority" bugs - try to fix as many compiler crashes as reasonable @@ -11,6 +9,8 @@ version 1.0 battle plan Not critical for 1.0 ==================== +- introduce ``nkStmtListExpr`` for template/macro invokations to produce + better stack traces - make 'break' not leave named blocks - make FlowVar compatible to Futures - make 'not nil' the default (produce warnings instead of errors for From c36d7ffc7c771fcece05bf882fee02fae4261edf Mon Sep 17 00:00:00 2001 From: Konstantin Molchanov Date: Wed, 27 Dec 2017 13:30:32 +0400 Subject: [PATCH 093/200] Tables: make `toCountTable` actually count the elements of the input openArray. --- lib/pure/collections/tables.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 01767956ea..d8c133bce3 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -968,7 +968,7 @@ proc initCountTable*[A](initialSize=64): CountTable[A] = proc toCountTable*[A](keys: openArray[A]): CountTable[A] = ## creates a new count table with every key in `keys` having a count of 1. result = initCountTable[A](rightSize(keys.len)) - for key in items(keys): result[key] = 1 + for key in items(keys): result.inc key proc `$`*[A](t: CountTable[A]): string = ## The `$` operator for count tables. From b592f069bbc2d30f11fd9414e149025653425dc0 Mon Sep 17 00:00:00 2001 From: Konstantin Molchanov Date: Wed, 27 Dec 2017 13:44:47 +0400 Subject: [PATCH 094/200] Tables: toCountTable: Update docs. --- lib/pure/collections/tables.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index d8c133bce3..777beabc39 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -966,7 +966,8 @@ proc initCountTable*[A](initialSize=64): CountTable[A] = newSeq(result.data, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = - ## creates a new count table with every key in `keys` having a count of 1. + ## creates a new count table with every key in `keys` having a count + ## of how many times it occurs in `keys`. result = initCountTable[A](rightSize(keys.len)) for key in items(keys): result.inc key From a74dfcfd00365e19757092e95933107d9e5adb7d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 27 Dec 2017 12:22:47 +0100 Subject: [PATCH 095/200] DFA: code cleanups and some support for consuming operations --- compiler/dfa.nim | 118 ++++------------------------------------------- 1 file changed, 9 insertions(+), 109 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 2e80a1b263..b648995f42 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -361,11 +361,6 @@ proc dfa(code: seq[Instr]) = var prevPc = -1 # this simulates a single linear control flow execution: while pc < code.len: - # according to the paper, it is better to shrink the working set here - # in this inner loop: - #let widx = w.find(pc) - #if widx >= 0: w.del(widx) - if prevPc >= 0: someChange = false # merge step and test for changes (we compute the fixpoints here): @@ -392,15 +387,13 @@ proc dfa(code: seq[Instr]) = when defined(debugDfa): echo "Excluding ", pc, " prev ", prevPc assign d[pc], intersect - if consuming >= 0: - if not c[pc].containsOrIncl(consuming): - someChange = true - consuming = -1 + if consuming >= 0: + if not c[pc].containsOrIncl(consuming): + someChange = true + consuming = -1 # our interpretation ![I!]: prevPc = pc - when defined(debugDfa): - echo "looking at ", pc case code[pc].kind of goto: # we must leave endless loops eventually: @@ -417,8 +410,6 @@ proc dfa(code: seq[Instr]) = #if not d[prevPc].missingOrExcl(): # someChange = true consuming = code[pc].sym.id - when defined(debugDfa): - echo "consumed: ", consuming inc pc of def: if not d[pc].containsOrIncl(code[pc].sym.id): @@ -433,105 +424,14 @@ proc dfa(code: seq[Instr]) = for i in 0.. 0: - var pc = w[^1] - # this simulates a single linear control flow execution: - while true: - # according to the paper, it is better to shrink the working set here - # in this inner loop: - let widx = w.find(pc) - if widx >= 0: w.del(widx) - # our interpretation ![I!]: - var sid = -1 - case code[pc].kind - of goto, fork: discard - of use, useWithinCall: - let sym = code[pc].sym - if s[pc].contains(sym.id): - localError(code[pc].n.info, "variable read before initialized: " & sym.name.s) - of def: - sid = code[pc].sym.id - - var pc2: int - if code[pc].kind == goto: - pc2 = pc + code[pc].dest - else: - pc2 = pc + 1 - if code[pc].kind == fork: - let lidx = pc + code[pc].dest - if sid >= 0 and s[lidx].missingOrExcl(sid): - w.add lidx - - if sid >= 0 and s[pc2].missingOrExcl(sid): - pc = pc2 - else: - break - if pc >= code.len: break - - when false: - case code[pc].kind - of use: - let s = code[pc].sym - if undefB.contains(s.id): - localError(code[pc].n.info, "variable read before initialized: " & s.name.s) - break - inc pc - of def: - let s = code[pc].sym - # exclude 'undef' for s for this path through the graph. - if not undefB.missingOrExcl(s.id): - inc pc - else: - break - #undefB.excl s.id - #inc pc - when false: - let prev = bindings.getOrDefault(s.id) - if prev != value: - # well now it has a value and we made progress, so - bindings[s.id] = value - inc pc - else: - break - of fork: - let diff = code[pc].dest - # we follow pc + 1 and remember the label for later: - w.add pc+diff - inc pc - of goto: - let diff = code[pc].dest - pc = pc + diff - if pc >= code.len: break - proc dataflowAnalysis*(s: PSym; body: PNode) = var c = Con(code: @[], blocks: @[]) gen(c, body) From 7b5448c755885d61b5213ad91ca95f630bbd6d28 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 27 Dec 2017 12:23:12 +0100 Subject: [PATCH 096/200] introduce 'core' as an alias for 'compilerproc' --- compiler/pragmas.nim | 10 +++++----- compiler/wordrecg.nim | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index b598cadb20..6f38def4fd 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -21,7 +21,7 @@ const const procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, - wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, + wCompilerProc, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe, @@ -31,7 +31,7 @@ const templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty, wDelegator, wExportNims, wUsed} macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc, - wNodecl, wMagic, wNosideeffect, wCompilerproc, wDeprecated, wExtern, + wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern, wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator, wExportNims, wUsed} iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect, @@ -52,14 +52,14 @@ const wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, wRaises, wLocks, wTags, wGcSafe} typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl, - wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow, + wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wExtern, wShallow, wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef, wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked, wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain, wPackage} fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern, wImportCpp, wImportObjC, wError, wGuard, wBitsize, wUsed} varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl, - wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern, + wMagic, wHeader, wDeprecated, wCompilerProc, wCore, wDynlib, wExtern, wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal, wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, @@ -773,7 +773,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, incl(sym.flags, sfNoReturn) of wDynlib: processDynLib(c, it, sym) - of wCompilerproc: + of wCompilerProc, wCore: noVal(it) # compilerproc may not get a string! cppDefine(c.graph.config, sym.name.s) if sfFromGeneric notin sym.flags: markCompilerProc(sym) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 8881acaddd..e458cad03f 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -45,7 +45,7 @@ type wImportc, wExportc, wExportNims, wIncompleteStruct, wRequiresInit, wAlign, wNodecl, wPure, wSideeffect, wHeader, wNosideeffect, wGcSafe, wNoreturn, wMerge, wLib, wDynlib, - wCompilerproc, wProcVar, wBase, wUsed, + wCompilerproc, wCore, wProcVar, wBase, wUsed, wFatal, wError, wWarning, wHint, wLine, wPush, wPop, wDefine, wUndef, wLinedir, wStacktrace, wLinetrace, wLink, wCompile, wLinksys, wDeprecated, wVarargs, wCallconv, wBreakpoint, wDebugger, @@ -131,7 +131,7 @@ const "incompletestruct", "requiresinit", "align", "nodecl", "pure", "sideeffect", "header", "nosideeffect", "gcsafe", "noreturn", "merge", "lib", "dynlib", - "compilerproc", "procvar", "base", "used", + "compilerproc", "core", "procvar", "base", "used", "fatal", "error", "warning", "hint", "line", "push", "pop", "define", "undef", "linedir", "stacktrace", "linetrace", "link", "compile", "linksys", "deprecated", "varargs", From a8b0a8a92da03593c3fe52dfbca8ee4eeed286d3 Mon Sep 17 00:00:00 2001 From: Konstantin Molchanov Date: Wed, 27 Dec 2017 17:29:39 +0400 Subject: [PATCH 097/200] Changelog: Document `toCountTable` behaviour change. --- changelog.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 5734a4cb12..de4b2f251b 100644 --- a/changelog.md +++ b/changelog.md @@ -146,7 +146,7 @@ This now needs to be written as: - Asynchronous programming for the JavaScript backend using the `asyncjs` module. - Extra semantic checks for procs with noreturn pragma: return type is not allowed, statements after call to noreturn procs are no longer allowed. -- Noreturn proc calls and raising exceptions branches are now skipped during common type +- Noreturn proc calls and raising exceptions branches are now skipped during common type deduction in if and case expressions. The following code snippets now compile: ```nim import strutils @@ -159,10 +159,29 @@ let b = case str: of nil, "": raise newException(ValueError, "Invalid boolean") elif str.startsWith("Y"): true elif str.startsWith("N"): false - else: false -let c = if str == "Y": true - elif str == "N": false + else: false +let c = if str == "Y": true + elif str == "N": false else: - echo "invalid bool" + echo "invalid bool" quit("this is the end") ``` +- Proc [toCountTable](https://nim-lang.org/docs/tables.html#toCountTable,openArray[A]) now produces a `CountTable` with values correspoding to the number of occurrences of the key in the input. It used to produce a table with all values set to `1`. + +Counting occurrences in a sequence used to be: + +```nim +let mySeq = @[1, 2, 1, 3, 1, 4] +var myCounter = initCountTable[int]() + +for item in mySeq: + myCounter.inc item +``` + +Now, you can simply do: + +```nim +let + mySeq = @[1, 2, 1, 3, 1, 4] + myCounter = mySeq.toCountTable() +``` From e49f18801c0891221ca76d3a94f5cafa2af40448 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 27 Dec 2017 19:35:57 +0300 Subject: [PATCH 098/200] Fixed compilation error (#6979) --- lib/pure/terminal.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 205aecb33b..ef5a95ed24 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -650,10 +650,10 @@ template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): template setCursorYPos(x: int) = setCursorYPos(stdout, x) -template cursorUp*(count=1) = cursorUp(stdout, f) -template cursorDown*(count=1) = cursorDown(stdout, f) -template cursorForward*(count=1) = cursorForward(stdout, f) -template cursorBackward*(count=1) = cursorBackward(stdout, f) +template cursorUp*(count=1) = cursorUp(stdout, count) +template cursorDown*(count=1) = cursorDown(stdout, count) +template cursorForward*(count=1) = cursorForward(stdout, count) +template cursorBackward*(count=1) = cursorBackward(stdout, count) template eraseLine*() = eraseLine(stdout) template eraseScreen*() = eraseScreen(stdout) template setStyle*(style: set[Style]) = From 0b0baece89883a2d91d9ebbff66538d974ee1fbc Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 27 Dec 2017 21:26:37 +0100 Subject: [PATCH 099/200] fixes #6980 --- compiler/semexprs.nim | 13 +++++++++++++ compiler/semstmts.nim | 2 +- tests/exprs/tstmtexprs.nim | 12 +++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 55c43ed09e..a0f519820b 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -780,6 +780,19 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode = proc semExprNoType(c: PContext, n: PNode): PNode = result = semExpr(c, n, {efWantStmt}) + # make an 'if' expression an 'if' statement again for backwards + # compatibility (.discardable was a bad idea!); bug #6980 + var isStmt = false + if result.kind == nkIfExpr: + isStmt = true + for condActionPair in result: + let action = condActionPair.lastSon + if not implicitlyDiscardable(action) and not + endsInNoReturn(action): + isStmt = false + if isStmt: + result.kind = nkIfStmt + result.typ = nil discardCheck(c, result) proc isTypeExpr(n: PNode): bool = diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index b1fa8c19b8..dcaa0263b0 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -732,7 +732,7 @@ proc semRaise(c: PContext, n: PNode): PNode = var typ = n.sons[0].typ if typ.kind != tyRef or typ.lastSon.kind != tyObject: localError(n.info, errExprCannotBeRaised) - + # check if the given object inherits from Exception var base = typ.lastSon while true: diff --git a/tests/exprs/tstmtexprs.nim b/tests/exprs/tstmtexprs.nim index 9283f72682..2a0ec2821f 100644 --- a/tests/exprs/tstmtexprs.nim +++ b/tests/exprs/tstmtexprs.nim @@ -140,4 +140,14 @@ echo( else: quo do (a: int) -> bool: a mod 3 != 0 -) \ No newline at end of file +) + +# bug #6980 + +proc fooBool: bool {.discardable.} = + true + +if true: + fooBool() +else: + raise newException(ValueError, "argh") From 52cc925e0e34580403b1a405ec8b4fa44c844de5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 22 Dec 2017 21:14:27 +0000 Subject: [PATCH 100/200] Fixes #6100. --- lib/pure/asyncfutures.nim | 6 +++--- lib/pure/asyncmacro.nim | 6 ++++++ tests/async/t6100.nim | 15 +++++++++++++++ tests/async/tfuturestream.nim | 2 +- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tests/async/t6100.nim diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 4bd3227a11..69db4d7125 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -263,12 +263,12 @@ proc mget*[T](future: FutureVar[T]): var T = ## Future has not been finished. result = Future[T](future).value -proc finished*[T](future: Future[T] | FutureVar[T]): bool = +proc finished*(future: FutureBase | FutureVar): bool = ## Determines whether ``future`` has completed. ## ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - when future is FutureVar[T]: - result = (Future[T](future)).finished + when future is FutureVar: + result = (FutureBase(future)).finished else: result = future.finished diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index a8e378d5c4..35523702d4 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -32,6 +32,12 @@ template createCb(retFutureSym, iteratorNameSym, try: if not nameIterVar.finished: var next = nameIterVar() + # Continue while the yielded future is already finished. + while (not next.isNil) and next.finished: + next = nameIterVar() + if nameIterVar.finished: + break + if next == nil: if not retFutureSym.finished: let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & diff --git a/tests/async/t6100.nim b/tests/async/t6100.nim new file mode 100644 index 0000000000..b4dc0f1469 --- /dev/null +++ b/tests/async/t6100.nim @@ -0,0 +1,15 @@ +discard """ + file: "t6100.nim" + exitcode: 0 + output: "10000000" +""" +import asyncdispatch + +let done = newFuture[int]() +done.complete(1) + +proc asyncSum: Future[int] {.async.} = + for _ in 1..10_000_000: + result += await done + +echo waitFor asyncSum() \ No newline at end of file diff --git a/tests/async/tfuturestream.nim b/tests/async/tfuturestream.nim index 9a8e986a09..d76752b7ea 100644 --- a/tests/async/tfuturestream.nim +++ b/tests/async/tfuturestream.nim @@ -18,8 +18,8 @@ var fs = newFutureStream[int]() proc alpha() {.async.} = for i in 0 .. 5: - await sleepAsync(1000) await fs.write(i) + await sleepAsync(1000) echo("Done") fs.complete() From f3a895f04321853f32b571a1f314d72c73274ff6 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 28 Dec 2017 00:50:45 +0100 Subject: [PATCH 101/200] fixes #6965 --- compiler/lookups.nim | 7 ++++--- compiler/semtypes.nim | 7 ++----- tests/modules/mrange.nim | 5 ++++- tests/modules/tambig_range.nim | 8 ++++++-- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 65cf504cf5..c409acc599 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -445,13 +445,14 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = if result != nil and result.kind == skStub: loadStub(result) -proc pickSym*(c: PContext, n: PNode; kind: TSymKind; +proc pickSym*(c: PContext, n: PNode; kinds: set[TSymKind]; flags: TSymFlags = {}): PSym = var o: TOverloadIter var a = initOverloadIter(o, c, n) while a != nil: - if a.kind == kind and flags <= a.flags: - return a + if a.kind in kinds and flags <= a.flags: + if result == nil: result = a + else: return nil # ambiguous a = nextOverloadIter(o, c, n) proc isInfixAs*(n: PNode): bool = diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f2fda3453b..cb66685b2f 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -320,11 +320,8 @@ proc semTypeIdent(c: PContext, n: PNode): PSym = if n.kind == nkSym: result = getGenSym(c, n.sym) else: - when defined(nimfix): - result = pickSym(c, n, skType) - if result.isNil: - result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared}) - else: + result = pickSym(c, n, {skType, skGenericParam}) + if result.isNil: result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared}) if result != nil: markUsed(n.info, result, c.graph.usageSym) diff --git a/tests/modules/mrange.nim b/tests/modules/mrange.nim index 9b78bf24b4..20c424a8c7 100644 --- a/tests/modules/mrange.nim +++ b/tests/modules/mrange.nim @@ -1,2 +1,5 @@ -proc range*() = echo "yo" \ No newline at end of file +proc range*() = echo "yo" + +proc set*(a: int) = + discard diff --git a/tests/modules/tambig_range.nim b/tests/modules/tambig_range.nim index 48e0e9f528..0103505210 100644 --- a/tests/modules/tambig_range.nim +++ b/tests/modules/tambig_range.nim @@ -1,9 +1,13 @@ discard """ errormsg: "ambiguous identifier: 'range' --use system.range or mrange.range" - line: 9 + line: 13 """ -# bug #6726 import mrange +# bug #6965 +type SomeObj = object + s: set[int8] + +# bug #6726 range() From f73015ad9ec6e977ca4b643ea9b03e164aeac431 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 22:48:53 +0000 Subject: [PATCH 102/200] Implement some simple pattern-based transformation for async tracebacks. --- lib/pure/asyncfutures.nim | 79 ++++++++++++++++++++++++++++---- tests/async/tasync_traceback.nim | 46 +++++++++++++++++++ 2 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 tests/async/tasync_traceback.nim diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 4bd3227a11..d78c6bcb75 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -217,17 +217,76 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) +proc processEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = + proc get(entries: seq[StackTraceEntry], i: int): StackTraceEntry = + if i >= entries.len: + return StackTraceEntry(procName: "", line: 0, filename: "") else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) + return entries[i] + + result = @[] + var i = 0 + while i < entries.len: + var entry = entries[i] + + if entry.procName.isNil: + # Start of a re-raise traceback which we do not care for. + break + + # Detect this pattern: + # (procname: a, line: 393, filename: asyncmacro.nim) + # (procname: cb0, line: 34, filename: asyncmacro.nim) + # (procname: aIter, line: 40, filename: tasync_traceback.nim) + let second = get(entries, i+1) + let third = get(entries, i+2) + let fitsPattern = + cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0 and + cmpIgnoreStyle($second.filename, "asyncmacro.nim") == 0 and + cmpIgnoreStyle($second.procName, "cb0") == 0 and + cmpIgnoreStyle($third.procName, $entry.procName & "iter") == 0 + + if fitsPattern: + entry = StackTraceEntry( + procName: entry.procName, + line: third.line, + filename: third.filename + ) + i.inc(2) + + result.add(entry) + i.inc + +proc injectStacktrace[T](future: Future[T]) = + when not defined(release): + const header = "Async traceback\n---------------\n" + + let originalMsg = future.error.msg + if header in originalMsg: + return + + let entries = getStackTraceEntries(future.error).processEntries() + future.error.msg = "\n" & header + + # Find longest filename & line number combo for alignment purposes. + var longestLeft = 0 + for entry in entries: + let left = $entry.filename & $entry.line + if left.len > longestLeft: + longestLeft = left.len + + # Format the entries. + for entry in entries: + let left = "$#($#)" % [$entry.filename, $entry.line] + future.error.msg.add("$1$2 $3\n" % [ + left, + spaces(longestLeft - left.len + 2), $entry.procName]) + + future.error.msg.add("Exception message: " & originalMsg & "\n") + future.error.msg.add("Exception type: ") + + # For debugging purposes TODO... + for entry in getStackTraceEntries(future.error): + future.error.msg.add "\n" & $entry proc read*[T](future: Future[T] | FutureVar[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim new file mode 100644 index 0000000000..dc9226617e --- /dev/null +++ b/tests/async/tasync_traceback.nim @@ -0,0 +1,46 @@ +discard """ + exitcode: 0 + output: "" +""" +import asyncdispatch + +# Tests to ensure our exception trace backs are friendly. + +# --- Simple test. --- +# +# What does this look like when it's synchronous? +# +# tasync_traceback.nim(23) tasync_traceback +# tasync_traceback.nim(21) a +# tasync_traceback.nim(18) b +# Error: unhandled exception: b failure [OSError] +# +# Good (not quite ideal, but gotta work within constraints) traceback, +# when exception is unhandled: +# +# +# +# +# <(the code responsible is in excpt:raiseExceptionAux)> +# Error: unhandled exception: b failure +# =============== +# Async traceback +# =============== +# +# tasync_traceback.nim(23) tasync_traceback +# +# tasync_traceback.nim(21) a +# tasync_traceback.nim(18) b + +proc b(): Future[int] {.async.} = + if true: + raise newException(OSError, "b failure") + +proc a(): Future[int] {.async.} = + return await b() + +let aFut = a() +# try: +discard waitFor aFut +# except Exception as exc: +# echo exc.msg From 9ca6afe73af8c187ea7786f7563ecc537ec31f51 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 25 Nov 2017 14:55:57 +0000 Subject: [PATCH 103/200] Refine the async tracebacks. --- lib/pure/asyncfutures.nim | 36 +++++++++++++++++++++++++------- tests/async/tasync_traceback.nim | 17 ++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index d78c6bcb75..d8385a9cbc 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -256,16 +256,31 @@ proc processEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = result.add(entry) i.inc +proc getHint(entry: StackTraceEntry): string = + ## We try to provide some hints about stack trace entries that the user + ## may not be familiar with, in particular calls inside the stdlib. + result = "" + case ($entry.procName).normalize() + of "cb0": + if cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0: + return "Resumes an async procedure" + of "processpendingcallbacks": + if cmpIgnoreStyle($entry.filename, "asyncdispatch.nim") == 0: + return "Executes pending callbacks" + of "poll": + if cmpIgnoreStyle($entry.filename, "asyncdispatch.nim") == 0: + return "Processes asynchronous completion events" + proc injectStacktrace[T](future: Future[T]) = when not defined(release): - const header = "Async traceback\n---------------\n" + const header = "Async traceback:\n" let originalMsg = future.error.msg if header in originalMsg: return let entries = getStackTraceEntries(future.error).processEntries() - future.error.msg = "\n" & header + future.error.msg = originalMsg & "\n" & header # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 @@ -274,19 +289,26 @@ proc injectStacktrace[T](future: Future[T]) = if left.len > longestLeft: longestLeft = left.len + const indent = " " # Format the entries. for entry in entries: let left = "$#($#)" % [$entry.filename, $entry.line] - future.error.msg.add("$1$2 $3\n" % [ + future.error.msg.add("$#$#$# $#\n" % [ + indent, left, - spaces(longestLeft - left.len + 2), $entry.procName]) + spaces(longestLeft - left.len + 2), + $entry.procName + ]) + let hint = getHint(entry) + if hint.len > 0: + future.error.msg.add(indent & "└─" & hint & "\n") future.error.msg.add("Exception message: " & originalMsg & "\n") future.error.msg.add("Exception type: ") - # For debugging purposes TODO... - for entry in getStackTraceEntries(future.error): - future.error.msg.add "\n" & $entry + # # For debugging purposes + # for entry in getStackTraceEntries(future.error): + # future.error.msg.add "\n" & $entry proc read*[T](future: Future[T] | FutureVar[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index dc9226617e..c69721f393 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -1,6 +1,13 @@ discard """ exitcode: 0 - output: "" + output: ''' +b failure +Async traceback: + tasync_traceback.nim(49) tasync_traceback + tasync_traceback.nim(47) a + tasync_traceback.nim(44) b +Exception message: b failure +Exception type:''' """ import asyncdispatch @@ -40,7 +47,7 @@ proc a(): Future[int] {.async.} = return await b() let aFut = a() -# try: -discard waitFor aFut -# except Exception as exc: -# echo exc.msg +try: + discard waitFor aFut +except Exception as exc: + echo exc.msg From a9a13e470b790442771e0003def074af69c715a5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 25 Nov 2017 21:04:01 +0000 Subject: [PATCH 104/200] Go through the re-raise stacks for more detailed tracebacks. --- lib/pure/asyncfutures.nim | 55 +++++++++++++++++++++++++------- tests/async/tasync_traceback.nim | 47 +++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index d8385a9cbc..f75859e4c0 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -217,6 +217,20 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) +proc diff[T](a, b: seq[T]): (int, seq[T]) = + ## Iterates through both sequences until the items do not match, + ## returns the remainder of `b` after the last item that does not match + ## together with the index of the last match. + ## + ## .. code-block::nim + ## doAssert(diff(@[1,2,42,123], @[1,2,123,678,21]) == (1, @[123,678,21])) + var lastIndex = 0 + for i in 0 .. = entries.len: @@ -230,8 +244,21 @@ proc processEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = var entry = entries[i] if entry.procName.isNil: - # Start of a re-raise traceback which we do not care for. - break + # Start of a re-raise traceback which may contain more info. + # Find where the re-raised traceback ends. + assert entry.line == -10 # Signifies start of re-raise block. + var reRaiseEnd = i+1 + while reRaiseEnd < entries.len and not entries[reRaiseEnd].procName.isNil: + reRaiseEnd.inc() + assert entries[reRaiseEnd].procName.isNil + assert entries[reRaiseEnd].line == -100 # Signifies end of re-raise block. + let reRaisedEntries = processEntries(entries[i+1 .. reRaiseEnd-1]) + + let (lastIndex, remainder) = diff(result, reRaisedEntries) + for i in 0.. 0: - future.error.msg.add(indent & "└─" & hint & "\n") + newMsg.add(indent & "└─" & hint & "\n") - future.error.msg.add("Exception message: " & originalMsg & "\n") - future.error.msg.add("Exception type: ") + newMsg.add("Exception message: " & exceptionMsg & "\n") + newMsg.add("Exception type:") # # For debugging purposes # for entry in getStackTraceEntries(future.error): - # future.error.msg.add "\n" & $entry + # newMsg.add "\n" & $entry + future.error.msg = newMsg proc read*[T](future: Future[T] | FutureVar[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index c69721f393..6ab7ef9153 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -3,10 +3,36 @@ discard """ output: ''' b failure Async traceback: - tasync_traceback.nim(49) tasync_traceback - tasync_traceback.nim(47) a - tasync_traceback.nim(44) b + tasync_traceback.nim(75) tasync_traceback + asyncmacro.nim(393) a + asyncmacro.nim(43) cb0 + └─Resumes an async procedure + asyncfutures.nim(211) callback= + asyncfutures.nim(190) addCallback + asyncfutures.nim(53) callSoon + asyncmacro.nim(34) cb0 + └─Resumes an async procedure + asyncmacro.nim(0) aIter + asyncfutures.nim(355) read + tasync_traceback.nim(73) a + tasync_traceback.nim(70) b Exception message: b failure +Exception type: + +bar failure +Async traceback: + tasync_traceback.nim(91) tasync_traceback + asyncdispatch.nim(1204) waitFor + asyncdispatch.nim(1253) poll + └─Processes asynchronous completion events + asyncdispatch.nim(181) processPendingCallbacks + └─Executes pending callbacks + asyncmacro.nim(34) cb0 + └─Resumes an async procedure + asyncmacro.nim(0) fooIter + asyncfutures.nim(355) read + tasync_traceback.nim(86) barIter +Exception message: bar failure Exception type:''' """ import asyncdispatch @@ -51,3 +77,18 @@ try: discard waitFor aFut except Exception as exc: echo exc.msg +echo() + +# From #6803 +proc bar(): Future[string] {.async.} = + await sleepAsync(100) + if true: + raise newException(OSError, "bar failure") + +proc foo(): Future[string] {.async.} = return await bar() + +try: + echo waitFor(foo()) +except Exception as exc: + echo exc.msg +echo() \ No newline at end of file From 3a790c9c7219e4f2e437ba60b959e402e62d2824 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 13:45:58 +0000 Subject: [PATCH 105/200] Rename cb0 to asyncProcName_continue + other improvements to async tracebacks. --- lib/pure/asyncfutures.nim | 97 ++++++++++++++++++++------------ lib/pure/asyncmacro.nim | 13 +++-- tests/async/tasync_traceback.nim | 1 + 3 files changed, 70 insertions(+), 41 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index f75859e4c0..086f9c729b 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -231,13 +231,9 @@ proc diff[T](a, b: seq[T]): (int, seq[T]) = break return (lastIndex, b[lastIndex .. ^1]) -proc processEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = - proc get(entries: seq[StackTraceEntry], i: int): StackTraceEntry = - if i >= entries.len: - return StackTraceEntry(procName: "", line: 0, filename: "") - else: - return entries[i] - +proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = + ## Merges stack trace entries containing re-raise entries into one + ## continuous stack trace. result = @[] var i = 0 while i < entries.len: @@ -252,7 +248,7 @@ proc processEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = reRaiseEnd.inc() assert entries[reRaiseEnd].procName.isNil assert entries[reRaiseEnd].line == -100 # Signifies end of re-raise block. - let reRaisedEntries = processEntries(entries[i+1 .. reRaiseEnd-1]) + let reRaisedEntries = mergeEntries(entries[i+1 .. reRaiseEnd-1]) let (lastIndex, remainder) = diff(result, reRaisedEntries) for i in 0..= entries.len: + return StackTraceEntry(procName: "", line: 0, filename: "") + else: + return entries[i] + + result = @[] + var i = 0 + while i < entries.len: + var entry = entries[i] + # Detect this pattern: # (procname: a, line: 393, filename: asyncmacro.nim) - # (procname: cb0, line: 34, filename: asyncmacro.nim) + # (procname: a_continue, line: 34, filename: asyncmacro.nim) # (procname: aIter, line: 40, filename: tasync_traceback.nim) let second = get(entries, i+1) + let third = get(entries, i+2) let fitsPattern = cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0 and cmpIgnoreStyle($second.filename, "asyncmacro.nim") == 0 and - cmpIgnoreStyle($second.procName, "cb0") == 0 and + ($second.procName).startsWith($entry.procName) and + ($second.procName).endsWith("continue") and cmpIgnoreStyle($third.procName, $entry.procName & "iter") == 0 if fitsPattern: @@ -287,10 +301,8 @@ proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. result = "" - case ($entry.procName).normalize() - of "cb0": - if cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0: - return "Resumes an async procedure" + let name = ($entry.procName).normalize() + case name of "processpendingcallbacks": if cmpIgnoreStyle($entry.filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" @@ -298,6 +310,35 @@ proc getHint(entry: StackTraceEntry): string = if cmpIgnoreStyle($entry.filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" + if name.endsWith("continue"): + if cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0: + return "Resumes an async procedure" + +proc `$`*(entries: seq[StackTraceEntry]): string = + result = "" + # Find longest filename & line number combo for alignment purposes. + var longestLeft = 0 + for entry in entries: + let left = $entry.filename & $entry.line + if left.len > longestLeft: + longestLeft = left.len + + const indent = 2 + # Format the entries. + for entry in entries: + let indentStr = spaces(indent) + + let left = "$#($#)" % [$entry.filename, $entry.line] + result.add("$#$#$# $#\n" % [ + indentStr, + left, + spaces(longestLeft - left.len + 2), + $entry.procName + ]) + let hint = getHint(entry) + if hint.len > 0: + result.add(indentStr & "└─" & hint & "\n") + proc injectStacktrace[T](future: Future[T]) = when not defined(release): const header = "\nAsync traceback:\n" @@ -309,29 +350,15 @@ proc injectStacktrace[T](future: Future[T]) = let start = exceptionMsg.find(header) exceptionMsg = exceptionMsg[0.. longestLeft: - longestLeft = left.len - - const indent = " " - # Format the entries. - for entry in entries: - let left = "$#($#)" % [$entry.filename, $entry.line] - newMsg.add("$#$#$# $#\n" % [ - indent, - left, - spaces(longestLeft - left.len + 2), - $entry.procName - ]) - let hint = getHint(entry) - if hint.len > 0: - newMsg.add(indent & "└─" & hint & "\n") + let entries = getStackTraceEntries(future.error).mergeEntries() + let shortEntries = entries.shortenEntries() + newMsg.add($shortEntries) + if entries.len > shortEntries.len: + newMsg.add("\nDetailed Async traceback:\n") + newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") newMsg.add("Exception type:") diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index a8e378d5c4..85d44b9f4b 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -25,10 +25,10 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = result = node[0] template createCb(retFutureSym, iteratorNameSym, - name, futureVarCompletions: untyped) = + strName, identName, futureVarCompletions: untyped) = var nameIterVar = iteratorNameSym #{.push stackTrace: off.} - proc cb0 {.closure.} = + proc identName {.closure.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -36,11 +36,11 @@ template createCb(retFutureSym, iteratorNameSym, if not retFutureSym.finished: let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & "`nil` Future?" - raise newException(AssertionError, msg % name) + raise newException(AssertionError, msg % strName) else: {.gcsafe.}: {.push hint[ConvFromXtoItselfNotNeeded]: off.} - next.callback = (proc() {.closure, gcsafe.})(cb0) + next.callback = (proc() {.closure, gcsafe.})(identName) {.pop.} except: futureVarCompletions @@ -52,7 +52,7 @@ template createCb(retFutureSym, iteratorNameSym, else: retFutureSym.fail(getCurrentException()) - cb0() + identName() #{.pop.} proc generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = @@ -389,9 +389,10 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = outerProcBody.add(closureIterator) # -> createCb(retFuture) - #var cbName = newIdentNode("cb") + var cbName = genSym(nskProc, prcName & "_continue") var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prcName), + cbName, createFutureVarCompletions(futureVarIdents, nil)) outerProcBody.add procCb diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 6ab7ef9153..5e4f1142f0 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -1,5 +1,6 @@ discard """ exitcode: 0 + disabled: "windows" output: ''' b failure Async traceback: From 391f877e6cc3ec36dc3d8977b519dabaa737108a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 15:06:13 +0000 Subject: [PATCH 106/200] Attempt to provide simplified and detailed tracebacks --- lib/pure/asyncfutures.nim | 46 +++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 086f9c729b..d836e3ab7b 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -274,25 +274,43 @@ proc shortenEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = # Detect this pattern: # (procname: a, line: 393, filename: asyncmacro.nim) + # ... # (procname: a_continue, line: 34, filename: asyncmacro.nim) + # ... # (procname: aIter, line: 40, filename: tasync_traceback.nim) - let second = get(entries, i+1) + proc searchBackwards(entries: seq[StackTraceEntry], + current: StackTraceEntry): (int, StackTraceEntry) = + # Search backwards for the beginning of the pattern. + result[0] = 0 + if not ($current.procName).normalize().endsWith("iter"): + return - let third = get(entries, i+2) - let fitsPattern = - cmpIgnoreStyle($entry.filename, "asyncmacro.nim") == 0 and - cmpIgnoreStyle($second.filename, "asyncmacro.nim") == 0 and - ($second.procName).startsWith($entry.procName) and - ($second.procName).endsWith("continue") and - cmpIgnoreStyle($third.procName, $entry.procName & "iter") == 0 + # Find (procname: a, line: 393, filename: asyncmacro.nim) + let start = entries.len-1 + var counter = start + while counter >= 0: + if cmpIgnoreStyle($entries[counter].procName & "iter", + $current.procName) == 0: + break + counter.dec() - if fitsPattern: - entry = StackTraceEntry( - procName: entry.procName, - line: third.line, - filename: third.filename + # Return when no beginning of pattern is found. + if counter < 0: + return + + result[0] = start - counter + result[1] = StackTraceEntry( + procName: entries[result[0]].procName, + line: current.line, + filename: current.filename ) - i.inc(2) + + let (itemsToRemove, newEntry) = searchBackwards(result, entry) + + if itemsToRemove != 0: + entry = newEntry + # Remove the previous entries. + result.setLen(result.len-itemsToRemove) result.add(entry) i.inc From 6301e3354387ad1d4c84a69c7f9d934b337121b4 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 15:09:34 +0000 Subject: [PATCH 107/200] Show only detailed async tracebacks. --- lib/pure/asyncfutures.nim | 64 +------------------------------- tests/async/tasync_traceback.nim | 27 +++++++++----- 2 files changed, 19 insertions(+), 72 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index d836e3ab7b..54b967f90d 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -225,7 +225,7 @@ proc diff[T](a, b: seq[T]): (int, seq[T]) = ## .. code-block::nim ## doAssert(diff(@[1,2,42,123], @[1,2,123,678,21]) == (1, @[123,678,21])) var lastIndex = 0 - for i in 0 .. = entries.len: - return StackTraceEntry(procName: "", line: 0, filename: "") - else: - return entries[i] - - result = @[] - var i = 0 - while i < entries.len: - var entry = entries[i] - - # Detect this pattern: - # (procname: a, line: 393, filename: asyncmacro.nim) - # ... - # (procname: a_continue, line: 34, filename: asyncmacro.nim) - # ... - # (procname: aIter, line: 40, filename: tasync_traceback.nim) - proc searchBackwards(entries: seq[StackTraceEntry], - current: StackTraceEntry): (int, StackTraceEntry) = - # Search backwards for the beginning of the pattern. - result[0] = 0 - if not ($current.procName).normalize().endsWith("iter"): - return - - # Find (procname: a, line: 393, filename: asyncmacro.nim) - let start = entries.len-1 - var counter = start - while counter >= 0: - if cmpIgnoreStyle($entries[counter].procName & "iter", - $current.procName) == 0: - break - counter.dec() - - # Return when no beginning of pattern is found. - if counter < 0: - return - - result[0] = start - counter - result[1] = StackTraceEntry( - procName: entries[result[0]].procName, - line: current.line, - filename: current.filename - ) - - let (itemsToRemove, newEntry) = searchBackwards(result, entry) - - if itemsToRemove != 0: - entry = newEntry - # Remove the previous entries. - result.setLen(result.len-itemsToRemove) - - result.add(entry) - i.inc - proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. @@ -372,11 +316,7 @@ proc injectStacktrace[T](future: Future[T]) = var newMsg = exceptionMsg & header let entries = getStackTraceEntries(future.error).mergeEntries() - let shortEntries = entries.shortenEntries() - newMsg.add($shortEntries) - if entries.len > shortEntries.len: - newMsg.add("\nDetailed Async traceback:\n") - newMsg.add($entries) + newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") newMsg.add("Exception type:") diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 5e4f1142f0..525914ef3b 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -4,35 +4,42 @@ discard """ output: ''' b failure Async traceback: - tasync_traceback.nim(75) tasync_traceback + tasync_traceback.nim(83) tasync_traceback asyncmacro.nim(393) a - asyncmacro.nim(43) cb0 + asyncmacro.nim(43) a_continue └─Resumes an async procedure asyncfutures.nim(211) callback= asyncfutures.nim(190) addCallback asyncfutures.nim(53) callSoon - asyncmacro.nim(34) cb0 + asyncmacro.nim(34) a_continue └─Resumes an async procedure asyncmacro.nim(0) aIter - asyncfutures.nim(355) read - tasync_traceback.nim(73) a - tasync_traceback.nim(70) b + asyncfutures.nim(340) read + asyncmacro.nim(34) a_continue + └─Resumes an async procedure + tasync_traceback.nim(81) aIter + asyncmacro.nim(393) b + asyncmacro.nim(34) b_continue + └─Resumes an async procedure + tasync_traceback.nim(78) bIter Exception message: b failure Exception type: bar failure Async traceback: - tasync_traceback.nim(91) tasync_traceback + tasync_traceback.nim(99) tasync_traceback asyncdispatch.nim(1204) waitFor asyncdispatch.nim(1253) poll └─Processes asynchronous completion events asyncdispatch.nim(181) processPendingCallbacks └─Executes pending callbacks - asyncmacro.nim(34) cb0 + asyncmacro.nim(34) foo_continue └─Resumes an async procedure asyncmacro.nim(0) fooIter - asyncfutures.nim(355) read - tasync_traceback.nim(86) barIter + asyncfutures.nim(340) read + asyncmacro.nim(34) bar_continue + └─Resumes an async procedure + tasync_traceback.nim(94) barIter Exception message: bar failure Exception type:''' """ From 383c80971cee93496c5317b7464021ac99711176 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 15:16:54 +0000 Subject: [PATCH 108/200] No need to recurse now that mergeEntries doesn't do any pattern matching. --- lib/pure/asyncfutures.nim | 3 ++- tests/async/tasync_traceback.nim | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 54b967f90d..a122e0c414 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -248,9 +248,10 @@ proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = reRaiseEnd.inc() assert entries[reRaiseEnd].procName.isNil assert entries[reRaiseEnd].line == -100 # Signifies end of re-raise block. - let reRaisedEntries = mergeEntries(entries[i+1 .. reRaiseEnd-1]) + let reRaisedEntries = entries[i+1 .. reRaiseEnd-1] let (lastIndex, remainder) = diff(result, reRaisedEntries) + # Insert all the entries after lastIndex. for i in 0.. Date: Wed, 13 Dec 2017 09:46:42 +0100 Subject: [PATCH 109/200] optimized friendly stack traces --- lib/pure/asyncfutures.nim | 48 ++++++++++++++++----------------------- lib/pure/asyncmacro.nim | 2 ++ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index a122e0c414..5675f5399a 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -1,4 +1,4 @@ -import os, tables, strutils, times, heapqueue, options, deques +import os, tables, strutils, times, heapqueue, options, deques, cstrutils # TODO: This shouldn't need to be included, but should ideally be exported. type @@ -217,19 +217,15 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) -proc diff[T](a, b: seq[T]): (int, seq[T]) = +proc diff[T](a, b: seq[T]; firstB, lastB: int): int = ## Iterates through both sequences until the items do not match, - ## returns the remainder of `b` after the last item that does not match + ## returns the remainder of `b[firstB..lastB]` after the last item that does not match ## together with the index of the last match. - ## - ## .. code-block::nim - ## doAssert(diff(@[1,2,42,123], @[1,2,123,678,21]) == (1, @[123,678,21])) - var lastIndex = 0 - for i in 0.. longestLeft: longestLeft = left.len - const indent = 2 + const indent = spaces(2) # Format the entries. for entry in entries: - let indentStr = spaces(indent) - let left = "$#($#)" % [$entry.filename, $entry.line] - result.add("$#$#$# $#\n" % [ - indentStr, + result.add((indent & "$#$# $#\n") % [ left, spaces(longestLeft - left.len + 2), $entry.procName ]) let hint = getHint(entry) if hint.len > 0: - result.add(indentStr & "└─" & hint & "\n") + result.add(indent & "└─" & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 85d44b9f4b..8c679929d3 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -389,6 +389,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = outerProcBody.add(closureIterator) # -> createCb(retFuture) + # NOTE: The "_continue" suffix is checked for in asyncfutures.nim to produce + # friendlier stack traces: var cbName = genSym(nskProc, prcName & "_continue") var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prcName), From b3055d8735ca012b77121c16bcf62e3f311084e8 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 13 Dec 2017 13:36:10 +0100 Subject: [PATCH 110/200] attempt to get the stack trace logic right --- lib/pure/asyncfutures.nim | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 5675f5399a..ea7a798fc7 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -217,16 +217,6 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) -proc diff[T](a, b: seq[T]; firstB, lastB: int): int = - ## Iterates through both sequences until the items do not match, - ## returns the remainder of `b[firstB..lastB]` after the last item that does not match - ## together with the index of the last match. - result = firstB - for i in 0.. 0 and e > i+1: + if result[last] != entries[e]: + newBlock = false + break + dec e + dec last + + if newBlock: + for j in i+1 ..< reRaiseEnd: result.add entries[j] + i = reRaiseEnd+1 continue From 1c2dee18d09c383bce51e9821f0f18656e71bb76 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 13 Dec 2017 13:48:48 +0100 Subject: [PATCH 111/200] attempt to get the stack trace logic right; fix boundaries --- lib/pure/asyncfutures.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index ea7a798fc7..f3b4234cc6 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -237,10 +237,10 @@ proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = # either the nested block is new, then we insert it, or we discard it # completely. 'diff' is the wrong idea here. - var last = result.len + var last = result.len-1 var e = reRaiseEnd-1 var newBlock = true - while last > 0 and e > i+1: + while last >= 0 and e >= i+1: if result[last] != entries[e]: newBlock = false break From 3593a4bdd1899194ed7cd3d21160bccfa6abb2e1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 14 Dec 2017 10:54:49 +0100 Subject: [PATCH 112/200] made the logic correct --- lib/pure/asyncfutures.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index f3b4234cc6..6e64981fe6 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -239,10 +239,10 @@ proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = # completely. 'diff' is the wrong idea here. var last = result.len-1 var e = reRaiseEnd-1 - var newBlock = true + var newBlock = false while last >= 0 and e >= i+1: if result[last] != entries[e]: - newBlock = false + newBlock = true break dec e dec last @@ -251,10 +251,9 @@ proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = for j in i+1 ..< reRaiseEnd: result.add entries[j] i = reRaiseEnd+1 - continue - - result.add(entry) - i.inc + else: + result.add(entry) + i.inc proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user From 7e6dc3679aaf463a957af7ff6bb34317c7656d4c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 27 Dec 2017 22:58:31 +0000 Subject: [PATCH 113/200] Simplify async traceback processing. --- lib/pure/asyncfutures.nim | 57 +++++++------------------ tests/async/tasync_traceback.nim | 72 +++++++++++++++++++------------- 2 files changed, 58 insertions(+), 71 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 6e64981fe6..bcc3ab613c 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -217,44 +217,6 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) -proc mergeEntries(entries: seq[StackTraceEntry]): seq[StackTraceEntry] = - ## Merges stack trace entries containing re-raise entries into one - ## continuous stack trace. - result = @[] - var i = 0 - while i < entries.len: - var entry = entries[i] - - if entry.procName.isNil: - # Start of a re-raise traceback which may contain more info. - # Find where the re-raised traceback ends. - assert entry.line == -10 # Signifies start of re-raise block. - var reRaiseEnd = i+1 - while reRaiseEnd < entries.len and not entries[reRaiseEnd].procName.isNil: - reRaiseEnd.inc() - assert entries[reRaiseEnd].procName.isNil - assert entries[reRaiseEnd].line == -100 # Signifies end of re-raise block. - - # either the nested block is new, then we insert it, or we discard it - # completely. 'diff' is the wrong idea here. - var last = result.len-1 - var e = reRaiseEnd-1 - var newBlock = false - while last >= 0 and e >= i+1: - if result[last] != entries[e]: - newBlock = true - break - dec e - dec last - - if newBlock: - for j in i+1 ..< reRaiseEnd: result.add entries[j] - - i = reRaiseEnd+1 - else: - result.add(entry) - i.inc - proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. @@ -275,22 +237,33 @@ proc `$`*(entries: seq[StackTraceEntry]): string = # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 for entry in entries: + if entry.procName.isNil: continue + let left = $entry.filename & $entry.line if left.len > longestLeft: longestLeft = left.len - const indent = spaces(2) + var indent = 2 # Format the entries. for entry in entries: + if entry.procName.isNil: + if entry.line == -10: + result.add(spaces(indent) & "#[\n") + indent.inc(2) + else: + indent.dec(2) + result.add(spaces(indent)& "]#\n") + continue + let left = "$#($#)" % [$entry.filename, $entry.line] - result.add((indent & "$#$# $#\n") % [ + result.add((spaces(indent) & "$#$# $#\n") % [ left, spaces(longestLeft - left.len + 2), $entry.procName ]) let hint = getHint(entry) if hint.len > 0: - result.add(indent & "└─" & hint & "\n") + result.add(spaces(indent+2) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): @@ -306,7 +279,7 @@ proc injectStacktrace[T](future: Future[T]) = var newMsg = exceptionMsg & header - let entries = getStackTraceEntries(future.error).mergeEntries() + let entries = getStackTraceEntries(future.error) newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 9e4f749e54..08f7e7317d 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -4,42 +4,56 @@ discard """ output: ''' b failure Async traceback: - tasync_traceback.nim(83) tasync_traceback - asyncmacro.nim(393) a - asyncmacro.nim(43) a_continue - └─Resumes an async procedure - asyncfutures.nim(211) callback= - asyncfutures.nim(190) addCallback - asyncfutures.nim(53) callSoon + tasync_traceback.nim(97) tasync_traceback + asyncmacro.nim(395) a asyncmacro.nim(34) a_continue - └─Resumes an async procedure - asyncmacro.nim(0) aIter - asyncfutures.nim(341) read - asyncmacro.nim(34) a_continue - └─Resumes an async procedure - tasync_traceback.nim(81) aIter - asyncmacro.nim(393) b + ## Resumes an async procedure + tasync_traceback.nim(95) aIter + asyncmacro.nim(395) b asyncmacro.nim(34) b_continue - └─Resumes an async procedure - tasync_traceback.nim(78) bIter + ## Resumes an async procedure + tasync_traceback.nim(92) bIter + #[ + tasync_traceback.nim(97) tasync_traceback + asyncmacro.nim(395) a + asyncmacro.nim(43) a_continue + ## Resumes an async procedure + asyncfutures.nim(211) callback= + asyncfutures.nim(190) addCallback + asyncfutures.nim(53) callSoon + asyncmacro.nim(34) a_continue + ## Resumes an async procedure + asyncmacro.nim(0) aIter + asyncfutures.nim(304) read + ]# Exception message: b failure Exception type: bar failure Async traceback: - tasync_traceback.nim(99) tasync_traceback - asyncdispatch.nim(1204) waitFor - asyncdispatch.nim(1253) poll - └─Processes asynchronous completion events - asyncdispatch.nim(181) processPendingCallbacks - └─Executes pending callbacks - asyncmacro.nim(34) foo_continue - └─Resumes an async procedure - asyncmacro.nim(0) fooIter - asyncfutures.nim(341) read - asyncmacro.nim(34) bar_continue - └─Resumes an async procedure - tasync_traceback.nim(94) barIter + tasync_traceback.nim(113) tasync_traceback + asyncdispatch.nim(1492) waitFor + asyncdispatch.nim(1496) poll + ## Processes asynchronous completion events + asyncdispatch.nim(1262) runOnce + asyncdispatch.nim(183) processPendingCallbacks + ## Executes pending callbacks + asyncmacro.nim(34) bar_continue + ## Resumes an async procedure + tasync_traceback.nim(108) barIter + #[ + tasync_traceback.nim(113) tasync_traceback + asyncdispatch.nim(1492) waitFor + asyncdispatch.nim(1496) poll + ## Processes asynchronous completion events + asyncdispatch.nim(1262) runOnce + asyncdispatch.nim(183) processPendingCallbacks + ## Executes pending callbacks + asyncmacro.nim(34) foo_continue + ## Resumes an async procedure + asyncmacro.nim(0) fooIter + asyncfutures.nim(304) read + ]# Exception message: bar failure Exception type:''' """ From caecd60e252253f256717cf30a1dfd0372ce447b Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 28 Dec 2017 13:52:23 +0000 Subject: [PATCH 114/200] Add more info in changelog about the #6223 change. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index de4b2f251b..1c5848ce89 100644 --- a/changelog.md +++ b/changelog.md @@ -111,6 +111,8 @@ This now needs to be written as: - The ``[]`` proc for strings now raises an ``IndexError`` exception when the specified slice is out of bounds. See issue [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details. + You can use ``substr(str, start, finish)`` to get the old behaviour back, + see [this commit](https://github.com/nim-lang/nimbot/commit/98cc031a27ea89947daa7f0bb536bcf86462941f) for an example. - ``strutils.split`` and ``strutils.rsplit`` with an empty string and a separator now returns that empty string. See issue [#4377](https://github.com/nim-lang/Nim/issues/4377). From e695f9d94e752fbf18012f9ee164af1ba0649fa1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 28 Dec 2017 19:48:21 +0100 Subject: [PATCH 115/200] make Nim compile with older nim versions --- compiler/semasgn.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index db08605cfb..67af6ade76 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -261,7 +261,7 @@ proc addParam(procType: PType; param: PSym) = rawAddSon(procType, param.typ) proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; - info: TLineInfo): PSym {.discardable.} = + info: TLineInfo): PSym = var a: TLiftCtx a.info = info a.c = c From 0a3cd6d9bacba524daab33578888b30ff17e773d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 28 Dec 2017 19:54:55 +0100 Subject: [PATCH 116/200] fixes a serious poll() regression that caused poll() to ignore the timeout parameter --- lib/pure/asyncdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 23eb80b37b..b62cf2e9bb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1493,7 +1493,7 @@ proc poll*(timeout = 500) = ## Waits for completion events and processes them. Raises ``ValueError`` ## if there are no pending operations. This runs the underlying OS ## `epoll`:idx: or `kqueue`:idx: primitive only once. - discard runOnce() + discard runOnce(timeout) # Common procedures between current and upcoming asyncdispatch include includes.asynccommon From a78d7a31f780c6cf1e421f820d9ed19a5db64ca7 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 13 Dec 2017 19:07:57 +0000 Subject: [PATCH 117/200] Add OpenSSL 1.1.0 support #5000 Add a simple online test --- lib/pure/net.nim | 2 +- lib/wrappers/openssl.nim | 97 ++++++++++++++++++++++++++++++--------- tests/untestable/tssl.nim | 36 +++++++++++++++ 3 files changed, 113 insertions(+), 22 deletions(-) create mode 100644 tests/untestable/tssl.nim diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 15f2c1228e..2ec724b151 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -413,7 +413,7 @@ proc isIpAddress*(address_str: string): bool {.tags: [].} = when defineSsl: CRYPTO_malloc_init() - SslLibraryInit() + doAssert SslLibraryInit() == 1 SslLoadErrorStrings() ErrLoadBioStrings() OpenSSL_add_all_algorithms() diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 55b0bc3f8e..1d0dc5897f 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -8,6 +8,17 @@ # ## OpenSSL support +## +## When OpenSSL is dynamically linked, the wrapper provides partial forward and backward +## compatibility for OpenSSL versions above and below 1.1.0 +## +## OpenSSL can be also statically linked using dynlibOverride:ssl for OpenSSL >= 1.1.0 +## +## Build and test examples: +## +## .. code-block:: +## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim +## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passL:-lcrypto --passL:-lssl -r tests/untestable/tssl.nim {.deadCodeElim: on.} @@ -25,8 +36,8 @@ when useWinVersion: from winlean import SocketHandle else: - const - versions = "(|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)" + const versions = "(.1.1|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + when defined(macosx): const DLLSSLName = "libssl" & versions & ".dylib" @@ -140,6 +151,7 @@ const SSL_OP_NO_SSLv2* = 0x01000000 SSL_OP_NO_SSLv3* = 0x02000000 SSL_OP_NO_TLSv1* = 0x04000000 + SSL_OP_NO_TLSv1_1* = 0x08000000 SSL_OP_ALL* = 0x000FFFFF SSL_VERIFY_NONE* = 0x00000000 SSL_VERIFY_PEER* = 0x00000001 @@ -191,16 +203,39 @@ const proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -when compileOption("dynlibOverride", "ssl"): - proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} - proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} - proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} +# TLS_method(), TLS_server_method(), TLS_client_method() are introduced in 1.1.0 +# and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2 +# SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0 - proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} +when compileOption("dynlibOverride", "ssl"): + # Static linking + proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_library_init*(): cint {.discardable.} = + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 + return OPENSSL_init_ssl(0.uint64, 0.uint8) + + proc TLS_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_method*(): PSSL_METHOD = + TLS_method() + + proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} template OpenSSL_add_all_algorithms*() = discard + + proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLSSLName, importc.} + + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long + OpenSSL_version_num() + + proc SSL_load_error_strings*() = + ## Removed from OpenSSL 1.1.0 + # This proc prevents breaking existing code calling SslLoadErrorStrings + # Static linking against OpenSSL < 1.1.0 is not supported + discard + else: # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some # symbols are loaded dynamically and we don't use them if not found. @@ -223,38 +258,58 @@ else: if not dl.isNil: result = symAddr(dl, name) + proc loadPSSLMethod(method1, method2: string): PSSL_METHOD = + ## Load from OpenSSL if available, otherwise + let m1 = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method1)) + if not m1.isNil: + return m1() + cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method2))() + proc SSL_library_init*(): cint {.discardable.} = - let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) - if not theProc.isNil: result = theProc() + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise + ## SSL_library_init + let theProc = cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](sslSym("OPENSSL_init_ssl")) + if not theProc.isNil: + return theProc(0, 0) + let olderProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) + if not olderProc.isNil: result = olderProc() proc SSL_load_error_strings*() = let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) if not theProc.isNil: theProc() proc SSLv23_client_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv23_client_method", "TLS_client_method") proc SSLv23_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv23_method", "TLS_method") proc SSLv2_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv2_method", "TLS_method") proc SSLv3_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv3_method", "TLS_method") + + proc TLS_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_method", "SSLv23_method") + + proc TLS_client_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_client_method", "SSLv23_client_method") + + proc TLS_server_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_server_method", "SSLv23_server_method") proc OpenSSL_add_all_algorithms*() = let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) if not theProc.isNil: theProc() + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long or 0 if not available + let theProc = cast[proc(): culong {.cdecl.}](sslSym("OpenSSL_version_num")) + result = + if theProc.isNil: 0.culong + else: theProc() + proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} diff --git a/tests/untestable/tssl.nim b/tests/untestable/tssl.nim new file mode 100644 index 0000000000..664ad805cf --- /dev/null +++ b/tests/untestable/tssl.nim @@ -0,0 +1,36 @@ +# +# Nim - SSL integration tests +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Warning: this test performs external networking. +## +## Test with: +## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim +## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passL:-lcrypto --passL:-lssl -r tests/untestable/tssl.nim +## The compilation is expected to succeed with any new/old version of OpenSSL, +## both with dynamic and static linking. +## The "howsmyssl" test is known to fail with OpenSSL < 1.1 due to insecure +## cypher suites being used. + +import httpclient, os +from strutils import contains, toHex + +from openssl import getOpenSSLVersion + +when isMainModule: + echo "version: 0x" & $getOpenSSLVersion().toHex() + + let client = newHttpClient() + # hacky SSL check + const url = "https://www.howsmyssl.com" + let report = client.getContent(url) + if not report.contains(">Probably Okay"): + let fn = getTempDir() / "sslreport.html" + echo "SSL CHECK ERROR, see " & fn + writeFile(fn, report) + quit(1) + + echo "done" From cf259fbd1c7e2642d00ed6decbceec353b53c84e Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 29 Dec 2017 20:01:49 +0100 Subject: [PATCH 118/200] fixes #6972 --- compiler/renderer.nim | 1 + compiler/sem.nim | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index d4b401c02e..6735cc1ce2 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -719,6 +719,7 @@ proc gcase(g: var TSrcGen, n: PNode) = var c: TContext initContext(c) var length = sonsLen(n) + if length == 0: return var last = if n.sons[length-1].kind == nkElse: -2 else: -1 if longMode(g, n, 0, last): incl(c.flags, rfLongMode) putWithSpace(g, tkCase, "case") diff --git a/compiler/sem.nim b/compiler/sem.nim index ababbd303c..d2831827ad 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -74,7 +74,7 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = localError(arg.info, errExprXHasNoType, renderTree(arg, {renderNoComments})) # error correction: - result = copyNode(arg) + result = copyTree(arg) result.typ = formal else: result = indexTypesMatch(c, formal, arg.typ, arg) @@ -168,9 +168,9 @@ proc commonType*(x, y: PType): PType = proc endsInNoReturn(n: PNode): bool = # check if expr ends in raise exception or call of noreturn proc var it = n - while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: + while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: it = it.lastSon - result = it.kind == nkRaiseStmt or + result = it.kind == nkRaiseStmt or it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags proc commonType*(x: PType, y: PNode): PType = From 29db57a804e03ac4d9d943ac7cf3966777fa5c8d Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 30 Dec 2017 00:29:53 +0100 Subject: [PATCH 119/200] fixes reported 'proc foo(): int = result' codegen problem --- compiler/semexprs.nim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index a0f519820b..51e75e91fb 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1425,11 +1425,7 @@ proc semProcBody(c: PContext, n: PNode): PNode = openScope(c) result = semExpr(c, n) if c.p.resultSym != nil and not isEmptyType(result.typ): - # transform ``expr`` to ``result = expr``, but not if the expr is already - # ``result``: - if result.kind == nkSym and result.sym == c.p.resultSym: - discard - elif result.kind == nkNilLit: + if result.kind == nkNilLit: # or ImplicitlyDiscardable(result): # new semantic: 'result = x' triggers the void context result.typ = nil From 3073f08e4857b14a6bd1b0022fe45f461bf88171 Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Sat, 30 Dec 2017 15:41:41 +0700 Subject: [PATCH 120/200] Add hasPendingOperations check to asyncdispatch.drain --- lib/pure/asyncdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b62cf2e9bb..42ffa236c9 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1487,7 +1487,7 @@ proc drain*(timeout = 500) = ## if there are no pending operations. In contrast to ``poll`` this ## processes as many events as are available. if runOnce(timeout): - while runOnce(0): discard + while hasPendingOperations() and runOnce(0): discard proc poll*(timeout = 500) = ## Waits for completion events and processes them. Raises ``ValueError`` From 41472f0e2827ee3e028db7faa76e368005728231 Mon Sep 17 00:00:00 2001 From: Parashurama Date: Wed, 23 Aug 2017 13:42:40 +0200 Subject: [PATCH 121/200] add support cast[integer] in VM --- compiler/vmgen.nim | 47 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 3790a8392f..17878b656b 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -30,7 +30,7 @@ import strutils, ast, astalgo, types, msgs, renderer, vmdef, trees, intsets, rodread, magicsys, options, lowerings - +import platform from os import splitFile when hasFFI: @@ -761,6 +761,49 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) = c.gABC(n, opcCard, dest, tmp) c.freeTemp(tmp) +proc genIntCast(c: PCtx; n: PNode; dest: var TDest) = + const allowedIntegers = {tyInt..tyInt64, tyUInt..tyUInt64, tyChar} + var signedIntegers = {tyInt8..tyInt32} + var unsignedIntegers = {tyUInt8..tyUInt32, tyChar} + let src = n.sons[1].typ.skipTypes(abstractRange)#.kind + let dst = n.sons[0].typ.skipTypes(abstractRange)#.kind + let src_size = src.getSize + + if platform.intSize < 8: + signedIntegers.incl(tyInt) + unsignedIntegers.incl(tyUInt) + if src_size == dst.getSize and src.kind in allowedIntegers and + dst.kind in allowedIntegers: + let tmp = c.genx(n.sons[1]) + var tmp2 = c.getTemp(n.sons[1].typ) + let tmp3 = c.getTemp(n.sons[1].typ) + if dest < 0: dest = c.getTemp(n[0].typ) + proc mkIntLit(ival: int): int = + result = genLiteral(c, newIntTypeNode(nkIntLit, ival, getSysType(tyInt))) + if src.kind in unsignedIntegers and dst.kind in signedIntegers: + # cast unsigned to signed integer of same size + # signedVal = (unsignedVal xor offset) -% offset + let offset = 1 shl (src_size * 8 - 1) + c.gABx(n, opcLdConst, tmp2, mkIntLit(offset)) + c.gABC(n, opcBitxorInt, tmp3, tmp, tmp2) + c.gABC(n, opcSubInt, dest, tmp3, tmp2) + elif src.kind in signedIntegers and dst.kind in unsignedIntegers: + # cast signed to unsigned integer of same size + # unsignedVal = (offset +% signedVal +% 1) and offset + let offset = (1 shl (src_size * 8)) - 1 + c.gABx(n, opcLdConst, tmp2, mkIntLit(offset)) + c.gABx(n, opcLdConst, dest, mkIntLit(offset+1)) + c.gABC(n, opcAddu, tmp3, tmp, dest) + c.gABC(n, opcNarrowU, tmp3, TRegister(src_size*8)) + c.gABC(n, opcBitandInt, dest, tmp3, tmp2) + else: + c.gABC(n, opcAsgnInt, dest, tmp) + c.freeTemp(tmp) + c.freeTemp(tmp2) + c.freeTemp(tmp3) + else: + globalError(n.info, errGenerated, "VM is only allowed to 'cast' between integers of same size") + proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = case m of mAnd: c.genAndOr(n, opcFJmp, dest) @@ -1844,7 +1887,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = if allowCast in c.features: genConv(c, n, n.sons[1], dest, opcCast) else: - globalError(n.info, errGenerated, "VM is not allowed to 'cast'") + genIntCast(c, n, dest) of nkTypeOfExpr: genTypeLit(c, n.typ, dest) of nkComesFrom: From 26a34d52a089f54ec6b073230742ed9894d8d197 Mon Sep 17 00:00:00 2001 From: Parashurama Date: Wed, 23 Aug 2017 13:44:22 +0200 Subject: [PATCH 122/200] add tests for integer casting in VM. --- tests/vm/tcastint.nim | 120 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/vm/tcastint.nim diff --git a/tests/vm/tcastint.nim b/tests/vm/tcastint.nim new file mode 100644 index 0000000000..7b9ddd7d9f --- /dev/null +++ b/tests/vm/tcastint.nim @@ -0,0 +1,120 @@ +discard """ + file: "tcastint.nim" + output: "OK" +""" + +type + Dollar = distinct int + XCoord = distinct int32 + Digit = range[-9..0] + +# those are necessary for comparisons below. +proc `==`(x, y: Dollar): bool {.borrow.} +proc `==`(x, y: XCoord): bool {.borrow.} + +proc dummy[T](x: T): T = x + +proc test() = + let U8 = 0b1011_0010'u8 + let I8 = 0b1011_0010'i8 + let C8 = 0b1011_0010'u8.char + let C8_1 = 0b1011_0011'u8.char + let U16 = 0b10100111_00101000'u16 + let I16 = 0b10100111_00101000'i16 + let U32 = 0b11010101_10011100_11011010_01010000'u32 + let I32 = 0b11010101_10011100_11011010_01010000'i32 + let U64A = 0b11000100_00111111_01111100_10001010_10011001_01001000_01111010_00010001'u64 + let I64A = 0b11000100_00111111_01111100_10001010_10011001_01001000_01111010_00010001'i64 + let U64B = 0b00110010_11011101_10001111_00101000_00000000_00000000_00000000_00000000'u64 + let I64B = 0b00110010_11011101_10001111_00101000_00000000_00000000_00000000_00000000'i64 + when sizeof(int) == 8: + let UX = U64A.uint + let IX = I64A.int + elif sizeof(int) == 4: + let UX = U32.uint + let IX = I32.int + elif sizeof(int) == 2: + let UX = U16.uint + let IX = I16.int + else: + let UX = U8.uint + let IX = I8.int + + doAssert(cast[char](I8) == C8) + doAssert(cast[uint8](I8) == U8) + doAssert(cast[uint16](I16) == U16) + doAssert(cast[uint32](I32) == U32) + doAssert(cast[uint64](I64A) == U64A) + doAssert(cast[uint64](I64B) == U64B) + doAssert(cast[int8](U8) == I8) + doAssert(cast[int16](U16) == I16) + doAssert(cast[int32](U32) == I32) + doAssert(cast[int64](U64A) == I64A) + doAssert(cast[int64](U64B) == I64B) + doAssert(cast[uint](IX) == UX) + doAssert(cast[int](UX) == IX) + + doAssert(cast[char](I8 + 1) == C8_1) + doAssert(cast[uint8](I8 + 1) == U8 + 1) + doAssert(cast[uint16](I16 + 1) == U16 + 1) + doAssert(cast[uint32](I32 + 1) == U32 + 1) + doAssert(cast[uint64](I64A + 1) == U64A + 1) + doAssert(cast[uint64](I64B + 1) == U64B + 1) + doAssert(cast[int8](U8 + 1) == I8 + 1) + doAssert(cast[int16](U16 + 1) == I16 + 1) + doAssert(cast[int32](U32 + 1) == I32 + 1) + doAssert(cast[int64](U64A + 1) == I64A + 1) + doAssert(cast[int64](U64B + 1) == I64B + 1) + doAssert(cast[uint](IX + 1) == UX + 1) + doAssert(cast[int](UX + 1) == IX + 1) + + doAssert(cast[char](I8.dummy) == C8.dummy) + doAssert(cast[uint8](I8.dummy) == U8.dummy) + doAssert(cast[uint16](I16.dummy) == U16.dummy) + doAssert(cast[uint32](I32.dummy) == U32.dummy) + doAssert(cast[uint64](I64A.dummy) == U64A.dummy) + doAssert(cast[uint64](I64B.dummy) == U64B.dummy) + doAssert(cast[int8](U8.dummy) == I8.dummy) + doAssert(cast[int16](U16.dummy) == I16.dummy) + doAssert(cast[int32](U32.dummy) == I32.dummy) + doAssert(cast[int64](U64A.dummy) == I64A.dummy) + doAssert(cast[int64](U64B.dummy) == I64B.dummy) + doAssert(cast[uint](IX.dummy) == UX.dummy) + doAssert(cast[int](UX.dummy) == IX.dummy) + + + doAssert(cast[int64](if false: U64B else: 0'u64) == (if false: I64B else: 0'i64)) + + block: + let raw = 3 + let money = Dollar(raw) # this must be a variable, is otherwise constant folded. + doAssert(cast[int](money) == raw) + doAssert(cast[Dollar](raw) == money) + block: + let raw = 150'i32 + let position = XCoord(raw) # this must be a variable, is otherwise constant folded. + doAssert(cast[int32](position) == raw) + doAssert(cast[XCoord](raw) == position) + block: + let raw = -2 + let digit = Digit(raw) + doAssert(cast[int](digit) == raw) + doAssert(cast[Digit](raw) == digit) + + when defined nimvm: + doAssert(not compiles(cast[float](I64A))) + doAssert(not compiles(cast[float32](I64A))) + + doAssert(not compiles(cast[char](I64A))) + doAssert(not compiles(cast[uint16](I64A))) + doAssert(not compiles(cast[uint32](I64A))) + + doAssert(not compiles(cast[uint16](I8))) + doAssert(not compiles(cast[uint32](I8))) + doAssert(not compiles(cast[uint64](I8))) + +test() +static: + test() + +echo "OK" From 3714e2f8714e0a1bc1efbafa26696b708fe9a4b5 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Thu, 28 Dec 2017 14:10:27 -0500 Subject: [PATCH 123/200] Add compile-time paragraph to manual Includes a note in the manual entry for case statements clarifying that the branch values must be known at compile time. --- doc/manual/stmts.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/stmts.txt b/doc/manual/stmts.txt index b24ec3b4a6..721b5cff81 100644 --- a/doc/manual/stmts.txt +++ b/doc/manual/stmts.txt @@ -296,6 +296,10 @@ empty ``discard`` statement should be used. For non ordinal types it is not possible to list every possible value and so these always require an ``else`` part. +As case statements perform compile-time exhaustiveness checks, the value in +every ``of`` branch must be known at compile time. This fact is also exploited +to generate more performant code. + As a special semantic extension, an expression in an ``of`` branch of a case statement may evaluate to a set or array constructor; the set or array is then expanded into a list of its elements: From 64d583d6caadb6c6e1f21bbc9385e2a53b81fe5b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 30 Dec 2017 12:55:53 +0100 Subject: [PATCH 124/200] destroyer pass: disable debug output --- compiler/destroyer.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index caa18af92f..0fdeceba0b 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -296,7 +296,8 @@ proc p(n: PNode; c: var Con): PNode = recurse(n, result) proc injectDestructorCalls*(owner: PSym; n: PNode): PNode = - echo "injecting into ", n + when defined(nimDebugDestroys): + echo "injecting into ", n var c: Con c.owner = owner c.tmp = newSym(skTemp, getIdent":d", owner, n.info) From a521f983921878e613c1b89f40c081eb5393055b Mon Sep 17 00:00:00 2001 From: Daniil Yarancev <21169548+Yardanico@users.noreply.github.com> Date: Sat, 30 Dec 2017 18:15:15 +0300 Subject: [PATCH 125/200] Add warnings about deprecation to times module (#7001) * Add warnings about deprecation to times module I've added warnings about some procedures being deprecated. Also I can't find what is the replacement for `getTimezone` proc, please help with that * Update times.nim * Update times.nim --- lib/pure/times.nim | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 606acbc1c4..85988e5858 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1352,25 +1352,35 @@ else: proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. + ## + ## **Deprecated since v0.18.0:** use ``Time`` instead Time(since1970) proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. + ## + ## **Deprecated since v0.18.0:** use ``Time`` instead Time(since1970) proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = ## Returns the time in seconds since the unix epoch. + ## + ## **Deprecated since v0.18.0:** use ``float`` instead float(time) proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-time representation, ## expressed relative to the user's specified time zone. + ## + ## **Deprecated since v0.18.0:** use ``local`` instead time.local proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). + ## expressed in Coordinated Universal Time (UTC). + ## + ## **Deprecated since v0.18.0:** use ``utc`` instead time.utc proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} = @@ -1468,4 +1478,4 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = y = year - a m = month + (12*a) - 2 d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 - result = d.WeekDay \ No newline at end of file + result = d.WeekDay From d1e10f9aa3e033414fb924e4f90736e46fde8256 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Sun, 31 Dec 2017 11:28:51 +0300 Subject: [PATCH 126/200] Fixed mutex usage in SharedList and SharedTable. Closes #6988 (#6990) --- lib/pure/collections/sharedlist.nim | 15 +++++++++++---- lib/pure/collections/sharedtables.nim | 20 +++++++++++++++----- lib/system/gc.nim | 2 +- lib/system/gc2.nim | 2 +- lib/system/gc_ms.nim | 2 +- tests/collections/ttables.nim | 3 ++- 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index e93ceb02fc..b3e677b79c 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -73,10 +73,10 @@ proc add*[A](x: var SharedList[A]; y: A) = node.d[node.dataLen] = y inc(node.dataLen) -proc initSharedList*[A](): SharedList[A] = - initLock result.lock - result.head = nil - result.tail = nil +proc init*[A](t: var SharedList[A]) = + initLock t.lock + t.head = nil + t.tail = nil proc clear*[A](t: var SharedList[A]) = withLock(t): @@ -92,4 +92,11 @@ proc deinitSharedList*[A](t: var SharedList[A]) = clear(t) deinitLock t.lock +proc initSharedList*[A](): SharedList[A] {.deprecated.} = + ## Deprecated. Use `init` instead. + ## This is not posix compliant, may introduce undefined behavior. + initLock result.lock + result.head = nil + result.tail = nil + {.pop.} diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 211a6ce6ac..4f311af874 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -192,19 +192,29 @@ proc del*[A, B](t: var SharedTable[A, B], key: A) = withLock t: delImpl() -proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] = +proc init*[A, B](t: var SharedTable[A, B], initialSize=64) = ## creates a new hash table that is empty. ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the ## `math `_ module or the ``rightSize`` proc from this module. assert isPowerOfTwo(initialSize) - result.counter = 0 - result.dataLen = initialSize - result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + t.counter = 0 + t.dataLen = initialSize + t.data = cast[KeyValuePairSeq[A, B]](allocShared0( sizeof(KeyValuePair[A, B]) * initialSize)) - initLock result.lock + initLock t.lock proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = deallocShared(t.data) deinitLock t.lock + +proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] {.deprecated.} = + ## Deprecated. Use `init` instead. + ## This is not posix compliant, may introduce undefined behavior. + assert isPowerOfTwo(initialSize) + result.counter = 0 + result.dataLen = initialSize + result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + sizeof(KeyValuePair[A, B]) * initialSize)) + initLock result.lock diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 68bf5f6c2d..dac06119d9 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -318,7 +318,7 @@ proc initGC() = init(gch.marked) init(gch.additionalRoots) when hasThreadSupport: - gch.toDispose = initSharedList[pointer]() + init(gch.toDispose) when useMarkForDebug or useBackupGc: type diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 4ecf3b2262..d57a01dc75 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -133,7 +133,7 @@ proc initGC() = init(gch.additionalRoots) init(gch.greyStack) when hasThreadSupport: - gch.toDispose = initSharedList[pointer]() + init(gch.toDispose) # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 272047bb7b..5fc48d848b 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -233,7 +233,7 @@ proc initGC() = init(gch.allocated) init(gch.marked) when hasThreadSupport: - gch.toDispose = initSharedList[pointer]() + init(gch.toDispose) proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 2b8af5bd97..7fe4c79b13 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -213,7 +213,8 @@ block clearCountTableTest: assert t.len() == 0 block withKeyTest: - var t = initSharedTable[int, int]() + var t: SharedTable[int, int] + t.init() t.withKey(1) do (k: int, v: var int, pairExists: var bool): assert(v == 0) pairExists = true From 74fb3e6e6e68b45e37c9c0688cc561ef161ff62d Mon Sep 17 00:00:00 2001 From: GULPF Date: Sun, 31 Dec 2017 14:40:46 +0100 Subject: [PATCH 127/200] Improve deprecation comments in times module --- lib/pure/times.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 85988e5858..7df1d01782 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1353,20 +1353,20 @@ proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprec ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. ## - ## **Deprecated since v0.18.0:** use ``Time`` instead + ## **Deprecated since v0.18.0:** use ``fromUnix`` instead Time(since1970) proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. ## - ## **Deprecated since v0.18.0:** use ``Time`` instead + ## **Deprecated since v0.18.0:** use ``fromUnix`` instead Time(since1970) proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = ## Returns the time in seconds since the unix epoch. ## - ## **Deprecated since v0.18.0:** use ``float`` instead + ## **Deprecated since v0.18.0:** use ``toUnix`` instead float(time) proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = @@ -1385,6 +1385,9 @@ proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} = ## Returns the offset of the local (non-DST) timezone in seconds west of UTC. + ## + ## **Deprecated since v0.18.0:** use ``now().utcOffset`` to get the current + ## utc offset (including DST). when defined(JS): return newDate().getTimezoneOffset() * 60 elif defined(freebsd) or defined(netbsd) or defined(openbsd): From 42cff6e0c5d9a818ed2a788b2c4207ac389d3092 Mon Sep 17 00:00:00 2001 From: oltolm Date: Sun, 31 Dec 2017 14:49:42 +0100 Subject: [PATCH 128/200] add support for building GUI applications with TCC (#7003) --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 5299b2dbf6..62990593d0 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -253,7 +253,7 @@ compiler tcc: compilerExe: "tcc", cppCompiler: "", compileTmpl: "-c $options $include -o $objfile $file", - buildGui: "UNAVAILABLE!", + buildGui: "-Wl,-subsystem=gui", buildDll: " -shared", buildLib: "", # XXX: not supported yet linkerExe: "tcc", From 3f2636c76580c8a086df4255a17b077622434d7d Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Mon, 1 Jan 2018 02:39:55 +0300 Subject: [PATCH 129/200] Remove obsolete method analyzeAndConsolidateOutput (#6998) After the change, when stacktraces rendered in reversed order (most recent call first), this method removed all stacktraces in the test failures. --- tests/testament/tester.nim | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 69b640fa28..870f9f8656 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -290,20 +290,6 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, if given.err == reSuccess: inc(r.passed) r.addResult(test, target, expectedmsg, givenmsg, given.err) -proc analyzeAndConsolidateOutput(s: string): string = - result = "" - let rows = s.splitLines - for i in 0 ..< rows.len: - if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1): - result = substr(rows[i], pos) & "\n" - for i in i+1 ..< rows.len: - result.add rows[i] & "\n" - if not (rows[i] =~ peg"['(']+ '(' \d+ ')' \s+"): - return - elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1): - result = substr(rows[i], pos) - return - proc testSpec(r: var TResults, test: TTest, target = targetC) = let tname = test.name.addFileExt(".nim") #echo "TESTING ", tname @@ -376,8 +362,7 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = if exitCode != expected.exitCode: r.addResult(test, target, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & - analyzeAndConsolidateOutput(bufB), - reExitCodesDiffer) + bufB, reExitCodesDiffer) continue if bufB != expectedOut and expected.action != actionRunNoSpec: From 37dde55f8d4c4d9fd7a3968756aa4885c9dbda9a Mon Sep 17 00:00:00 2001 From: data-man Date: Tue, 2 Jan 2018 01:44:45 +0300 Subject: [PATCH 130/200] Add a shared collections to the docs --- doc/lib.rst | 4 ++++ web/website.ini | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/lib.rst b/doc/lib.rst index 58dedc49c3..755c11899f 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -92,6 +92,10 @@ Collections and algorithms * `sequtils `_ This module implements operations for the built-in seq type which were inspired by functional programming languages. +* `sharedtables `_ + Nim shared hash table support. Contains shared tables. +* `sharedlist `_ + Nim shared linked list support. Contains shared singly linked list. String handling diff --git a/web/website.ini b/web/website.ini index 32b1936d56..273c3223d3 100644 --- a/web/website.ini +++ b/web/website.ini @@ -51,6 +51,7 @@ srcdoc2: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser" srcdoc2: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes" srcdoc2: "pure/json;pure/base64;pure/scgi" srcdoc2: "pure/collections/tables;pure/collections/sets;pure/collections/lists" +srcdoc2: "pure/collections/sharedlist;pure/collections/sharedtables" srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/collections/deques;pure/encodings" srcdoc2: "pure/events;pure/collections/sequtils;pure/cookies" srcdoc2: "pure/memfiles;pure/subexes;pure/collections/critbits" From 720c73e6d5c8b21a2d6d528a3a8947e498e9d1ad Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 3 Jan 2018 02:36:29 +0100 Subject: [PATCH 131/200] symbol files: fixes the logic for multi-methods --- compiler/cgen.nim | 8 ++++---- compiler/passes.nim | 4 ++-- compiler/sem.nim | 4 ++++ compiler/transf.nim | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 217138dd04..573a14927a 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1258,7 +1258,7 @@ proc resetModule*(m: BModule) = # indicate that this is now cached module # the cache will be invalidated by nullifying gModules - m.fromCache = true + #m.fromCache = true m.g = nil # we keep only the "merge info" information for the module @@ -1390,7 +1390,7 @@ proc writeModule(m: BModule, pending: bool) = # generate code for the init statements of the module: let cfile = getCFile(m) - if not m.fromCache or optForceFullMake in gGlobalOptions: + if m.rd == nil or optForceFullMake in gGlobalOptions: genInitCode(m) finishTypeDescriptions(m) if sfMainModule in m.module.flags: @@ -1465,10 +1465,10 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = if g.generatedHeader != nil: finishModule(g.generatedHeader) while g.forwardedProcsCounter > 0: for m in cgenModules(g): - if not m.fromCache: + if m.rd == nil: finishModule(m) for m in cgenModules(g): - if m.fromCache: + if m.rd != nil: m.updateCachedModule else: m.writeModule(pending=true) diff --git a/compiler/passes.nim b/compiler/passes.nim index b84fe2f4d7..29b27627d3 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -18,7 +18,7 @@ import type TPassContext* = object of RootObj # the pass's context - fromCache*: bool # true if created by "openCached" + rd*: PRodReader # != nil if created by "openCached" PPassContext* = ref TPassContext @@ -118,7 +118,7 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym, if not isNil(gPasses[i].openCached): a[i] = gPasses[i].openCached(g, module, rd) if a[i] != nil: - a[i].fromCache = true + a[i].rd = rd else: a[i] = nil diff --git a/compiler/sem.nim b/compiler/sem.nim index d2831827ad..1098e9961f 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -501,6 +501,8 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext = result = myOpen(graph, module, rd.cache) + +proc replayMethodDefs(graph: ModuleGraph; rd: PRodReader) = for m in items(rd.methods): methodDef(graph, m, true) proc isImportSystemStmt(n: PNode): bool = @@ -607,6 +609,8 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = addCodeForGenerics(c, result) if c.module.ast != nil: result.add(c.module.ast) + if c.rd != nil: + replayMethodDefs(graph, c.rd) popOwner(c) popProcCon(c) if c.runnableExamples != nil: testExamples(c) diff --git a/compiler/transf.nim b/compiler/transf.nim index 6bc809fd20..f8f7f87464 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -914,7 +914,7 @@ proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode = # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip # this step! We have to rely that the semantic pass transforms too errornous # nodes into an empty node. - if c.fromCache or nfTransf in n.flags: return n + if c.rd != nil or nfTransf in n.flags: return n pushTransCon(c, newTransCon(owner)) result = PNode(transform(c, n)) popTransCon(c) From d5f539dc874ef3b7781eba465d382fd2dce58943 Mon Sep 17 00:00:00 2001 From: qqquinta Date: Wed, 3 Jan 2018 13:40:19 +0200 Subject: [PATCH 132/200] jsgen: bool genConv generates boolean values instead of numeric (#7016) --- compiler/jsgen.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 65a6a5dae9..dac2de7464 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2051,10 +2051,10 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) = return case dest.kind: of tyBool: - r.res = "(($1)? 1:0)" % [r.res] + r.res = "(!!($1))" % [r.res] r.kind = resExpr of tyInt: - r.res = "($1|0)" % [r.res] + r.res = "(($1)|0)" % [r.res] else: # TODO: What types must we handle here? discard From 8941f5bd9c3f90f4879647fe86d875e46598ecbd Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Wed, 3 Jan 2018 14:41:10 +0300 Subject: [PATCH 133/200] Use safe limit for toRational(float, int) (#7021) Current limit `high(int32)` is not safe for 32-bit platforms and it will overflow (even when running its own test suite). Similar behaviour would be when try to set limit to `high(int64)` on 64-bit platforms. This change selects safe maximum value based on platform size of int. Safe maximum considered half of int size (for backward compatiblity). --- lib/pure/rationals.nim | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 7fb24c26f3..7907b4e6c0 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -39,7 +39,7 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] = result.num = x result.den = 1 -proc toRational*(x: float, n: int = high(int32)): Rational[int] = +proc toRational*(x: float, n: int = high(int) shr (sizeof(int) div 2 * 8)): Rational[int] = ## Calculates the best rational numerator and denominator ## that approximates to `x`, where the denominator is ## smaller than `n` (default is the largest possible @@ -323,8 +323,13 @@ when isMainModule: assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7 assert toInt(z) == 0 - assert toRational(0.98765432) == 2111111029 // 2137499919 - assert toRational(PI) == 817696623 // 260280919 + when sizeof(int) == 8: + assert toRational(0.98765432) == 2111111029 // 2137499919 + assert toRational(PI) == 817696623 // 260280919 + when sizeof(int) == 4: + assert toRational(0.98765432) == 80 // 81 + assert toRational(PI) == 355 // 113 + assert toRational(0.1) == 1 // 10 assert toRational(0.9) == 9 // 10 From ce983383fc1c22e4ade4cbfb591f86723ada51fc Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Wed, 3 Jan 2018 14:42:09 +0300 Subject: [PATCH 134/200] Add a notes about integer casting to the changelog (#6996) --- changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.md b/changelog.md index 1c5848ce89..21ab2b87aa 100644 --- a/changelog.md +++ b/changelog.md @@ -187,3 +187,6 @@ let mySeq = @[1, 2, 1, 3, 1, 4] myCounter = mySeq.toCountTable() ``` + +- Added support for casting between integers of same bitsize in VM (compile time and nimscript). + This allow to among other things to reinterpret signed integers as unsigned. From 23c77ffa3aac532175ec289c5e8c31f4a28f1b2c Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Wed, 3 Jan 2018 06:42:39 -0500 Subject: [PATCH 135/200] Faster nimgrep (#6983) * compile nimgrep with -d:release * nimgrep: only parse pattern once at startup --- koch.nim | 2 +- tools/nimgrep.nim | 46 +++++++++++++++++++--------------------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/koch.nim b/koch.nim index 3ef9a340ab..7bb7ea4024 100644 --- a/koch.nim +++ b/koch.nim @@ -260,7 +260,7 @@ proc buildTools(latest: bool) = " nimsuggest/nimsuggest.nim" let nimgrepExe = "bin/nimgrep".exe - nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim" + nimexec "c -d:release -o:" & nimgrepExe & " tools/nimgrep.nim" when defined(windows): buildVccTool() #nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim" diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index 8dff722ec0..e9c1b26fac 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -45,6 +45,9 @@ type TOptions = set[TOption] TConfirmEnum = enum ceAbort, ceYes, ceAll, ceNo, ceNone + Pattern = Regex | Peg + +using pattern: Pattern var filenames: seq[string] = @[] @@ -118,7 +121,7 @@ proc highlight(s, match, repl: string, t: tuple[first, last: int], stdout.write("\n") stdout.flushFile() -proc processFile(filename: string) = +proc processFile(pattern; filename: string) = var filenameShown = false template beforeHighlight = if not filenameShown and optVerbose notin options: @@ -135,18 +138,8 @@ proc processFile(filename: string) = if optVerbose in options: stdout.writeLine(filename) stdout.flushFile() - var pegp: Peg - var rep: Regex var result: string - if optRegex in options: - if {optIgnoreCase, optIgnoreStyle} * options != {}: - rep = re(pattern, {reExtended, reIgnoreCase}) - else: - rep = re(pattern) - else: - pegp = peg(pattern) - if optReplace in options: result = newStringOfCap(buffer.len) @@ -156,11 +149,7 @@ proc processFile(filename: string) = for j in 0..high(matches): matches[j] = "" var reallyReplace = true while i < buffer.len: - var t: tuple[first, last: int] - if optRegex notin options: - t = findBounds(buffer, pegp, matches, i) - else: - t = findBounds(buffer, rep, matches, i) + let t = findBounds(buffer, pattern, matches, i) if t.first < 0: break inc(line, countLines(buffer, i, t.first-1)) @@ -170,11 +159,7 @@ proc processFile(filename: string) = if optReplace notin options: highlight(buffer, wholeMatch, "", t, line, showRepl=false) else: - var r: string - if optRegex notin options: - r = replace(wholeMatch, pegp, replacement % matches) - else: - r = replace(wholeMatch, rep, replacement % matches) + let r = replace(wholeMatch, pattern, replacement % matches) if optConfirm in options: highlight(buffer, wholeMatch, r, t, line, showRepl=true) case confirm() @@ -246,17 +231,17 @@ proc styleInsensitive(s: string): string = addx() else: addx() -proc walker(dir: string) = +proc walker(pattern; dir: string) = for kind, path in walkDir(dir): case kind of pcFile: if extensions.len == 0 or path.hasRightExt(extensions): - processFile(path) + processFile(pattern, path) of pcDir: if optRecursive in options: - walker(path) + walker(pattern, path) else: discard - if existsFile(dir): processFile(dir) + if existsFile(dir): processFile(pattern, dir) proc writeHelp() = stdout.write(Usage) @@ -332,11 +317,18 @@ else: pattern = "\\y " & pattern elif optIgnoreCase in options: pattern = "\\i " & pattern + let pegp = peg(pattern) + for f in items(filenames): + walker(pegp, f) else: + var reflags = {reStudy, reExtended} if optIgnoreStyle in options: pattern = styleInsensitive(pattern) if optWord in options: pattern = r"\b (:?" & pattern & r") \b" - for f in items(filenames): - walker(f) + if {optIgnoreCase, optIgnoreStyle} * options != {}: + reflags.incl reIgnoreCase + let rep = re(pattern, reflags) + for f in items(filenames): + walker(rep, f) From 22cc7c62e5f1c22e0eda1037a6587153ffa6badf Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 3 Jan 2018 11:46:55 +0000 Subject: [PATCH 136/200] Fixes `times` module compilation on cpp backend. (#7004) --- lib/posix/posix_other.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index e552bf807e..01bc1c1e5e 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -34,7 +34,7 @@ type {.deprecated: [TSocketHandle: SocketHandle].} type - Time* {.importc: "time_t", header: "".} = distinct int + Time* {.importc: "time_t", header: "".} = distinct clong Timespec* {.importc: "struct timespec", header: "", final, pure.} = object ## struct timespec From bbfe6e81ad9a09fc57710809d5325e13a8c95cbe Mon Sep 17 00:00:00 2001 From: Eduardo Bart Date: Wed, 3 Jan 2018 09:56:35 -0200 Subject: [PATCH 137/200] Add newSeqUninitialized, closes #6401 (#6402) --- lib/system.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 85643891ba..4d86107372 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -743,6 +743,18 @@ proc newSeqOfCap*[T](cap: Natural): seq[T] {. ## ``cap``. discard +when not defined(JS): + proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] = + ## creates a new sequence of type ``seq[T]`` with length ``len``. + ## + ## Only available for numbers types. Note that the sequence will be + ## uninitialized. After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + + result = newSeqOfCap[T](len) + var s = cast[PGenericSeq](result) + s.len = len + proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. magic: "LengthOpenArray", noSideEffect.} proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} From e593fef3206c359344c93bf7a3716da644f1906f Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 3 Jan 2018 13:24:20 +0100 Subject: [PATCH 138/200] memfiles: better error checking for Windows; refs #6361 --- lib/pure/memfiles.nim | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 9b2d252675..5c73381ffb 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -257,10 +257,13 @@ proc close*(f: var MemFile) = when defined(windows): if f.wasOpened: error = unmapViewOfFile(f.mem) == 0 - lastErr = osLastError() - error = (closeHandle(f.mapHandle) == 0) or error - if f.fHandle != INVALID_HANDLE_VALUE: - error = (closeHandle(f.fHandle) == 0) or error + if not error: + error = closeHandle(f.mapHandle) == 0 + if not error and f.fHandle != INVALID_HANDLE_VALUE: + discard closeHandle(f.fHandle) + f.fHandle = INVALID_HANDLE_VALUE + if error: + lastErr = osLastError() else: error = munmap(f.mem, f.size) != 0 lastErr = osLastError() From 8bcaadc9e4c10aad17339f9cea69418dd5d2bdf8 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 3 Jan 2018 13:31:03 +0100 Subject: [PATCH 139/200] memfiles: enable test; refs #6361 --- tests/stdlib/tmemfiles2.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim index 665e92e8a2..7ea94cffc5 100644 --- a/tests/stdlib/tmemfiles2.nim +++ b/tests/stdlib/tmemfiles2.nim @@ -1,10 +1,8 @@ discard """ file: "tmemfiles2.nim" - disabled: true output: '''Full read size: 20 Half read size: 10 Data: Hello''' """ -# doesn't work on windows. fmReadWrite doesn't create a file. import memfiles, os var mm, mm_full, mm_half: MemFile From 30d182e5d6a8a64f637b486e507e7e5e4a1662d9 Mon Sep 17 00:00:00 2001 From: GULPF Date: Wed, 3 Jan 2018 23:54:36 +0100 Subject: [PATCH 140/200] Unexport epochday procs (#7024) --- lib/pure/times.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7df1d01782..42e89e7cee 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -176,7 +176,7 @@ proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline. assert monthday <= getDaysInMonth(month, year), $year & "-" & $ord(month) & "-" & $monthday & " is not a valid date" -proc toEpochDay*(monthday: MonthdayRange, month: Month, year: int): int64 = +proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = ## Get the epoch day from a year/month/day date. ## The epoch day is the number of days since 1970/01/01 (it might be negative). assertValidDate monthday, month, year @@ -191,7 +191,7 @@ proc toEpochDay*(monthday: MonthdayRange, month: Month, year: int): int64 = let doe = yoe * 365 + yoe div 4 - yoe div 100 + doy return era * 146097 + doe - 719468 -proc fromEpochDay*(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = +proc fromEpochDay(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = ## Get the year/month/day date from a epoch day. ## The epoch day is the number of days since 1970/01/01 (it might be negative). # Based on http://howardhinnant.github.io/date_algorithms.html From 05e3a06b6e33f96b0a7a102270d3493eea9ebc1f Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 4 Jan 2018 00:19:04 +0100 Subject: [PATCH 141/200] nimbase.h: make 'endif' nesting correct --- lib/nimbase.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index b12d8e34dd..31075bbd2f 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -400,11 +400,11 @@ typedef struct TStringDesc* string; // NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0 #ifndef NAN -#ifndef _HUGE_ENUF -#define _HUGE_ENUF 1e+300 // _HUGE_ENUF*_HUGE_ENUF must overflow -#endif -#define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF)) -#define NAN ((float)(NAN_INFINITY * 0.0F)) +# ifndef _HUGE_ENUF +# define _HUGE_ENUF 1e+300 // _HUGE_ENUF*_HUGE_ENUF must overflow +# endif +# define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF)) +# define NAN ((float)(NAN_INFINITY * 0.0F)) #endif #ifndef INF @@ -482,7 +482,6 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } On disagreement, your C compiler will say something like: "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; -#endif #ifdef __cplusplus # define NIM_EXTERNC extern "C" @@ -509,3 +508,5 @@ extern Libc::Env *genodeEnv; /* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */ #define NIM_CHECK_SIZE(typ, sz) \ _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size") + +#endif /* NIMBASE_H */ From 80fef7c8182f8b1427697ef923266094bb49abb2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 Jan 2018 14:16:33 +0100 Subject: [PATCH 142/200] allocators.nim: minor typo --- lib/core/allocators.nim | 49 +++++++++++++++++ lib/core/refs.nim | 55 +++++++++++++++++++ lib/core/seqs.nim | 117 ++++++++++++++++++++++++++++++++++++++++ lib/core/strs.nim | 111 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 lib/core/allocators.nim create mode 100644 lib/core/refs.nim create mode 100644 lib/core/seqs.nim create mode 100644 lib/core/strs.nim diff --git a/lib/core/allocators.nim b/lib/core/allocators.nim new file mode 100644 index 0000000000..093da296a2 --- /dev/null +++ b/lib/core/allocators.nim @@ -0,0 +1,49 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type + TypeLayout* = object + size*, alignment*: int + destructor*: proc (self: pointer; a: Allocator) {.nimcall.} + trace*: proc (self: pointer; a: Allocator) {.nimcall.} + when false: + construct*: proc (self: pointer; a: Allocator) {.nimcall.} + copy*, deepcopy*, sink*: proc (self, other: pointer; a: Allocator) {.nimcall.} + + Allocator* {.inheritable.} = ptr object + alloc*: proc (a: Allocator; size: int; alignment = 8): pointer {.nimcall.} + dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.} + realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.} + visit*: proc (fieldAddr: ptr pointer; a: Allocator) {.nimcall.} + +#proc allocArray(a: Allocator; L, elem: TypeLayout; n: int): pointer +#proc deallocArray(a: Allocator; p: pointer; L, elem: TypeLayout; n: int) + +proc getTypeLayout*(t: typedesc): ptr TypeLayout {.magic: "getTypeLayout".} + +var + currentAllocator {.threadvar.}: Allocator + +proc getCurrentAllocator*(): Allocator = + result = currentAllocator + +proc setCurrentAllocator*(a: Allocator) = + currentAllocator = a + +proc alloc*(size: int): pointer = + let a = getCurrentAllocator() + result = a.alloc(a, size) + +proc dealloc*(p: pointer; size: int) = + let a = getCurrentAllocator() + a.dealloc(a, size) + +proc realloc*(p: pointer; oldSize, newSize: int): pointer = + let a = getCurrentAllocator() + result = a.realloc(a, oldSize, newSize) diff --git a/lib/core/refs.nim b/lib/core/refs.nim new file mode 100644 index 0000000000..71c999a747 --- /dev/null +++ b/lib/core/refs.nim @@ -0,0 +1,55 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Default ref implementation used by Nim's core. + +import allocators + +type + TracingGc = ptr object of Allocator + + GcHeader = object + t: ptr TypeLayout + + GcFrame {.core.} = object + prev: ptr GcFrame + marker: proc (self: GcFrame; a: Allocator) + +proc `=trace`[T](a: ref T) = + if not marked(a): + mark(a) + `=trace`(a[]) + +proc linkGcFrame(f: ptr GcFrame) {.core.} +proc unlinkGcFrame() {.core.} + +proc setGcFrame(f: ptr GcFrame) {.core.} + +proc registerGlobal(p: pointer; t: ptr TypeLayout) {.core.} +proc unregisterGlobal(p: pointer; t: ptr TypeLayout) {.core.} + +proc registerThreadvar(p: pointer; t: ptr TypeLayout) {.core.} +proc unregisterThreadvar(p: pointer; t: ptr TypeLayout) {.core.} + +proc newImpl(t: ptr TypeLayout): pointer = + let a = getCurrentAllocator() + let r = cast[ptr GcHeader](a.alloc(a, t.size + sizeof(GcHeader), t.alignment)) + r.typ = t + result = r +! sizeof(GcHeader) + +template new*[T](x: var ref T) = + x = newImpl(getTypeLayout(x)) + + +when false: + # implement these if your GC requires them: + proc writeBarrierLocal() {.core.} + proc writeBarrierGlobal() {.core.} + + proc writeBarrierGeneric() {.core.} diff --git a/lib/core/seqs.nim b/lib/core/seqs.nim new file mode 100644 index 0000000000..6be95a3bca --- /dev/null +++ b/lib/core/seqs.nim @@ -0,0 +1,117 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import allocators + +## Default seq implementation used by Nim's core. +type + seq*[T] = object + len, cap: int + data: ptr UncheckedArray[T] + +template frees(s) = dealloc(s.data, s.cap * sizeof(T)) + +# XXX make code memory safe for overflows in '*' +proc nimSeqLiteral[T](x: openArray[T]): seq[T] {.core.} = + seq[T](len: x.len, cap: x.len, data: x) + +when defined(nimHasTrace): + proc `=trace`[T](s: seq[T]; a: Allocator) = + for i in 0 ..< s.len: `=trace`(s.data[i], a) + +proc `=destroy`[T](x: var seq[T]) = + if x.data != nil: + when not supportsCopyMem(T): + for i in 0..= x.cap: resize(x) + result = addr(x.data[x.len]) + inc x.len + +template add*[T](x: var seq[T]; y: T) = + reserveSlot(x)[] = y + +proc shrink*[T](x: var seq[T]; newLen: int) = + assert newLen <= x.len + assert newLen >= 0 + when not supportsCopyMem(T): + for i in countdown(x.len - 1, newLen - 1): + `=destroy`(x.data[i]) + x.len = newLen + +proc grow*[T](x: var seq[T]; newLen: int; value: T) = + if newLen <= x.len: return + assert newLen >= 0 + if x.cap == 0: x.cap = newLen + else: x.cap = max(newLen, (x.cap * 3) shr 1) + x.data = cast[type(x.data)](realloc(x.data, x.cap * sizeof(T))) + for i in x.len..= s.cap: resize(s) + s.data[s.len] = c + s.data[s.len+1] = '\0' + inc s.len + +proc ensure(s: var string; newLen: int) = + let old = s.cap + if newLen >= old: + s.cap = max((old * 3) shr 1, newLen) + if s.cap > 0: + s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1)) + +proc add*(s: var string; y: string) = + if y.len != 0: + let newLen = s.len + y.len + ensure(s, newLen) + copyMem(addr s.data[len], y.data, y.data.len + 1) + s.len = newLen + +proc len*(s: string): int {.inline.} = s.len + +proc newString*(len: int): string = + result.len = len + result.cap = len + if len > 0: + result.data = alloc0(len+1) + +converter toCString(x: string): cstring {.core.} = + if x.len == 0: cstring"" else: cast[cstring](x.data) + +proc newStringOfCap*(cap: int): string = + result.len = 0 + result.cap = cap + if cap > 0: + result.data = alloc(cap+1) + +proc `&`*(a, b: string): string = + let sum = a.len + b.len + result = newStringOfCap(sum) + result.len = sum + copyMem(addr result.data[0], a.data, a.len) + copyMem(addr result.data[a.len], b.data, b.len) + if sum > 0: + result.data[sum] = '\0' + +proc concat(x: openArray[string]): string {.core.} = + ## used be the code generator to optimize 'x & y & z ...' + var sum = 0 + for i in 0 ..< x.len: inc(sum, x[i].len) + result = newStringOfCap(sum) + sum = 0 + for i in 0 ..< x.len: + let L = x[i].len + copyMem(addr result.data[sum], x[i].data, L) + inc(sum, L) + From 2e635ab28cb60be5e63e01ecf074596ccc385696 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 Jan 2018 15:59:03 +0100 Subject: [PATCH 143/200] new runtime: added typelayouts.nim --- lib/core/allocators.nim | 14 ------------ lib/core/refs.nim | 48 +++++++++++++++++++++++++++++++++++++--- lib/core/typelayouts.nim | 19 ++++++++++++++++ 3 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 lib/core/typelayouts.nim diff --git a/lib/core/allocators.nim b/lib/core/allocators.nim index 093da296a2..d6608a2037 100644 --- a/lib/core/allocators.nim +++ b/lib/core/allocators.nim @@ -8,24 +8,10 @@ # type - TypeLayout* = object - size*, alignment*: int - destructor*: proc (self: pointer; a: Allocator) {.nimcall.} - trace*: proc (self: pointer; a: Allocator) {.nimcall.} - when false: - construct*: proc (self: pointer; a: Allocator) {.nimcall.} - copy*, deepcopy*, sink*: proc (self, other: pointer; a: Allocator) {.nimcall.} - Allocator* {.inheritable.} = ptr object alloc*: proc (a: Allocator; size: int; alignment = 8): pointer {.nimcall.} dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.} realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.} - visit*: proc (fieldAddr: ptr pointer; a: Allocator) {.nimcall.} - -#proc allocArray(a: Allocator; L, elem: TypeLayout; n: int): pointer -#proc deallocArray(a: Allocator; p: pointer; L, elem: TypeLayout; n: int) - -proc getTypeLayout*(t: typedesc): ptr TypeLayout {.magic: "getTypeLayout".} var currentAllocator {.threadvar.}: Allocator diff --git a/lib/core/refs.nim b/lib/core/refs.nim index 71c999a747..e1575b68c7 100644 --- a/lib/core/refs.nim +++ b/lib/core/refs.nim @@ -9,23 +9,66 @@ ## Default ref implementation used by Nim's core. -import allocators +# We cannot use the allocator interface here as we require a heap walker to +# exist. Thus we import 'alloc' directly here to get our own heap that is +# all under the GC's control and can use the ``allObjects`` iterator which +# is crucial for the "sweep" phase. +import typelayouts, alloc type TracingGc = ptr object of Allocator + visit*: proc (fieldAddr: ptr pointer; a: Allocator) {.nimcall.} + + GcColor = enum + white = 0, black = 1, grey = 2 ## to flip the meaning of white/black + ## perform (1 - col) GcHeader = object t: ptr TypeLayout + color: GcColor + Cell = ptr GcHeader GcFrame {.core.} = object prev: ptr GcFrame marker: proc (self: GcFrame; a: Allocator) + Phase = enum + None, Marking, Sweeping + + GcHeap = object + r: MemRegion + phase: Phase + currBlack, currWhite: GcColor + greyStack: seq[Cell] + +var + gch {.threadvar.}: GcHeap + proc `=trace`[T](a: ref T) = if not marked(a): mark(a) `=trace`(a[]) +template usrToCell(p: pointer): Cell = + +template cellToUsr(cell: Cell): pointer = + cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(GcHeader))) + +template usrToCell(usr: pointer): Cell = + cast[Cell](cast[ByteAddress](usr)-%ByteAddress(sizeof(GcHeader))) + +template markGrey(x: Cell) = + if x.color == gch.currWhite and phase == Marking: + x.color = grey + add(gch.greyStack, x) + +proc `=`[T](dest: var ref T; src: ref T) = + ## full write barrier implementation. + if src != nil: + let s = usrToCell(src) + markGrey(s) + system.`=`(dest, src) + proc linkGcFrame(f: ptr GcFrame) {.core.} proc unlinkGcFrame() {.core.} @@ -38,8 +81,7 @@ proc registerThreadvar(p: pointer; t: ptr TypeLayout) {.core.} proc unregisterThreadvar(p: pointer; t: ptr TypeLayout) {.core.} proc newImpl(t: ptr TypeLayout): pointer = - let a = getCurrentAllocator() - let r = cast[ptr GcHeader](a.alloc(a, t.size + sizeof(GcHeader), t.alignment)) + let r = cast[Cell](rawAlloc(t.size + sizeof(GcHeader))) r.typ = t result = r +! sizeof(GcHeader) diff --git a/lib/core/typelayouts.nim b/lib/core/typelayouts.nim new file mode 100644 index 0000000000..445ce77c48 --- /dev/null +++ b/lib/core/typelayouts.nim @@ -0,0 +1,19 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type + TypeLayout* = object + size*, alignment*: int + destructor*: proc (self: pointer; a: Allocator) {.nimcall.} + trace*: proc (self: pointer; a: Allocator) {.nimcall.} + when false: + construct*: proc (self: pointer; a: Allocator) {.nimcall.} + copy*, deepcopy*, sink*: proc (self, other: pointer; a: Allocator) {.nimcall.} + +proc getTypeLayout(t: typedesc): ptr TypeLayout {.magic: "getTypeLayout".} From 3ae434a086145768b93405aaa35e58afab451879 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 Jan 2018 21:08:34 +0100 Subject: [PATCH 144/200] symbol files: do not regenerate method dispatchers for now --- compiler/cgen.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 573a14927a..5ea7f84e6d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1443,6 +1443,10 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = result = n if b == nil or passes.skipCodegen(n): return var m = BModule(b) + # if the module is cached, we don't regenerate the main proc + # nor the dispatchers? But if the dispatchers changed? + # XXX emit the dispatchers into its own .c file? + if b.rd != nil: return if n != nil: m.initProc.options = initProcOptions(m) genStmts(m.initProc, n) From 9f943dbc8e929f9df2bb14ad3b6ee9da138a44ac Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Fri, 5 Jan 2018 02:58:42 -0500 Subject: [PATCH 145/200] Don't zeroMem result of boehmAlloc() (#7029) From the man page: "Unlike the standard implementations of malloc, GC_malloc clears the newly allocated storage. GC_malloc_atomic does not." --- lib/system/mmdisp.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 9ac039e192..45e0c74c0e 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -109,7 +109,6 @@ when defined(boehmgc): if result == nil: raiseOutOfMem() proc alloc0(size: Natural): pointer = result = alloc(size) - zeroMem(result, size) proc realloc(p: pointer, newsize: Natural): pointer = result = boehmRealloc(p, newsize) if result == nil: raiseOutOfMem() @@ -119,8 +118,7 @@ when defined(boehmgc): result = boehmAlloc(size) if result == nil: raiseOutOfMem() proc allocShared0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) + result = allocShared(size) proc reallocShared(p: pointer, newsize: Natural): pointer = result = boehmRealloc(p, newsize) if result == nil: raiseOutOfMem() From c344fb311d196dadc652d9cd03e72b321afeae56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Fri, 5 Jan 2018 09:49:46 +0100 Subject: [PATCH 146/200] Allow timezone procs to be closures --- lib/pure/times.nim | 4 ++-- tests/stdlib/ttimes.nim | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7df1d01782..58b3c72237 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -129,8 +129,8 @@ type ## The ``times`` module only supplies implementations for the systems local time and UTC. ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly ## and are only exported so that ``Timezone`` can be implemented by other modules. - zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} - zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign, closure.} + zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign, closure.} name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index a6ac186cc7..1f8ae6a225 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -281,6 +281,30 @@ suite "ttimes": test "parseTest": runTimezoneTests() + test "dynamic timezone": + proc staticOffset(offset: int): Timezone = + proc zoneInfoFromTz(adjTime: Time): ZonedTime = + result.isDst = false + result.utcOffset = offset + result.adjTime = adjTime + + proc zoneInfoFromUtc(time: Time): ZonedTime = + result.isDst = false + result.utcOffset = offset + result.adjTime = fromUnix(time.toUnix - offset) + + result.name = "" + result.zoneInfoFromTz = zoneInfoFromTz + result.zoneInfoFromUtc = zoneInfoFromUtc + + let tz = staticOffset(-9000) + let dt = initDateTime(1, mJan, 2000, 12, 00, 00, tz) + check dt.utcOffset == -9000 + check dt.isDst == false + check $dt == "2000-01-01T12:00:00+02:30" + check $dt.utc == "2000-01-01T09:30:00+00:00" + check $dt.utc.inZone(tz) == $dt + test "isLeapYear": check isLeapYear(2016) check (not isLeapYear(2015)) From c098ee3c7a2dfeedfcc3a3a33ca83b4b1d69df63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Fri, 5 Jan 2018 11:14:31 +0100 Subject: [PATCH 147/200] Remove nimcall pragma from tz procs --- lib/pure/times.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 58b3c72237..3617e59195 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -129,8 +129,8 @@ type ## The ``times`` module only supplies implementations for the systems local time and UTC. ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly ## and are only exported so that ``Timezone`` can be implemented by other modules. - zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign, closure.} - zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign, closure.} + zoneInfoFromUtc*: proc (time: Time): ZonedTime {.tags: [], raises: [], benign.} + zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. From 6cf8a72d498f5cb8a532c2ff3259bc7aecf474ef Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Fri, 5 Jan 2018 19:32:05 +0300 Subject: [PATCH 148/200] Windows: fixes getch bug; fixes #6966 (#7031) --- lib/pure/terminal.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index ef5a95ed24..f15cee66ac 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -634,7 +634,10 @@ proc getch*(): char = doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: continue - return char(keyEvent.uChar) + if keyEvent.uChar == 0: + return char(keyEvent.wVirtualKeyCode) + else: + return char(keyEvent.uChar) else: let fd = getFileHandle(stdin) var oldMode: Termios From 6ca563dd2e6380e4e2062b9129f582a4910baf68 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Fri, 5 Jan 2018 19:36:56 +0300 Subject: [PATCH 149/200] Add a more number parsers to the scanf macro (#6985) --- lib/pure/parseutils.nim | 17 +++++++++++++++++ lib/pure/strscans.nim | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index a602b0e1be..57387e62ed 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -87,6 +87,23 @@ proc parseOct*(s: string, number: var int, start = 0): int {. inc(i) if foundDigit: result = i-start +proc parseBin*(s: string, number: var int, start = 0): int {. + rtl, extern: "npuParseBin", noSideEffect.} = + ## parses an binary number and stores its value in ``number``. Returns + ## the number of the parsed characters or 0 in case of an error. + var i = start + var foundDigit = false + if s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2) + while true: + case s[i] + of '_': discard + of '0'..'1': + number = number shl 1 or (ord(s[i]) - ord('0')) + foundDigit = true + else: break + inc(i) + if foundDigit: result = i-start + proc parseIdent*(s: string, ident: var string, start = 0): int = ## parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index f33e7451fc..42bb281eb1 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -31,7 +31,10 @@ As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: ================= ======================================================== -``$i`` Matches an integer. This uses ``parseutils.parseInt``. +``$b`` Matches an decimal integer. This uses ``parseutils.parseBin``. +``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. +``$i`` Matches an decimal integer. This uses ``parseutils.parseInt``. +``$h`` Matches an hex integer. This uses ``parseutils.parseHex``. ``$f`` Matches a floating pointer number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``. ``$s`` Skips optional whitespace. @@ -335,11 +338,29 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b else: error("no string var given for $w") inc i + of 'b': + if i < results.len or getType(results[i]).typeKind != ntyInt: + matchBind "parseBin" + else: + error("no int var given for $b") + inc i + of 'o': + if i < results.len or getType(results[i]).typeKind != ntyInt: + matchBind "parseOct" + else: + error("no int var given for $o") + inc i of 'i': if i < results.len or getType(results[i]).typeKind != ntyInt: matchBind "parseInt" else: - error("no int var given for $d") + error("no int var given for $i") + inc i + of 'h': + if i < results.len or getType(results[i]).typeKind != ntyInt: + matchBind "parseHex" + else: + error("no int var given for $h") inc i of 'f': if i < results.len or getType(results[i]).typeKind != ntyFloat: @@ -645,6 +666,14 @@ when isMainModule: doAssert intval == 89 doAssert floatVal == 33.25 + var binval: int + var octval: int + var hexval: int + doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) + doAssert binval == 0b0101 + doAssert octval == 0o1234 + doAssert hexval == 0xabcd + let xx = scanf("$abc", "$$$i", intval) doAssert xx == false From 9bc263839904787e68602dc55ddbd9832c1066b6 Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Sat, 6 Jan 2018 09:08:58 -0500 Subject: [PATCH 150/200] Fix typos in scanf docs (#7035) --- lib/pure/strscans.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 42bb281eb1..83a82dd0bf 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -31,10 +31,10 @@ As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: ================= ======================================================== -``$b`` Matches an decimal integer. This uses ``parseutils.parseBin``. +``$b`` Matches a binary integer. This uses ``parseutils.parseBin``. ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. -``$i`` Matches an decimal integer. This uses ``parseutils.parseInt``. -``$h`` Matches an hex integer. This uses ``parseutils.parseHex``. +``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. +``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. ``$f`` Matches a floating pointer number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``. ``$s`` Skips optional whitespace. From 06e68feadb4becb653e5e55fd2ef85e1eafd9ba4 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 6 Jan 2018 17:27:19 +0100 Subject: [PATCH 151/200] strscans: fix the type checking logic; improve the documentation --- lib/pure/strscans.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 83a82dd0bf..2bd87837ff 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -333,37 +333,37 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add resLen.notZero conds.add resLen of 'w': - if i < results.len or getType(results[i]).typeKind != ntyString: + if i < results.len and getType(results[i]).typeKind == ntyString: matchBind "parseIdent" else: error("no string var given for $w") inc i of 'b': - if i < results.len or getType(results[i]).typeKind != ntyInt: + if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseBin" else: error("no int var given for $b") inc i of 'o': - if i < results.len or getType(results[i]).typeKind != ntyInt: + if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseOct" else: error("no int var given for $o") inc i of 'i': - if i < results.len or getType(results[i]).typeKind != ntyInt: + if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseInt" else: error("no int var given for $i") inc i of 'h': - if i < results.len or getType(results[i]).typeKind != ntyInt: + if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseHex" else: error("no int var given for $h") inc i of 'f': - if i < results.len or getType(results[i]).typeKind != ntyFloat: + if i < results.len and getType(results[i]).typeKind == ntyFloat: matchBind "parseFloat" else: error("no float var given for $f") @@ -378,7 +378,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b else: error("invalid format string") of '*', '+': - if i < results.len or getType(results[i]).typeKind != ntyString: + if i < results.len and getType(results[i]).typeKind == ntyString: var min = ord(pattern[p] == '+') var q=p+1 var token = "" @@ -462,7 +462,7 @@ template success*(x: int): bool = x != 0 template nxt*(input: string; idx, step: int = 1) = inc(idx, step) macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = - ## See top level documentation of his module of how ``scanp`` works. + ## ``scanp`` is currently undocumented. type StmtTriple = tuple[init, cond, action: NimNode] template interf(x): untyped = bindSym(x, brForceOpen) From e316665b7b25b3e45a66766260bfa645c92beb97 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 7 Jan 2018 10:17:19 +0100 Subject: [PATCH 152/200] work in progress: 'sink' and 'lent' types --- compiler/ast.nim | 4 ++-- compiler/semstmts.nim | 4 +++- compiler/semtypes.nim | 5 +++++ compiler/types.nim | 4 ++-- lib/system.nim | 4 ++++ 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 27a44c6c2b..f5114feb03 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -354,7 +354,7 @@ type tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, - tyOptAsRef, tyUnused1, tyUnused2, + tyOptAsRef, tySink, tyLent, tyVarargs, tyUnused, tyProxy # used as errornous type (for idetools) @@ -640,7 +640,7 @@ type mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, mNimvm, mIntDefine, mStrDefine, mRunnableExamples, - mException + mException, mBultinType # things that we can evaluate safely at compile time, even if not asked for it: const diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index dcaa0263b0..d8754a9315 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -739,7 +739,9 @@ proc semRaise(c: PContext, n: PNode): PNode = if base.sym.magic == mException: break if base.lastSon == nil: - localError(n.info, "raised object of type $1 does not inherit from Exception", [typ.sym.name.s]) + localError(n.info, + "raised object of type $1 does not inherit from Exception", + [typeToString(typ)]) return base = base.lastSon diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index cb66685b2f..b0171fcbe0 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1618,6 +1618,11 @@ proc processMagicType(c: PContext, m: PSym) = of mPNimrodNode: incl m.typ.flags, tfTriggersCompileTime of mException: discard + of mBuiltinType: + case m.name.s + of "lent": setMagicType(m, tyLent, ptrSize) + of "sink": setMagicType(m, tySink, 0) + else: localError(m.info, errTypeExpected) else: localError(m.info, errTypeExpected) proc semGenericConstraints(c: PContext, x: PType): PType = diff --git a/compiler/types.nim b/compiler/types.nim index 495a1977d8..0d3a25980d 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -988,11 +988,11 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = sameTypeOrNilAux(a.sons[0], b.sons[0], c) and sameValue(a.n.sons[0], b.n.sons[0]) and sameValue(a.n.sons[1], b.n.sons[1]) - of tyGenericInst, tyAlias, tyInferred: + of tyGenericInst, tyAlias, tyInferred, tyLent, tySink: cycleCheck() result = sameTypeAux(a.lastSon, b.lastSon, c) of tyNone: result = false - of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("sameFlags") + of tyUnused, tyOptAsRef: internalError("sameFlags") proc sameBackendType*(x, y: PType): bool = var c = initSameTypeClosure() diff --git a/lib/system.nim b/lib/system.nim index 4d86107372..de91c4dda2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -249,6 +249,10 @@ type when defined(nimHasOpt): type opt*{.magic: "Opt".}[T] +when defined(nimHasSink): + type sink*{.magic: "BuiltinType".}[T] + type lent*{.magic: "BuiltinType".}[T] + proc high*[T: Ordinal](x: T): T {.magic: "High", noSideEffect.} ## returns the highest possible index of an array, a sequence, a string or ## the highest possible value of an ordinal value `x`. As a special From 08af53032b731fb999cb24c198c8723977316938 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Jan 2018 12:20:00 +0100 Subject: [PATCH 153/200] net.nim: minor documentation update --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 15f2c1228e..aad6ab3e82 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1135,11 +1135,11 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## ## When 0 is returned the socket's connection has been closed. ## - ## This function will throw an EOS exception when an error occurs. A value + ## This function will throw an OSError exception when an error occurs. A value ## lower than 0 is never returned. ## ## A timeout may be specified in milliseconds, if enough data is not received - ## within the time specified an ETimeout exception will be raised. + ## within the time specified an TimeoutError exception will be raised. ## ## **Note**: ``data`` must be initialised. ## From 51c45c72014381ebca7fce83559d48f28557b9de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Jan 2018 12:21:42 +0100 Subject: [PATCH 154/200] symbol files: introduce more switches for debugging --- compiler/ccgmerge.nim | 2 +- compiler/cgen.nim | 1 - compiler/commands.nim | 9 +++++++-- compiler/main.nim | 2 +- compiler/options.nim | 10 ++++++++-- compiler/rodread.nim | 7 +++---- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 58a03ecd2a..f667be70f2 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -96,7 +96,7 @@ proc writeIntSet(a: IntSet, s: var string) = s.add('}') proc genMergeInfo*(m: BModule): Rope = - if optSymbolFiles notin gGlobalOptions: return nil + if not compilationCachePresent: return nil var s = "/*\tNIM_merge_INFO:" s.add(tnl) s.add("typeCache:{") diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5ea7f84e6d..630426cfd1 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1336,7 +1336,6 @@ proc getCFile(m: BModule): string = proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = injectG(graph.config) - assert optSymbolFiles in gGlobalOptions var m = newModule(g, module) readMergeInfo(getCFile(m), m) result = m diff --git a/compiler/commands.nim b/compiler/commands.nim index 386d7bda86..2d9f769597 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -261,7 +261,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "assertions", "a": result = contains(gOptions, optAssert) of "deadcodeelim": result = contains(gGlobalOptions, optDeadCodeElim) of "run", "r": result = contains(gGlobalOptions, optRun) - of "symbolfiles": result = contains(gGlobalOptions, optSymbolFiles) + of "symbolfiles": result = gSymbolFiles != disabledSf of "genscript": result = contains(gGlobalOptions, optGenScript) of "threads": result = contains(gGlobalOptions, optThreads) of "taintmode": result = contains(gGlobalOptions, optTaintMode) @@ -598,7 +598,12 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(switch, arg, pass, info) helpOnError(pass) of "symbolfiles": - processOnOffSwitchG({optSymbolFiles}, arg, pass, info) + case arg.normalize + of "on": gSymbolFiles = enabledSf + of "off": gSymbolFiles = disabledSf + of "writeonly": gSymbolFiles = writeOnlySf + of "readonly": gSymbolFiles = readOnlySf + else: localError(info, errOnOrOffExpectedButXFound, arg) of "skipcfg": expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optSkipConfigFile) diff --git a/compiler/main.nim b/compiler/main.nim index 08fc4b138a..9bf8bb7c08 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -21,7 +21,7 @@ import from magicsys import systemModule, resetSysTypes proc rodPass = - if optSymbolFiles in gGlobalOptions: + if gSymbolFiles in {enabledSf, writeOnlySf}: registerPass(rodwritePass) proc codegenPass = diff --git a/compiler/options.nim b/compiler/options.nim index 8c4fe485eb..0732e49896 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -48,7 +48,6 @@ type # please make sure we have under 32 options optGenScript, # generate a script file to compile the *.c files optGenMapping, # generate a mapping file optRun, # run the compiled project - optSymbolFiles, # use symbol files for speeding up compilation optCaasEnabled # compiler-as-a-service is running optSkipConfigFile, # skip the general config file optSkipProjConfigFile, # skip the project's config file @@ -147,12 +146,19 @@ var newDestructors*: bool gDynlibOverrideAll*: bool +type + SymbolFilesOption* = enum + disabledSf, enabledSf, writeOnlySf, readOnlySf + +var gSymbolFiles*: SymbolFilesOption + proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools} proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc template preciseStack*(): bool = gPreciseStack template compilationCachePresent*: untyped = - {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {} + gSymbolFiles in {enabledSf, writeOnlySf} +# {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {} template optPreserveOrigSource*: untyped = optEmbedOrigSrc in gGlobalOptions diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 83765c1b73..dfa8fc52ba 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -861,12 +861,11 @@ proc loadMethods(r: PRodReader) = if r.s[r.pos] == ' ': inc(r.pos) proc getHash*(fileIdx: int32): SecureHash = - internalAssert fileIdx >= 0 and fileIdx < gMods.len - - if gMods[fileIdx].hashDone: + if fileIdx <% gMods.len and gMods[fileIdx].hashDone: return gMods[fileIdx].hash result = secureHashFile(fileIdx.toFullPath) + if fileIdx >= gMods.len: setLen(gMods, fileIdx+1) gMods[fileIdx].hash = result template growCache*(cache, pos) = @@ -912,7 +911,7 @@ proc checkDep(fileIdx: int32; cache: IdentCache): TReasonForRecompile = proc handleSymbolFile*(module: PSym; cache: IdentCache): PRodReader = let fileIdx = module.fileIdx - if optSymbolFiles notin gGlobalOptions: + if gSymbolFiles in {disabledSf, writeOnlySf}: module.id = getID() return nil idgen.loadMaxIds(options.gProjectPath / options.gProjectName) From e4081a720190dfdeb347442cdc2c01745476ff9c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Jan 2018 23:09:26 +0100 Subject: [PATCH 155/200] preparations for language extensions: 'sink' and 'lent' types --- compiler/aliases.nim | 2 +- compiler/ast.nim | 10 +++---- compiler/ccgcalls.nim | 8 +++--- compiler/ccgexprs.nim | 22 ++++++++-------- compiler/ccgtrav.nim | 3 ++- compiler/ccgtypes.nim | 26 +++++++++--------- compiler/ccgutils.nim | 4 +-- compiler/cgmeth.nim | 2 +- compiler/destroyer.nim | 8 +++--- compiler/evalffi.nim | 18 ++++++------- compiler/jsgen.nim | 36 +++++++++++++------------ compiler/jstypes.nim | 6 ++--- compiler/lambdalifting.nim | 2 +- compiler/lowerings.nim | 4 +-- compiler/renderer.nim | 4 +-- compiler/sem.nim | 4 +-- compiler/semasgn.nim | 6 ++--- compiler/semcall.nim | 2 +- compiler/semdata.nim | 10 +++---- compiler/semexprs.nim | 54 +++++++++++++++++++------------------- compiler/semfold.nim | 2 +- compiler/semmagic.nim | 4 +-- compiler/semobjconstr.nim | 4 +-- compiler/semstmts.nim | 30 ++++++++++----------- compiler/semtypes.nim | 20 +++++++------- compiler/semtypinst.nim | 6 ++--- compiler/suggest.nim | 8 +++--- compiler/transf.nim | 6 ++--- compiler/trees.nim | 2 +- compiler/types.nim | 32 +++++++++++----------- compiler/vmdeps.nim | 4 ++- compiler/vmgen.nim | 6 ++--- compiler/vmmarshal.nim | 4 +-- lib/system.nim | 2 +- 34 files changed, 183 insertions(+), 178 deletions(-) diff --git a/compiler/aliases.nim b/compiler/aliases.nim index cd7e7f19ae..490ac987a7 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -49,7 +49,7 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult = if a.sons[0] != nil: result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker) if result == arNo: result = isPartOfAux(a.n, b, marker) - of tyGenericInst, tyDistinct, tyAlias: + of tyGenericInst, tyDistinct, tyAlias, tySink: result = isPartOfAux(lastSon(a), b, marker) of tyArray, tySet, tyTuple: for i in countup(0, sonsLen(a) - 1): diff --git a/compiler/ast.nim b/compiler/ast.nim index f5114feb03..54c33a038e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -640,7 +640,7 @@ type mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, mNimvm, mIntDefine, mStrDefine, mRunnableExamples, - mException, mBultinType + mException, mBuiltinType # things that we can evaluate safely at compile time, even if not asked for it: const @@ -940,13 +940,13 @@ const tyGenericParam} StructuralEquivTypes*: TTypeKinds = {tyNil, tyTuple, tyArray, - tySet, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc, tyOpenArray, + tySet, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyOpenArray, tyVarargs} ConcreteTypes*: TTypeKinds = { # types of the expr that may occur in:: # var x = expr tyBool, tyChar, tyEnum, tyArray, tyObject, - tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc, + tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyPointer, tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64} @@ -1427,7 +1427,7 @@ proc propagateToOwner*(owner, elem: PType) = owner.flags.incl tfHasMeta if tfHasAsgn in elem.flags: - let o2 = owner.skipTypes({tyGenericInst, tyAlias}) + let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink}) if o2.kind in {tyTuple, tyObject, tyArray, tySequence, tyOpt, tySet, tyDistinct}: o2.flags.incl tfHasAsgn @@ -1435,7 +1435,7 @@ proc propagateToOwner*(owner, elem: PType) = if owner.kind notin {tyProc, tyGenericInst, tyGenericBody, tyGenericInvocation, tyPtr}: - let elemB = elem.skipTypes({tyGenericInst, tyAlias}) + let elemB = elem.skipTypes({tyGenericInst, tyAlias, tySink}) if elemB.isGCedMem or tfHasGCedMem in elemB.flags: # for simplicity, we propagate this flag even to generics. We then # ensure this doesn't bite us in sempass2. diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index d4fad041d9..db2a9ddc91 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -71,7 +71,7 @@ proc isInCurrentFrame(p: BProc, n: PNode): bool = if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil: result = p.prc.id == n.sym.owner.id of nkDotExpr, nkBracketExpr: - if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyPtr,tyRef}: + if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}: result = isInCurrentFrame(p, n.sons[0]) of nkHiddenStdConv, nkHiddenSubConv, nkConv: result = isInCurrentFrame(p, n.sons[1]) @@ -331,7 +331,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = # skip the deref: var ri = ri[i] while ri.kind == nkObjDownConv: ri = ri[0] - let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias}) + let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink}) if t.kind == tyVar: let x = if ri.kind == nkHiddenAddr: ri[0] else: ri if x.typ.kind == tyPtr: @@ -527,7 +527,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = line(p, cpsStmts, pl) proc genCall(p: BProc, e: PNode, d: var TLoc) = - if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure: + if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure: genClosureCall(p, nil, e, d) elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags: genInfixCall(p, nil, e, d) @@ -538,7 +538,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = postStmtActions(p) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = - if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure: + if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags: genInfixCall(p, le, ri, d) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5a25a98530..0a312b4c72 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -149,7 +149,7 @@ proc getStorageLoc(n: PNode): TStorageLoc = else: result = OnUnknown of nkDerefExpr, nkHiddenDeref: case n.sons[0].typ.kind - of tyVar: result = OnUnknown + of tyVar, tyLent: result = OnUnknown of tyPtr: result = OnStack of tyRef: result = OnHeap else: internalError(n.info, "getStorageLoc") @@ -368,7 +368,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = else: linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCString, - tyInt..tyUInt64, tyRange, tyVar: + tyInt..tyUInt64, tyRange, tyVar, tyLent: linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) else: internalError("genAssignment: " & $ty.kind) @@ -1707,7 +1707,7 @@ proc genRangeChck(p: BProc, n: PNode, d: var TLoc, magic: string) = rope(magic)]), a.storage) proc genConv(p: BProc, e: PNode, d: var TLoc) = - let destType = e.typ.skipTypes({tyVar, tyGenericInst, tyAlias}) + let destType = e.typ.skipTypes({tyVar, tyGenericInst, tyAlias, tySink}) if sameBackendType(destType, e.sons[1].typ): expr(p, e.sons[1], d) else: @@ -1783,7 +1783,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = "$# = #subInt64($#, $#);$n"] const fun: array[mInc..mDec, string] = ["$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n"] - let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tyVar, tyRange}) + let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyRange}) if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}: binaryStmt(p, e, d, opr[op]) else: @@ -1793,7 +1793,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = initLocExpr(p, e.sons[1], a) initLocExpr(p, e.sons[2], b) - let ranged = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tyVar}) + let ranged = skipTypes(e.sons[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent}) let res = binaryArithOverflowRaw(p, ranged, a, b, if underlying.kind == tyInt64: fun64[op] else: fun[op]) putIntoDest(p, a, e.sons[1], "($#)($#)" % [ @@ -2021,9 +2021,9 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) = var r = rdLoc(a) var nilCheck: Rope = nil var t = skipTypes(a.t, abstractInst) - while t.kind in {tyVar, tyPtr, tyRef}: - if t.kind != tyVar: nilCheck = r - if t.kind != tyVar or not p.module.compileToCpp: + while t.kind in {tyVar, tyLent, tyPtr, tyRef}: + if t.kind notin {tyVar, tyLent}: nilCheck = r + if t.kind notin {tyVar, tyLent} or not p.module.compileToCpp: r = "(*$1)" % [r] t = skipTypes(t.lastSon, abstractInst) if not p.module.compileToCpp: @@ -2056,7 +2056,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = var a: TLoc initLocExpr(p, arg, a) var r = rdLoc(a) - let isRef = skipTypes(arg.typ, abstractInst).kind in {tyRef, tyPtr, tyVar} + let isRef = skipTypes(arg.typ, abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent} if isRef: add(r, "->Sup") else: @@ -2068,7 +2068,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = # (see bug #837). However sometimes using a temporary is not correct: # init(TFigure(my)) # where it is passed to a 'var TFigure'. We test # this by ensuring the destination is also a pointer: - if d.k == locNone and skipTypes(n.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}: + if d.k == locNone and skipTypes(n.typ, abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}: getTemp(p, n.typ, d) linefmt(p, cpsStmts, "$1 = &$2;$n", rdLoc(d), r) else: @@ -2307,7 +2307,7 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope = of tyBool: result = rope"NIM_FALSE" of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0" of tyFloat..tyFloat128: result = rope"0.0" - of tyCString, tyString, tyVar, tyPointer, tyPtr, tySequence, tyExpr, + of tyCString, tyString, tyVar, tyLent, tyPointer, tyPtr, tySequence, tyExpr, tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil: result = rope"NIM_NIL" of tyProc: diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 275c2ddb62..ad5d1c95f5 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -66,7 +66,8 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) = var p = c.p case typ.kind - of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred: + of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred, + tySink: genTraverseProc(c, accessor, lastSon(typ)) of tyArray: let arraySize = lengthOrd(typ.sons[0]) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index c9cd3b1250..24d3a0dfb4 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -119,7 +119,7 @@ proc scopeMangledParam(p: BProc; param: PSym) = const irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation, - tyDistinct, tyRange, tyStatic, tyAlias, tyInferred} + tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred} proc typeName(typ: PType): Rope = let typ = typ.skipTypes(irrelevantForBackend) @@ -139,7 +139,7 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope = t = t.lastSon else: break - let typ = if typ.kind == tyAlias: typ.lastSon else: typ + let typ = if typ.kind in {tyAlias, tySink}: typ.lastSon else: typ if typ.loc.r == nil: typ.loc.r = typ.typeName & $sig else: @@ -170,7 +170,7 @@ proc mapType(typ: PType): TCTypeKind = internalAssert typ.isResolvedUserTypeClass return mapType(typ.lastSon) of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal, - tyTypeDesc, tyAlias, tyInferred: + tyTypeDesc, tyAlias, tySink, tyInferred: result = mapType(lastSon(typ)) of tyEnum: if firstOrd(typ) < 0: @@ -183,7 +183,7 @@ proc mapType(typ: PType): TCTypeKind = of 8: result = ctInt64 else: internalError("mapType") of tyRange: result = mapType(typ.sons[0]) - of tyPtr, tyVar, tyRef, tyOptAsRef: + of tyPtr, tyVar, tyLent, tyRef, tyOptAsRef: var base = skipTypes(typ.lastSon, typedescInst) case base.kind of tyOpenArray, tyArray, tyVarargs: result = ctPtrToArray @@ -242,7 +242,7 @@ proc isInvalidReturnType(rettype: PType): bool = case mapType(rettype) of ctArray: result = not (skipTypes(rettype, typedescInst).kind in - {tyVar, tyRef, tyPtr}) + {tyVar, tyLent, tyRef, tyPtr}) of ctStruct: let t = skipTypes(rettype, typedescInst) if rettype.isImportedCppType or t.isImportedCppType: return false @@ -328,7 +328,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyStatic: if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ) else: internalError("tyStatic for getSimpleTypeDesc") - of tyGenericInst, tyAlias: + of tyGenericInst, tyAlias, tySink: result = getSimpleTypeDesc(m, lastSon typ) else: result = nil @@ -348,7 +348,7 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope = if result == nil: result = cacheGetType(m.typeCache, sig) proc structOrUnion(t: PType): Rope = - let t = t.skipTypes({tyAlias}) + let t = t.skipTypes({tyAlias, tySink}) (if tfUnion in t.flags: rope("union") else: rope("struct")) proc getForwardStructFormat(m: BModule): string = @@ -396,7 +396,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope = result = getTypeDescAux(m, t, check) proc paramStorageLoc(param: PSym): TStorageLoc = - if param.typ.skipTypes({tyVar, tyTypeDesc}).kind notin { + if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin { tyArray, tyOpenArray, tyVarargs}: result = OnStack else: @@ -430,11 +430,11 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, add(params, param.loc.r) # declare the len field for open arrays: var arr = param.typ - if arr.kind == tyVar: arr = arr.sons[0] + if arr.kind in {tyVar, tyLent}: arr = arr.lastSon var j = 0 while arr.kind in {tyOpenArray, tyVarargs}: # this fixes the 'sort' bug: - if param.typ.kind == tyVar: param.loc.storage = OnUnknown + if param.typ.kind in {tyVar, tyLent}: param.loc.storage = OnUnknown # need to pass hidden parameter: addf(params, ", NI $1Len_$2", [param.loc.r, j.rope]) inc(j) @@ -641,7 +641,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope = excl(check, t.id) return case t.kind - of tyRef, tyOptAsRef, tyPtr, tyVar: + of tyRef, tyOptAsRef, tyPtr, tyVar, tyLent: var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and compileToCpp(m): "&" else: "*" var et = origTyp.skipTypes(abstractInst).lastSon @@ -872,7 +872,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope = of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)]) else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", [result, rope(getSize(t))]) - of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, + of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyUserTypeClass, tyUserTypeClassInst, tyInferred: result = getTypeDescAux(m, lastSon(t), check) else: @@ -1227,7 +1227,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = m.g.typeInfoMarker[sig] = result case t.kind of tyEmpty, tyVoid: result = rope"0" - of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar: + of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent: genTypeInfoAuxBase(m, t, t, result, rope"0", info) of tyStatic: if t.n != nil: result = genTypeInfo(m, lastSon t, info) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index b1a268c9ee..fe28d2209b 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -110,13 +110,13 @@ proc getUniqueType*(key: PType): PType = of tyDistinct: if key.deepCopy != nil: result = key else: result = getUniqueType(lastSon(key)) - of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tyInferred: + of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tySink, tyInferred: result = getUniqueType(lastSon(key)) #let obj = lastSon(key) #if obj.sym != nil and obj.sym.name.s == "TOption": # echo "for ", typeToString(key), " I returned " # debug result - of tyPtr, tyRef, tyVar: + of tyPtr, tyRef, tyVar, tyLent: let elemType = lastSon(key) if elemType.kind in {tyBool, tyChar, tyInt..tyUInt64}: # no canonicalization for integral types, so that e.g. ``ptr pid_t`` is diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 6f7d9f4895..0513e88f42 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -68,7 +68,7 @@ proc sameMethodBucket(a, b: PSym): MethodResult = while true: aa = skipTypes(aa, {tyGenericInst, tyAlias}) bb = skipTypes(bb, {tyGenericInst, tyAlias}) - if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef}: + if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent}: aa = aa.lastSon bb = bb.lastSon else: diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 0fdeceba0b..55da69985a 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -174,7 +174,7 @@ proc patchHead(n: PNode) = if n[1].typ.isNil: # XXX toptree crashes without this workaround. Figure out why. return - let t = n[1].typ.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred}) + let t = n[1].typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred}) template patch(op, field) = if s.name.s == op and field != nil and field != s: n.sons[0].sym = field @@ -198,15 +198,15 @@ template genOp(opr, opname) = result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest)) proc genSink(t: PType; dest: PNode): PNode = - let t = t.skipTypes({tyGenericInst, tyAlias}) + let t = t.skipTypes({tyGenericInst, tyAlias, tySink}) genOp(if t.sink != nil: t.sink else: t.assignment, "=sink") proc genCopy(t: PType; dest: PNode): PNode = - let t = t.skipTypes({tyGenericInst, tyAlias}) + let t = t.skipTypes({tyGenericInst, tyAlias, tySink}) genOp(t.assignment, "=") proc genDestroy(t: PType; dest: PNode): PNode = - let t = t.skipTypes({tyGenericInst, tyAlias}) + let t = t.skipTypes({tyGenericInst, tyAlias, tySink}) genOp(t.destructor, "=destroy") proc addTopVar(c: var Con; v: PNode) = diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 51b65258b5..5bf8f358a5 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -86,10 +86,10 @@ proc mapType(t: ast.PType): ptr libffi.TType = else: result = nil of tyFloat, tyFloat64: result = addr libffi.type_double of tyFloat32: result = addr libffi.type_float - of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr, + of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr, tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil: result = addr libffi.type_pointer - of tyDistinct, tyAlias: + of tyDistinct, tyAlias, tySink: result = mapType(t.sons[0]) else: result = nil @@ -112,12 +112,12 @@ template `+!`(x, y: untyped): untyped = proc packSize(v: PNode, typ: PType): int = ## computes the size of the blob case typ.kind - of tyPtr, tyRef, tyVar: + of tyPtr, tyRef, tyVar, tyLent: if v.kind in {nkNilLit, nkPtrLit}: result = sizeof(pointer) else: result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon) - of tyDistinct, tyGenericInst, tyAlias: + of tyDistinct, tyGenericInst, tyAlias, tySink: result = packSize(v, typ.sons[0]) of tyArray: # consider: ptr array[0..1000_000, int] which is common for interfacing; @@ -209,7 +209,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = awr(cstring, cstring(v.strVal)) else: globalError(v.info, "cannot map pointer/proc value to FFI") - of tyPtr, tyRef, tyVar: + of tyPtr, tyRef, tyVar, tyLent: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway discard @@ -231,7 +231,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = packObject(v, typ, res) of tyNil: discard - of tyDistinct, tyGenericInst, tyAlias: + of tyDistinct, tyGenericInst, tyAlias, tySink: pack(v, typ.sons[0], res) else: globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) @@ -364,7 +364,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = result = n else: awi(nkPtrLit, cast[ByteAddress](p)) - of tyPtr, tyRef, tyVar: + of tyPtr, tyRef, tyVar, tyLent: let p = rd(pointer, x) if p.isNil: setNil() @@ -388,14 +388,14 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = aws(nkStrLit, $p) of tyNil: setNil() - of tyDistinct, tyGenericInst, tyAlias: + of tyDistinct, tyGenericInst, tyAlias, tySink: result = unpack(x, typ.lastSon, n) else: # XXX what to do with 'array' here? globalError(n.info, "cannot map value from FFI " & typeToString(typ)) proc fficast*(x: PNode, destTyp: PType): PNode = - if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer, + if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer, tyProc, tyCString, tyString, tySequence}: result = newNodeIT(x.kind, x.info, destTyp) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index dac2de7464..ac1a4b5d50 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -173,7 +173,7 @@ const proc mapType(typ: PType): TJSTypeKind = let t = skipTypes(typ, abstractInst) case t.kind - of tyVar, tyRef, tyPtr: + of tyVar, tyRef, tyPtr, tyLent: if skipTypes(t.lastSon, abstractInst).kind in MappedToObject: result = etyObject else: @@ -196,14 +196,15 @@ proc mapType(typ: PType): TJSTypeKind = tyExpr, tyStmt, tyTypeDesc, tyBuiltInTypeClass, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, tyVoid: result = etyNone - of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst: + of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst, + tySink: result = mapType(typ.lastSon) of tyStatic: if t.n != nil: result = mapType(lastSon t) else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString - of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("mapType") + of tyUnused, tyOptAsRef: internalError("mapType") proc mapType(p: PProc; typ: PType): TJSTypeKind = if p.target == targetPHP: result = etyObject @@ -869,8 +870,8 @@ proc generateHeader(p: PProc, typ: PType): Rope = add(result, name) add(result, "_Idx") elif not (i == 1 and param.name.s == "this"): - let k = param.typ.skipTypes({tyGenericInst, tyAlias}).kind - if k in {tyVar, tyRef, tyPtr, tyPointer}: + let k = param.typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind + if k in {tyVar, tyRef, tyPtr, tyLent, tyPointer}: add(result, "&") add(result, "$") add(result, name) @@ -899,7 +900,7 @@ const proc needsNoCopy(p: PProc; y: PNode): bool = result = (y.kind in nodeKindsNeedNoCopy) or - (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}) or + (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyLent, tyVar}) or p.target == targetPHP proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = @@ -1077,7 +1078,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) = proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) = var ty = skipTypes(n.sons[0].typ, abstractVarRange) - if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.lastSon, abstractVarRange) + if ty.kind in {tyRef, tyPtr, tyLent}: ty = skipTypes(ty.lastSon, abstractVarRange) case ty.kind of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs: genArrayAddr(p, n, r) @@ -1300,7 +1301,7 @@ proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes; emitted: ptr int = add(r.res, ", ") add(r.res, a.res) if emitted != nil: inc emitted[] - elif n.typ.kind == tyVar and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex: + elif n.typ.kind in {tyVar, tyLent} and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex: # this fixes bug #5608: let tmp = getTemp(p) add(r.res, "($1 = $2, $1[0]), $1[1]" % [tmp, a.rdLoc]) @@ -1499,7 +1500,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("0", indirect) of tyFloat..tyFloat128: result = putToSeq("0.0", indirect) - of tyRange, tyGenericInst, tyAlias: + of tyRange, tyGenericInst, tyAlias, tySink: result = createVar(p, lastSon(typ), indirect) of tySet: result = putToSeq("{}" | "array()", indirect) @@ -1546,7 +1547,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = createObjInitList(p, t, initIntSet(), initList) result = ("{$1}" | "array($#)") % [initList] if indirect: result = "[$1]" % [result] - of tyVar, tyPtr, tyRef: + of tyVar, tyPtr, tyLent, tyRef: if mapType(p, t) == etyBaseIndex: result = putToSeq("[null, 0]", indirect) else: @@ -1579,7 +1580,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = let mname = mangleName(v, p.target) lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n", [returnType, mname, createVar(p, v.typ, isIndirect(v))]) - if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex: + if v.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and mapType(p, v.typ) == etyBaseIndex: lineF(p, "var $1_Idx = 0;$n", [ mname ]) else: discard mangleName(v, p.target) @@ -1774,7 +1775,7 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) = proc genOf(p: PProc, n: PNode, r: var TCompRes) = var x: TCompRes - let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyTypeDesc}) + let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyLent, tyTypeDesc}) gen(p, n.sons[1], x) if tfFinal in t.flags: r.res = "($1.m_type == $2)" % [x.res, genTypeInfo(p, t)] @@ -2161,7 +2162,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = let mname = mangleName(resultSym, p.target) let resVar = createVar(p, resultSym.typ, isIndirect(resultSym)) resultAsgn = p.indentLine(("var $# = $#;$n" | "$$$# = $#;$n") % [mname, resVar]) - if resultSym.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, resultSym.typ) == etyBaseIndex: + if resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and + mapType(p, resultSym.typ) == etyBaseIndex: resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) gen(p, prc.ast.sons[resultPos], a) if mapType(p, resultSym.typ) == etyBaseIndex: @@ -2218,10 +2220,10 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) = if dest.kind == src.kind: # no-op conversion return - let toInt = (dest.kind in tyInt .. tyInt32) - let toUint = (dest.kind in tyUInt .. tyUInt32) - let fromInt = (src.kind in tyInt .. tyInt32) - let fromUint = (src.kind in tyUInt .. tyUInt32) + let toInt = (dest.kind in tyInt..tyInt32) + let toUint = (dest.kind in tyUInt..tyUInt32) + let fromInt = (src.kind in tyInt..tyInt32) + let fromUint = (src.kind in tyUInt..tyUInt32) if toUint and (fromInt or fromUint): let trimmer = unsignedTrimmer(dest.size) diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index d9df04e4bc..3768acf27d 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -122,7 +122,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) = [name, genTypeInfo(p, typ.sons[0])]) proc genEnumInfoPHP(p: PProc; t: PType): Rope = - let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias}) + let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink}) result = "$$NTI$1" % [rope(t.id)] p.declareGlobal(t.id, result) if containsOrIncl(p.g.typeInfoGenerated, t.id): return @@ -141,7 +141,7 @@ proc genEnumInfoPHP(p: PProc; t: PType): Rope = proc genTypeInfo(p: PProc, typ: PType): Rope = if p.target == targetPHP: return makeJSString(typeToString(typ, preferModuleInfo)) - let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias}) + let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink}) result = "NTI$1" % [rope(t.id)] if containsOrIncl(p.g.typeInfoGenerated, t.id): return case t.kind @@ -152,7 +152,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % [result, rope(ord(t.kind))] prepend(p.g.typeInfo, s) - of tyVar, tyRef, tyPtr, tySequence, tyRange, tySet: + of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet: var s = "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % [result, rope(ord(t.kind))] diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index cf43ba15d3..fca5ef52eb 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -190,7 +190,7 @@ proc interestingVar(s: PSym): bool {.inline.} = proc illegalCapture(s: PSym): bool {.inline.} = result = skipTypes(s.typ, abstractInst).kind in - {tyVar, tyOpenArray, tyVarargs} or + {tyVar, tyOpenArray, tyVarargs, tyLent} or s.kind == skResult proc isInnerProc(s: PSym): bool = diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 9612ff0aba..8510bf7ee7 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -330,7 +330,7 @@ proc typeNeedsNoDeepCopy(t: PType): bool = # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var' # for the stricter check and likewise we can skip 'seq' for a less # strict check: - if t.kind in {tyVar, tySequence}: t = t.sons[0] + if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon result = not containsGarbageCollectedRef(t) proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType; @@ -469,7 +469,7 @@ proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym, # we pick n's type here, which hopefully is 'tyArray' and not # 'tyOpenArray': var argType = n[i].typ.skipTypes(abstractInst) - if i < formals.len and formals[i].typ.kind == tyVar: + if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}: localError(n[i].info, "'spawn'ed function cannot have a 'var' parameter") #elif containsTyRef(argType): # localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure") diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 6735cc1ce2..4afaf859d8 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -325,8 +325,8 @@ proc lsub(g: TSrcGen; n: PNode): int proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string = proc skip(t: PType): PType = result = t - while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct, - tyOrdinal, tyAlias}: + while result.kind in {tyGenericInst, tyRange, tyVar, tyLent, tyDistinct, + tyOrdinal, tyAlias, tySink}: result = lastSon(result) if n.typ != nil and n.typ.skip.kind in {tyBool, tyEnum}: let enumfields = n.typ.skip.n diff --git a/compiler/sem.nim b/compiler/sem.nim index 1098e9961f..0e97a66b2e 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -102,8 +102,8 @@ proc commonType*(x, y: PType): PType = # if expressions, etc.: if x == nil: return x if y == nil: return y - var a = skipTypes(x, {tyGenericInst, tyAlias}) - var b = skipTypes(y, {tyGenericInst, tyAlias}) + var a = skipTypes(x, {tyGenericInst, tyAlias, tySink}) + var b = skipTypes(y, {tyGenericInst, tyAlias, tySink}) result = x if a.kind in {tyExpr, tyNil}: result = y elif b.kind in {tyExpr, tyNil}: result = x diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 67af6ade76..bbd2baf6e0 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -242,9 +242,9 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = tyTypeDesc, tyGenericInvocation, tyForward: internalError(c.info, "assignment requested for type: " & typeToString(t)) of tyOrdinal, tyRange, tyInferred, - tyGenericInst, tyStatic, tyVar, tyAlias: + tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink: liftBodyAux(c, lastSon(t), body, x, y) - of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("liftBodyAux") + of tyUnused, tyOptAsRef: internalError("liftBodyAux") proc newProcType(info: TLineInfo; owner: PSym): PType = result = newType(tyProc, owner) @@ -306,7 +306,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym = - let t = typ.skipTypes({tyGenericInst, tyVar, tyAlias}) + let t = typ.skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink}) result = t.assignment if result.isNil: result = liftBody(c, t, attachedAsgn, info) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a51b9afe30..c580f8fd54 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -377,7 +377,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode = proc canDeref(n: PNode): bool {.inline.} = result = n.len >= 2 and (let t = n[1].typ; - t != nil and t.skipTypes({tyGenericInst, tyAlias}).kind in {tyPtr, tyRef}) + t != nil and t.skipTypes({tyGenericInst, tyAlias, tySink}).kind in {tyPtr, tyRef}) proc tryDeref(n: PNode): PNode = result = newNodeI(nkHiddenDeref, n.info) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 8affee649f..3996188dc2 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -259,19 +259,19 @@ proc makePtrType*(c: PContext, baseType: PType): PType = proc makeTypeWithModifier*(c: PContext, modifier: TTypeKind, baseType: PType): PType = - assert modifier in {tyVar, tyPtr, tyRef, tyStatic, tyTypeDesc} + assert modifier in {tyVar, tyLent, tyPtr, tyRef, tyStatic, tyTypeDesc} - if modifier in {tyVar, tyTypeDesc} and baseType.kind == modifier: + if modifier in {tyVar, tyLent, tyTypeDesc} and baseType.kind == modifier: result = baseType else: result = newTypeS(modifier, c) addSonSkipIntLit(result, baseType.assertNotNil) -proc makeVarType*(c: PContext, baseType: PType): PType = - if baseType.kind == tyVar: +proc makeVarType*(c: PContext, baseType: PType; kind = tyVar): PType = + if baseType.kind == kind: result = baseType else: - result = newTypeS(tyVar, c) + result = newTypeS(kind, c) addSonSkipIntLit(result, baseType.assertNotNil) proc makeTypeDesc*(c: PContext, typ: PType): PType = diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 51e75e91fb..577580f2ec 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -32,7 +32,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # XXX tyGenericInst here? if result.typ.kind == tyProc and tfUnresolved in result.typ.flags: localError(n.info, errProcHasNoConcreteType, n.renderTree) - if result.typ.kind == tyVar: result = newDeref(result) + if result.typ.kind in {tyVar, tyLent}: result = newDeref(result) elif {efWantStmt, efAllowStmt} * flags != {}: result.typ = newTypeS(tyVoid, c) else: @@ -52,7 +52,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result.typ = errorType(c) else: if efNoProcvarCheck notin flags: semProcvarCheck(c, result) - if result.typ.kind == tyVar: result = newDeref(result) + if result.typ.kind in {tyVar, tyLent}: result = newDeref(result) proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExpr(c, n, flags) @@ -350,7 +350,7 @@ proc changeType(n: PNode, newType: PType, check: bool) = for i in countup(0, sonsLen(n) - 1): changeType(n.sons[i], elemType(newType), check) of nkPar: - let tup = newType.skipTypes({tyGenericInst, tyAlias}) + let tup = newType.skipTypes({tyGenericInst, tyAlias, tySink}) if tup.kind != tyTuple: if tup.kind == tyObject: return globalError(n.info, "no tuple type for constructor") @@ -393,7 +393,7 @@ proc arrayConstrType(c: PContext, n: PNode): PType = if sonsLen(n) == 0: rawAddSon(typ, newTypeS(tyEmpty, c)) # needs an empty basetype! else: - var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal, tyAlias}) + var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink}) addSonSkipIntLit(typ, t) typ.sons[0] = makeRangeType(c, 0, sonsLen(n) - 1, n.info) result = typ @@ -417,7 +417,7 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = let yy = semExprWithType(c, x) var typ = yy.typ addSon(result, yy) - #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal}) + #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal}) for i in countup(1, sonsLen(n) - 1): x = n.sons[i] if x.kind == nkExprColonExpr and sonsLen(x) == 2: @@ -471,7 +471,7 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode = of nkSym: # n.sym.typ can be nil in 'check' mode ... if n.sym.typ != nil and - skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar: + skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}: incl(n.sym.flags, sfAddrTaken) result = newHiddenAddrTaken(c, n) of nkDotExpr: @@ -479,12 +479,12 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode = if n.sons[1].kind != nkSym: internalError(n.info, "analyseIfAddressTaken") return - if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar: + if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}: incl(n.sons[1].sym.flags, sfAddrTaken) result = newHiddenAddrTaken(c, n) of nkBracketExpr: checkMinSonsLen(n, 1) - if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind != tyVar: + if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}: if n.sons[0].kind == nkSym: incl(n.sons[0].sym.flags, sfAddrTaken) result = newHiddenAddrTaken(c, n) else: @@ -499,7 +499,7 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) = # get the real type of the callee # it may be a proc var with a generic alias type, so we skip over them - var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias}) + var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}) if n.sons[0].kind == nkSym and n.sons[0].sym.magic in FakeVarParams: # BUGFIX: check for L-Value still needs to be done for the arguments! @@ -692,7 +692,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = else: n.sons[0] = semExpr(c, n.sons[0], {efInCall}) let t = n.sons[0].typ - if t != nil and t.kind == tyVar: + if t != nil and t.kind in {tyVar, tyLent}: n.sons[0] = newDeref(n.sons[0]) elif n.sons[0].kind == nkBracketExpr: let s = bracketedMacro(n.sons[0]) @@ -865,7 +865,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, const tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass} - tyDotOpTransparent = {tyVar, tyPtr, tyRef, tyAlias} + tyDotOpTransparent = {tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink} proc readTypeParameter(c: PContext, typ: PType, paramName: PIdent, info: TLineInfo): PNode = @@ -998,8 +998,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = while p != nil and p.selfSym == nil: p = p.next if p != nil and p.selfSym != nil: - var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef, - tyAlias}) + var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, + tyAlias, tySink}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1108,7 +1108,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = return nil if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon - ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias}) + ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1175,7 +1175,7 @@ proc semDeref(c: PContext, n: PNode): PNode = checkSonsLen(n, 1) n.sons[0] = semExprWithType(c, n.sons[0]) result = n - var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyAlias}) + var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyAlias, tySink}) case t.kind of tyRef, tyPtr: n.typ = t.lastSon else: result = nil @@ -1195,7 +1195,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = n.sons[0] = semExprWithType(c, n.sons[0], {efNoProcvarCheck, efNoEvaluateGeneric}) let arr = skipTypes(n.sons[0].typ, {tyGenericInst, - tyVar, tyPtr, tyRef, tyAlias}) + tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) case arr.kind of tyArray, tyOpenArray, tyVarargs, tySequence, tyString, tyCString: @@ -1223,7 +1223,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = n.sons[0] = makeDeref(n.sons[0]) # [] operator for tuples requires constant expression: n.sons[1] = semConstExpr(c, n.sons[1]) - if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias}).kind in + if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias, tySink}).kind in {tyInt..tyInt64}: var idx = getOrdValue(n.sons[1]) if idx >= 0 and idx < sonsLen(arr): n.typ = arr.sons[int(idx)] @@ -1362,7 +1362,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = # a = b # both are vars, means: a[] = b[] # a = b # b no 'var T' means: a = addr(b) var le = a.typ - if (skipTypes(le, {tyGenericInst, tyAlias}).kind != tyVar and + if (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and isAssignable(c, a) == arNone) or skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}: # Direct assignment to a discriminant is allowed! @@ -1455,18 +1455,18 @@ proc semProcBody(c: PContext, n: PNode): PNode = closeScope(c) proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = - var t = skipTypes(restype, {tyGenericInst, tyAlias}) + var t = skipTypes(restype, {tyGenericInst, tyAlias, tySink}) case t.kind - of tyVar: - t.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892 + of tyVar, tyLent: + if t.kind == tyVar: t.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892 if n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv}: n.sons[0] = n.sons[0].sons[1] n.sons[0] = takeImplicitAddr(c, n.sons[0]) of tyTuple: for i in 0.. 1: - var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias}) + var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink}) if exp.kind == tyVarargs: exp = elemType(exp) if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return 50 return 100 @@ -309,7 +309,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} = let m = s.getModule() if m != nil and sfSystemModule in m.flags: if s.kind == skType: return - var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias}) + var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink}) if exp.kind == tyVarargs: exp = elemType(exp) if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return result = sigmatch.argtypeMatches(c, s.typ.sons[1], firstArg) @@ -378,8 +378,8 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) t = t.sons[0] suggestOperations(c, n, field, typ, outputs) else: - let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias}) - typ = skipTypes(typ, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias}) + let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias, tySink}) + typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) if typ.kind == tyObject: var t = typ while true: diff --git a/compiler/transf.nim b/compiler/transf.nim index f8f7f87464..14ff58c905 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -93,7 +93,7 @@ proc getCurrOwner(c: PTransf): PSym = proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode = let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) - r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias}) + r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink}) incl(r.flags, sfFromGeneric) let owner = getCurrOwner(c) if owner.isIterator and not c.tooEarly: @@ -331,7 +331,7 @@ proc transformYield(c: PTransf, n: PNode): PTransNode = # c.transCon.forStmt.len == 3 means that there is one for loop variable # and thus no tuple unpacking: if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons - if skipTypes(e.typ, {tyGenericInst, tyAlias}).kind == tyTuple and + if skipTypes(e.typ, {tyGenericInst, tyAlias, tySink}).kind == tyTuple and c.transCon.forStmt.len != 3: e = skipConv(e) if e.kind == nkPar: @@ -506,7 +506,7 @@ proc putArgInto(arg: PNode, formal: PType): TPutArgInto = if putArgInto(arg.sons[i], formal) != paDirectMapping: return result = paDirectMapping else: - if skipTypes(formal, abstractInst).kind == tyVar: result = paVarAsgn + if skipTypes(formal, abstractInst).kind in {tyVar, tyLent}: result = paVarAsgn else: result = paFastAsgn proc findWrongOwners(c: PTransf, n: PNode) = diff --git a/compiler/trees.nim b/compiler/trees.nim index 7efefdc2ed..577ea75eeb 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -102,7 +102,7 @@ proc isDeepConstExpr*(n: PNode): bool = if not isDeepConstExpr(n.sons[i]): return false if n.typ.isNil: result = true else: - let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias}) + let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink}) if t.kind in {tyRef, tyPtr}: return false if t.kind != tyObject or not isCaseObj(t.n): result = true diff --git a/compiler/types.nim b/compiler/types.nim index 0d3a25980d..b4f78b5611 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -51,17 +51,17 @@ const # TODO: Remove tyTypeDesc from each abstractX and (where necessary) # replace with typedescX abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, - tyTypeDesc, tyAlias, tyInferred} + tyTypeDesc, tyAlias, tyInferred, tySink, tyLent} abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, - tyAlias, tyInferred} + tyAlias, tyInferred, tySink, tyLent} abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc, - tyAlias, tyInferred} + tyAlias, tyInferred, tySink} abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, - tyTypeDesc, tyAlias, tyInferred} + tyTypeDesc, tyAlias, tyInferred, tySink} abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, - tyInferred} + tyInferred, tySink} skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc, tyAlias, - tyInferred} + tyInferred, tySink, tyLent} # typedescX is used if we're sure tyTypeDesc should be included (or skipped) typedescPtrs* = abstractPtrs + {tyTypeDesc} typedescInst* = abstractInst + {tyTypeDesc} @@ -388,8 +388,8 @@ const "int", "int8", "int16", "int32", "int64", "float", "float32", "float64", "float128", "uint", "uint8", "uint16", "uint32", "uint64", - "unused0", "unused1", - "unused2", "varargs[$1]", "unused", "Error Type", + "opt", "sink", + "lent", "varargs[$1]", "unused", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "inferred", "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", @@ -539,7 +539,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = add(result, typeToString(t.sons[i])) if i < sonsLen(t) - 1: add(result, ", ") add(result, ')') - of tyPtr, tyRef, tyVar: + of tyPtr, tyRef, tyVar, tyLent: result = typeToStr[t.kind] if t.len >= 2: setLen(result, result.len-1) @@ -968,7 +968,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = if result and ExactGenericParams in c.flags: result = a.sym.position == b.sym.position of tyGenericInvocation, tyGenericBody, tySequence, - tyOpenArray, tySet, tyRef, tyPtr, tyVar, + tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyLent, tySink, tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyOpt: cycleCheck() if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n @@ -988,7 +988,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = sameTypeOrNilAux(a.sons[0], b.sons[0], c) and sameValue(a.n.sons[0], b.n.sons[0]) and sameValue(a.n.sons[1], b.n.sons[1]) - of tyGenericInst, tyAlias, tyInferred, tyLent, tySink: + of tyGenericInst, tyAlias, tyInferred: cycleCheck() result = sameTypeAux(a.lastSon, b.lastSon, c) of tyNone: result = false @@ -1101,11 +1101,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if containsOrIncl(marker, typ.id): return var t = skipTypes(typ, abstractInst-{tyTypeDesc}) case t.kind - of tyVar: + of tyVar, tyLent: if kind in {skProc, skFunc, skConst}: return t var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc}) case t2.kind - of tyVar: + of tyVar, tyLent: if taHeap notin flags: result = t2 # ``var var`` is illegal on the heap of tyOpenArray: if kind != skParam: result = t @@ -1143,7 +1143,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyRange: if skipTypes(t.sons[0], abstractInst-{tyTypeDesc}).kind notin {tyChar, tyEnum, tyInt..tyFloat128, tyUInt8..tyUInt32}: result = t - of tyOpenArray, tyVarargs: + of tyOpenArray, tyVarargs, tySink: if kind != skParam: result = t else: result = typeAllowedAux(marker, t.sons[0], skVar, flags) of tySequence, tyOpt: @@ -1174,7 +1174,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil - of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("typeAllowedAux") + of tyUnused, tyOptAsRef: internalError("typeAllowedAux") proc typeAllowed*(t: PType, kind: TSymKind): PType = # returns 'nil' on success and otherwise the part of the type that is @@ -1322,7 +1322,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = if typ.callConv == ccClosure: result = 2 * ptrSize else: result = ptrSize a = ptrSize - of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray: + of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyLent, tyOpenArray: let base = typ.lastSon if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion): result = szIllegalRecursion diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index fb277272b1..bb6c47324d 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -209,6 +209,8 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; else: result = mapTypeToBracket("ref", mRef, t, info) of tyVar: result = mapTypeToBracket("var", mVar, t, info) + of tyLent: result = mapTypeToBracket("lent", mBuiltinType, t, info) + of tySink: result = mapTypeToBracket("sink", mBuiltinType, t, info) of tySequence: result = mapTypeToBracket("seq", mSeq, t, info) of tyOpt: result = mapTypeToBracket("opt", mOpt, t, info) of tyProc: @@ -274,7 +276,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add atomicType("static", mNone) if t.n != nil: result.add t.n.copyTree - of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("mapTypeToAstX") + of tyUnused, tyOptAsRef: internalError("mapTypeToAstX") proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode = result = mapTypeToAstX(t, info, false, true) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 17878b656b..a22acdff0d 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1288,7 +1288,7 @@ proc whichAsgnOpc(n: PNode): TOpcode = opcAsgnStr of tyFloat..tyFloat128: opcAsgnFloat - of tyRef, tyNil, tyVar: + of tyRef, tyNil, tyVar, tyLent: opcAsgnRef else: opcAsgnComplex @@ -1481,7 +1481,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = cannotEval(n) template needsRegLoad(): untyped = - gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar})) + gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar, tyLent})) proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; flags: TGenFlags) = @@ -1553,7 +1553,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode = result = newNodeIT(nkFloatLit, info, t) of tyCString, tyString: result = newNodeIT(nkStrLit, info, t) - of tyVar, tyPointer, tyPtr, tySequence, tyExpr, + of tyVar, tyLent, tyPointer, tyPtr, tySequence, tyExpr, tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil: result = newNodeIT(nkNilLit, info, t) of tyProc: diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index 0939a5953f..5f725994e8 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -102,7 +102,7 @@ proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet) = else: storeAny(s, t.lastSon, a[i], stored) s.add("]") - of tyRange, tyGenericInst, tyAlias: storeAny(s, t.lastSon, a, stored) + of tyRange, tyGenericInst, tyAlias, tySink: storeAny(s, t.lastSon, a, stored) of tyEnum: # we need a slow linear search because of enums with holes: for e in items(t.n): @@ -275,7 +275,7 @@ proc loadAny(p: var JsonParser, t: PType, next(p) return raiseParseErr(p, "float expected") - of tyRange, tyGenericInst, tyAlias: result = loadAny(p, t.lastSon, tab) + of tyRange, tyGenericInst, tyAlias, tySink: result = loadAny(p, t.lastSon, tab) else: internalError "cannot marshal at compile-time " & t.typeToString diff --git a/lib/system.nim b/lib/system.nim index de91c4dda2..8f83fb8c38 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -249,7 +249,7 @@ type when defined(nimHasOpt): type opt*{.magic: "Opt".}[T] -when defined(nimHasSink): +when defined(nimNewRuntime): type sink*{.magic: "BuiltinType".}[T] type lent*{.magic: "BuiltinType".}[T] From 2015895357cd1d32b4ae93a9b527b6d1171d8189 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 8 Jan 2018 01:47:54 +0100 Subject: [PATCH 156/200] sink type begins to compile --- compiler/semexprs.nim | 15 +++++++++++++-- compiler/sigmatch.nim | 30 ++++++++++++++++-------------- compiler/types.nim | 2 ++ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 577580f2ec..795fa1910a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -181,9 +181,18 @@ proc semConv(c: PContext, n: PNode): PNode = result = newNodeI(nkConv, n.info) var targetType = semTypeNode(c, n.sons[0], nil).skipTypes({tyTypeDesc}) maybeLiftType(targetType, c, n[0].info) - result.addSon copyTree(n.sons[0]) - var op = semExprWithType(c, n.sons[1]) + if targetType.kind in {tySink, tyLent}: + let baseType = semTypeNode(c, n.sons[1], nil).skipTypes({tyTypeDesc}) + let t = newTypeS(targetType.kind, c) + t.rawAddSonNoPropagationOfTypeFlags baseType + result = newNodeI(nkType, n.info) + result.typ = makeTypeDesc(c, t) + return + + result.addSon copyTree(n.sons[0]) + + var op = semExprWithType(c, n.sons[1]) if targetType.isMetaType: let final = inferWithMetatype(c, targetType, op, true) result.addSon final @@ -191,6 +200,8 @@ proc semConv(c: PContext, n: PNode): PNode = return result.typ = targetType + # XXX op is overwritten later on, this is likely added too early + # here or needs to be overwritten too then. addSon(result, op) if not isSymChoice(op): diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 3d0b0ed3d6..fffe92b2fa 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -180,7 +180,8 @@ proc sumGeneric(t: PType): int = while true: case t.kind of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct, - tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody: + tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody, + tyLent: t = t.lastSon inc result of tyOr: @@ -207,7 +208,7 @@ proc sumGeneric(t: PType): int = of tyStatic: return t.sons[0].sumGeneric + 1 of tyGenericParam, tyExpr, tyStmt: break - of tyAlias: t = t.lastSon + of tyAlias, tySink: t = t.lastSon of tyBool, tyChar, tyEnum, tyObject, tyPointer, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64, tyCompositeTypeClass: @@ -464,7 +465,7 @@ proc skipToObject(t: PType; skipped: var SkippedPtr): PType = inc ptrs skipped = skippedPtr r = r.lastSon - of tyGenericBody, tyGenericInst, tyAlias: + of tyGenericBody, tyGenericInst, tyAlias, tySink: r = r.lastSon else: break @@ -524,7 +525,7 @@ proc allowsNil(f: PType): TTypeRelation {.inline.} = result = if tfNotNil notin f.flags: isSubtype else: isNone proc inconsistentVarTypes(f, a: PType): bool {.inline.} = - result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar) + result = f.kind != a.kind and (f.kind in {tyVar, tyLent} or a.kind in {tyVar, tyLent}) proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = ## For example we have: @@ -889,7 +890,7 @@ proc inferStaticsInRange(c: var TCandidate, doInferStatic(lowerBound, upperBound.intVal + 1 - lengthOrd(concrete)) template subtypeCheck() = - if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar}: + if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}: result = isNone proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = @@ -897,7 +898,7 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = assert f.kind == a.kind template baseTypesCheck(lhs, rhs: PType): bool = - lhs.kind notin {tyPtr, tyRef, tyVar} and + lhs.kind notin {tyPtr, tyRef, tyVar, tyLent} and typeRel(c, lhs, rhs, {trNoCovariance}) == isSubtype case f.kind @@ -983,17 +984,17 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, template doBind: bool = trDontBind notin flags # var and static arguments match regular modifier-free types - var a = aOrig.skipTypes({tyStatic, tyVar}).maybeSkipDistinct(c.calleeSym) + var a = aOrig.skipTypes({tyStatic, tyVar, tyLent}).maybeSkipDistinct(c.calleeSym) # XXX: Theoretically, maybeSkipDistinct could be called before we even # start the param matching process. This could be done in `prepareOperand` # for example, but unfortunately `prepareOperand` is not called in certain # situation when nkDotExpr are rotated to nkDotCalls - if aOrig.kind == tyAlias: + if aOrig.kind in {tyAlias, tySink}: return typeRel(c, f, lastSon(aOrig)) if a.kind == tyGenericInst and - skipTypes(f, {tyVar}).kind notin { + skipTypes(f, {tyVar, tyLent}).kind notin { tyGenericBody, tyGenericInvocation, tyGenericInst, tyGenericParam} + tyTypeClasses: return typeRel(c, f, lastSon(a)) @@ -1105,8 +1106,8 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, of tyFloat32: result = handleFloatRange(f, a) of tyFloat64: result = handleFloatRange(f, a) of tyFloat128: result = handleFloatRange(f, a) - of tyVar: - if aOrig.kind == tyVar: result = typeRel(c, f.base, aOrig.base) + of tyVar, tyLent: + if aOrig.kind == f.kind: result = typeRel(c, f.base, aOrig.base) else: result = typeRel(c, f.base, aOrig, flags + {trNoCovariance}) subtypeCheck() of tyArray: @@ -1311,7 +1312,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, of tyEmpty, tyVoid: if a.kind == f.kind: result = isEqual - of tyAlias: + of tyAlias, tySink: result = typeRel(c, lastSon(f), a) of tyGenericInst: @@ -1497,7 +1498,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, considerPreviousT: let targetKind = f.sons[0].kind let effectiveArgType = a.skipTypes({tyRange, tyGenericInst, - tyBuiltInTypeClass, tyAlias}) + tyBuiltInTypeClass, tyAlias, tySink}) let typeClassMatches = targetKind == effectiveArgType.kind and not effectiveArgType.isEmptyContainer if typeClassMatches or @@ -2068,7 +2069,8 @@ proc prepareNamedParam(a: PNode) = proc arrayConstr(c: PContext, n: PNode): PType = result = newTypeS(tyArray, c) rawAddSon(result, makeRangeType(c, 0, 0, n.info)) - addSonSkipIntLit(result, skipTypes(n.typ, {tyGenericInst, tyVar, tyOrdinal})) + addSonSkipIntLit(result, skipTypes(n.typ, + {tyGenericInst, tyVar, tyLent, tyOrdinal})) proc arrayConstr(c: PContext, info: TLineInfo): PType = result = newTypeS(tyArray, c) diff --git a/compiler/types.nim b/compiler/types.nim index b4f78b5611..cbbfa86317 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -581,6 +581,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = if len(prag) != 0: add(result, "{." & prag & ".}") of tyVarargs: result = typeToStr[t.kind] % typeToString(t.sons[0]) + of tySink: + result = "sink " & typeToString(t.sons[0]) else: result = typeToStr[t.kind] result.addTypeFlags(t) From 5492190bc63326069233a8aaf9f2e567cfc555f8 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 8 Jan 2018 10:49:00 +0100 Subject: [PATCH 157/200] Fix lists of paths in posix environment (#7034) Empty paths in a colon separated list would be considered as the current directory, so have to ensure $PATH and $LD_LIBRARY_PATH are not empty before separating it with : --- .gitlab-ci.yml | 2 +- .travis.yml | 2 +- ci/build.sh | 2 +- ci/deps.sh | 2 +- koch.nim | 4 ++-- tests/testament/categories.nim | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76c94c8e75..c37b4c8d4b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ stages: .linux_set_path: &linux_set_path_def before_script: - - export PATH=$(pwd)/bin:$PATH + - export PATH=$(pwd)/bin${PATH:+:$PATH} tags: - linux diff --git a/.travis.yml b/.travis.yml index 6b8cdbe03b..a3fa3f1da1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_script: - sh build.sh - cd .. - sed -i -e 's,cc = gcc,cc = clang,' config/nim.cfg - - export PATH=$(pwd)/bin:$PATH + - export PATH=$(pwd)/bin${PATH:+:$PATH} script: - nim c koch - ./koch boot diff --git a/ci/build.sh b/ci/build.sh index a0fee14977..6321fffba8 100644 --- a/ci/build.sh +++ b/ci/build.sh @@ -6,7 +6,7 @@ cd csources sh build.sh cd .. # Add Nim to the PATH -export PATH=$(pwd)/bin:$PATH +export PATH=$(pwd)/bin${PATH:+:$PATH} # Bootstrap. nim -v nim c koch diff --git a/ci/deps.sh b/ci/deps.sh index 7471785a0d..f0f831a2a3 100644 --- a/ci/deps.sh +++ b/ci/deps.sh @@ -7,7 +7,7 @@ apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev li gcc -v -export PATH=$(pwd)/bin:$PATH +export PATH=$(pwd)/bin${PATH:+:$PATH} # Nimble deps nim e install_nimble.nims diff --git a/koch.nim b/koch.nim index 7bb7ea4024..d51b902ee8 100644 --- a/koch.nim +++ b/koch.nim @@ -97,7 +97,7 @@ proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = if not absolute.isAbsolute: absolute = getCurrentDir() / absolute echo("Adding to $PATH: ", absolute) - putEnv("PATH", prevPath & PathSep & absolute) + putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) echo(cmd) if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) @@ -402,7 +402,7 @@ proc winReleaseArch(arch: string) = template withMingw(path, body) = let prevPath = getEnv("PATH") - putEnv("PATH", path & PathSep & prevPath) + putEnv("PATH", (if path.len > 0: path & PathSep else: "") & prevPath) try: body finally: diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 33b93e3c4d..42e19d3dd0 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -90,7 +90,7 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = # posix relies on crappy LD_LIBRARY_PATH (ugh!): var libpath = getEnv"LD_LIBRARY_PATH".string # Temporarily add the lib directory to LD_LIBRARY_PATH: - putEnv("LD_LIBRARY_PATH", "tests/dll:" & libpath) + putEnv("LD_LIBRARY_PATH", "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv("LD_LIBRARY_PATH", libpath) var nimrtlDll = DynlibFormat % "nimrtl" safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) From ce31789431d1a7925a047fa31779a499f8dcf437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Str=C3=B8mberg?= Date: Mon, 8 Jan 2018 13:30:09 +0100 Subject: [PATCH 158/200] Ast and concrete syntax different. Change variable name to a from v, to match the ast and other examples. --- doc/astspec.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/astspec.txt b/doc/astspec.txt index 57f6b9d8c9..6d755c2e20 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -918,7 +918,7 @@ This is equivalent to ``var``, but with ``nnkLetSection`` rather than Concrete syntax: .. code-block:: nim - let v = 3 + let a = 3 AST: From b168efd1ab06422d147b1938aed6e0ed19e6134f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 8 Jan 2018 13:43:43 +0100 Subject: [PATCH 159/200] make strformat.fmt take the same signature as strfmt.fmt in order to force an ambiguity error; refs #6958 --- lib/pure/strformat.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 180cbcbecc..e04c80794b 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -224,7 +224,7 @@ template callFormatOption(res, arg, option) {.dirty.} = else: res.add format(arg, option) -macro fmt*(pattern: string): untyped = +macro fmt*(pattern: string{lit}): untyped = ## For a specification of the ``fmt`` macro, see the module level documentation. runnableExamples: template check(actual, expected: string) = @@ -332,7 +332,7 @@ macro fmt*(pattern: string): untyped = # works: import times - var nullTime: TimeInfo + var nullTime: DateTime check fmt"{nullTime:yyyy-mm-dd}", "0000-00-00" # Unicode string tests @@ -609,7 +609,6 @@ proc format*(value: string; specifier: string; res: var string) = ## sense to call this directly, but it is required to exist ## by the ``fmt`` macro. let spec = parseStandardFormatSpecifier(specifier) - var fmode = ffDefault case spec.typ of 's', '\0': discard else: From ceb8ba49576ed0f27cf6ec031369c42ddf68757d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 8 Jan 2018 18:02:38 +0100 Subject: [PATCH 160/200] fixes #7018 --- compiler/semexprs.nim | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 51e75e91fb..f4f691889e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1865,17 +1865,16 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = magicsAfterOverloadResolution(c, result, flags) of mRunnableExamples: if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList: - if n.sons[0].kind == nkIdent: - if sfMainModule in c.module.flags: - let inp = toFullPath(c.module.info) - if c.runnableExamples == nil: - c.runnableExamples = newTree(nkStmtList, - newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) - let imports = newTree(nkStmtList) - extractImports(n.lastSon, imports) - for imp in imports: c.runnableExamples.add imp - c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon) - result = setMs(n, s) + if sfMainModule in c.module.flags: + let inp = toFullPath(c.module.info) + if c.runnableExamples == nil: + c.runnableExamples = newTree(nkStmtList, + newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) + let imports = newTree(nkStmtList) + extractImports(n.lastSon, imports) + for imp in imports: c.runnableExamples.add imp + c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon) + result = setMs(n, s) else: result = emptyNode else: From c924fac5c889c5eace083707e62c60f16544573b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 8 Jan 2018 18:22:18 +0100 Subject: [PATCH 161/200] fixes #7019 --- compiler/renderer.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 6735cc1ce2..c45de0db9c 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -898,6 +898,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = put(g, tkBracketLe, "[") gcomma(g, n, 2) put(g, tkBracketRi, "]") + elif n.len > 1 and n.lastSon.kind == nkStmtList: + gsub(g, n[0]) + if n.len > 2: + put(g, tkParLe, "(") + gcomma(g, n, 1, -2) + put(g, tkParRi, ")") + put(g, tkColon, ":") + gsub(g, n, n.len-1) else: if sonsLen(n) >= 1: gsub(g, n.sons[0]) put(g, tkParLe, "(") From fd1883f90aecd15a3d9596c5fc45bac49b431b4b Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Mon, 8 Jan 2018 23:26:03 +0300 Subject: [PATCH 162/200] Fixes for new runtime (#7037) --- lib/core/allocators.nim | 10 +++++----- lib/core/seqs.nim | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/core/allocators.nim b/lib/core/allocators.nim index d6608a2037..4edd00f369 100644 --- a/lib/core/allocators.nim +++ b/lib/core/allocators.nim @@ -9,7 +9,7 @@ type Allocator* {.inheritable.} = ptr object - alloc*: proc (a: Allocator; size: int; alignment = 8): pointer {.nimcall.} + alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.} dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.} realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.} @@ -22,14 +22,14 @@ proc getCurrentAllocator*(): Allocator = proc setCurrentAllocator*(a: Allocator) = currentAllocator = a -proc alloc*(size: int): pointer = +proc alloc*(size: int; alignment: int = 8): pointer = let a = getCurrentAllocator() - result = a.alloc(a, size) + result = a.alloc(a, size, alignment) proc dealloc*(p: pointer; size: int) = let a = getCurrentAllocator() - a.dealloc(a, size) + a.dealloc(a, p, size) proc realloc*(p: pointer; oldSize, newSize: int): pointer = let a = getCurrentAllocator() - result = a.realloc(a, oldSize, newSize) + result = a.realloc(a, p, oldSize, newSize) diff --git a/lib/core/seqs.nim b/lib/core/seqs.nim index 6be95a3bca..c32cf3690b 100644 --- a/lib/core/seqs.nim +++ b/lib/core/seqs.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -import allocators +import allocators, typetraits ## Default seq implementation used by Nim's core. type @@ -115,3 +115,25 @@ proc `@`*[T](elems: openArray[T]): seq[T] = result.data[i] = elems[i] proc len*[T](x: seq[T]): int {.inline.} = x.len + +proc `$`*[T](x: seq[T]): string = + result = "@[" + var firstElement = true + for i in 0.. Date: Tue, 9 Jan 2018 00:18:32 +0100 Subject: [PATCH 163/200] Add additional $ fallback to fmt --- lib/pure/strformat.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index e04c80794b..d1dedb625c 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -221,8 +221,10 @@ template callFormat(res, arg) {.dirty.} = template callFormatOption(res, arg, option) {.dirty.} = when compiles(format(arg, option, res)): format(arg, option, res) - else: + elif compiles(format(arg, option)): res.add format(arg, option) + else: + format($arg, option, res) macro fmt*(pattern: string{lit}): untyped = ## For a specification of the ``fmt`` macro, see the module level documentation. From 624bd847fb2dfdeca00548947ba248a7b32d2e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Tue, 9 Jan 2018 00:33:39 +0100 Subject: [PATCH 164/200] Add test case --- tests/stdlib/tstrformat.nim | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/stdlib/tstrformat.nim diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim new file mode 100644 index 0000000000..4e5c614a78 --- /dev/null +++ b/tests/stdlib/tstrformat.nim @@ -0,0 +1,13 @@ +discard """ + action: "run" +""" + +import strformat + +type Obj = object + +proc `$`(o: Obj): string = "foobar" + +var o: Obj +doAssert fmt"{o}" == "foobar" +doAssert fmt"{o:10}" == "foobar " \ No newline at end of file From b31151f68ea47a38fc8f5b65d51b23c12c591a3f Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Tue, 9 Jan 2018 13:22:29 +0300 Subject: [PATCH 165/200] New runtime: fix allocator inheritable (#7046) --- lib/core/allocators.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/allocators.nim b/lib/core/allocators.nim index 4edd00f369..62f5e9756b 100644 --- a/lib/core/allocators.nim +++ b/lib/core/allocators.nim @@ -8,7 +8,7 @@ # type - Allocator* {.inheritable.} = ptr object + Allocator* = ptr object {.inheritable.} alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.} dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.} realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.} From 849664744bf2272d579af0a1718d5ee0f09a2233 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Jan 2018 13:00:22 +0100 Subject: [PATCH 166/200] another attempt to make the fragmentation test more robust for Windows --- tests/fragmentation/tfragment_gc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 1781f66105..92367a6dd9 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -24,7 +24,7 @@ let occupiedOk = occ < 64 * 1024 * 1024 if not occupiedOk: echo "occupied ", formatSize(occ) echo "occupied ok: ", occupiedOk -let totalOk = total < 210 * 1024 * 1024 +let totalOk = total < 230 * 1024 * 1024 if not totalOk: echo "total peak memory ", formatSize(total) echo "total ok: ", totalOk From 2c9e56a783e36b0f9db3f2f73d76c910f36a9ffd Mon Sep 17 00:00:00 2001 From: cooldome Date: Tue, 9 Jan 2018 14:25:22 +0000 Subject: [PATCH 167/200] Implement custom annotations (#6987) --- changelog.md | 3 + compiler/ast.nim | 1 + compiler/pragmas.nim | 112 ++++++++++++++++++++----------- compiler/semexprs.nim | 3 +- compiler/semstmts.nim | 3 + compiler/semtempl.nim | 5 +- doc/manual/pragmas.txt | 67 ++++++++++++++++++ lib/core/macros.nim | 54 +++++++++++++++ tests/pragmas/custom_pragma.nim | 5 ++ tests/pragmas/tcustom_pragma.nim | 43 ++++++++++++ 10 files changed, 254 insertions(+), 42 deletions(-) create mode 100644 tests/pragmas/custom_pragma.nim create mode 100644 tests/pragmas/tcustom_pragma.nim diff --git a/changelog.md b/changelog.md index 21ab2b87aa..993923e5c4 100644 --- a/changelog.md +++ b/changelog.md @@ -190,3 +190,6 @@ let - Added support for casting between integers of same bitsize in VM (compile time and nimscript). This allow to among other things to reinterpret signed integers as unsigned. +- Pragmas now support call syntax, for example: ``{.exportc"myname".}`` and ``{.exportc("myname").}`` +- Custom pragmas are now supported using pragma ``pragma``, please see language manual for details + diff --git a/compiler/ast.nim b/compiler/ast.nim index 54c33a038e..69f2eb1c78 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -305,6 +305,7 @@ const sfEscapes* = sfProcvar # param escapes sfBase* = sfDiscriminant sfIsSelf* = sfOverriden # param is 'self' + sfCustomPragma* = sfRegister # symbol is custom pragma template const # getting ready for the future expr/stmt merge diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index bdaecf91d3..b6229796fa 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -17,6 +17,7 @@ import const FirstCallConv* = wNimcall LastCallConv* = wNoconv + nkPragmaCallKinds = {nkExprColonExpr, nkCall, nkCallStrLit} const procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, @@ -29,7 +30,7 @@ const converterPragmas* = procPragmas methodPragmas* = procPragmas+{wBase}-{wImportCpp} templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty, - wDelegator, wExportNims, wUsed} + wDelegator, wExportNims, wUsed, wPragma} macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc, wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern, wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator, @@ -74,7 +75,7 @@ proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode = let p = procAst[pragmasPos] if p.kind == nkEmpty: return nil for it in p: - if it.kind == nkExprColonExpr and it[0].kind == nkIdent and + if it.kind in nkPragmaCallKinds and it.len == 2 and it[0].kind == nkIdent and it[0].ident.id == ord(name): return it[1] @@ -89,7 +90,7 @@ proc pragmaAsm*(c: PContext, n: PNode): char = if n != nil: for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] - if it.kind == nkExprColonExpr and it.sons[0].kind == nkIdent: + if it.kind in nkPragmaCallKinds and it.len == 2 and it.sons[0].kind == nkIdent: case whichKeyword(it.sons[0].ident) of wSubsChar: if it.sons[1].kind == nkCharLit: result = chr(int(it.sons[1].intVal)) @@ -151,7 +152,7 @@ proc newEmptyStrNode(n: PNode): PNode {.noinline.} = result.strVal = "" proc getStrLitNode(c: PContext, n: PNode): PNode = - if n.kind != nkExprColonExpr: + if n.kind notin nkPragmaCallKinds or n.len != 2: localError(n.info, errStringLiteralExpected) # error correction: result = newEmptyStrNode(n) @@ -168,7 +169,7 @@ proc expectStrLit(c: PContext, n: PNode): string = result = getStrLitNode(c, n).strVal proc expectIntLit(c: PContext, n: PNode): int = - if n.kind != nkExprColonExpr: + if n.kind notin nkPragmaCallKinds or n.len != 2: localError(n.info, errIntLiteralExpected) else: n.sons[1] = c.semConstExpr(c, n.sons[1]) @@ -177,7 +178,7 @@ proc expectIntLit(c: PContext, n: PNode): int = else: localError(n.info, errIntLiteralExpected) proc getOptionalStr(c: PContext, n: PNode, defaultStr: string): string = - if n.kind == nkExprColonExpr: result = expectStrLit(c, n) + if n.kind in nkPragmaCallKinds: result = expectStrLit(c, n) else: result = defaultStr proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) = @@ -186,7 +187,7 @@ proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) = proc processMagic(c: PContext, n: PNode, s: PSym) = #if sfSystemModule notin c.module.flags: # liMessage(n.info, errMagicOnlyInSystem) - if n.kind != nkExprColonExpr: + if n.kind notin nkPragmaCallKinds or n.len != 2: localError(n.info, errStringLiteralExpected) return var v: string @@ -204,7 +205,7 @@ proc wordToCallConv(sw: TSpecialWord): TCallingConvention = result = TCallingConvention(ord(ccDefault) + ord(sw) - ord(wNimcall)) proc isTurnedOn(c: PContext, n: PNode): bool = - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds and n.len == 2: let x = c.semConstBoolExpr(c, n.sons[1]) n.sons[1] = x if x.kind == nkIntLit: return x.intVal != 0 @@ -223,7 +224,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) = else: excl(c.module.flags, flag) proc processCallConv(c: PContext, n: PNode) = - if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent): + if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent: var sw = whichKeyword(n.sons[1].ident) case sw of FirstCallConv..LastCallConv: @@ -244,7 +245,7 @@ proc getLib(c: PContext, kind: TLibKind, path: PNode): PLib = result.isOverriden = options.isDynlibOverride(path.strVal) proc expectDynlibNode(c: PContext, n: PNode): PNode = - if n.kind != nkExprColonExpr: + if n.kind notin nkPragmaCallKinds or n.len != 2: localError(n.info, errStringLiteralExpected) # error correction: result = newEmptyStrNode(n) @@ -264,7 +265,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) = if not lib.isOverriden: c.optionStack[^1].dynlib = lib else: - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds: var lib = getLib(c, libDynamic, expectDynlibNode(c, n)) if not lib.isOverriden: addToLib(lib, sym) @@ -279,7 +280,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) = sym.typ.callConv = ccCDecl proc processNote(c: PContext, n: PNode) = - if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and + if (n.kind in nkPragmaCallKinds) and (sonsLen(n) == 2) and (n.sons[0].kind == nkBracketExpr) and (n.sons[0].sons.len == 2) and (n.sons[0].sons[1].kind == nkIdent) and @@ -307,7 +308,7 @@ proc processNote(c: PContext, n: PNode) = invalidPragma(n) proc processOption(c: PContext, n: PNode): bool = - if n.kind != nkExprColonExpr: result = true + if n.kind notin nkPragmaCallKinds or n.len != 2: result = true elif n.sons[0].kind == nkBracketExpr: processNote(c, n) elif n.sons[0].kind != nkIdent: result = true else: @@ -355,8 +356,8 @@ proc processOption(c: PContext, n: PNode): bool = else: result = true proc processPush(c: PContext, n: PNode, start: int) = - if n.sons[start-1].kind == nkExprColonExpr: - localError(n.info, errGenerated, "':' after 'push' not supported") + if n.sons[start-1].kind in nkPragmaCallKinds: + localError(n.info, errGenerated, "'push' can't have arguments") var x = newOptionEntry() var y = c.optionStack[^1] x.options = gOptions @@ -381,14 +382,14 @@ proc processPop(c: PContext, n: PNode) = c.optionStack.setLen(c.optionStack.len - 1) proc processDefine(c: PContext, n: PNode) = - if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent): + if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent): defineSymbol(n.sons[1].ident.s) message(n.info, warnDeprecated, "define") else: invalidPragma(n) proc processUndef(c: PContext, n: PNode) = - if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent): + if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent): undefSymbol(n.sons[1].ident.s) message(n.info, warnDeprecated, "undef") else: @@ -420,7 +421,7 @@ proc processCompile(c: PContext, n: PNode) = localError(n.info, errStringLiteralExpected) result = "" - let it = if n.kind == nkExprColonExpr: n.sons[1] else: n + let it = if n.kind in nkPragmaCallKinds and n.len == 2: n.sons[1] else: n if it.kind == nkPar and it.len == 2: let s = getStrLit(c, it, 0) let dest = getStrLit(c, it, 1) @@ -453,7 +454,7 @@ proc pragmaBreakpoint(c: PContext, n: PNode) = discard getOptionalStr(c, n, "") proc pragmaWatchpoint(c: PContext, n: PNode) = - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds and n.len == 2: n.sons[1] = c.semExpr(c, n.sons[1]) else: invalidPragma(n) @@ -494,7 +495,7 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode = result = newNode(nkAsmStmt, n.info) proc pragmaEmit(c: PContext, n: PNode) = - if n.kind != nkExprColonExpr: + if n.kind notin nkPragmaCallKinds or n.len != 2: localError(n.info, errStringLiteralExpected) else: let n1 = n[1] @@ -512,12 +513,12 @@ proc pragmaEmit(c: PContext, n: PNode) = localError(n.info, errStringLiteralExpected) proc noVal(n: PNode) = - if n.kind == nkExprColonExpr: invalidPragma(n) + if n.kind in nkPragmaCallKinds and n.len > 1: invalidPragma(n) proc pragmaUnroll(c: PContext, n: PNode) = if c.p.nestedLoopCounter <= 0: invalidPragma(n) - elif n.kind == nkExprColonExpr: + elif n.kind in nkPragmaCallKinds and n.len == 2: var unrollFactor = expectIntLit(c, n) if unrollFactor <% 32: n.sons[1] = newIntNode(nkIntLit, unrollFactor) @@ -525,10 +526,11 @@ proc pragmaUnroll(c: PContext, n: PNode) = invalidPragma(n) proc pragmaLine(c: PContext, n: PNode) = - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds and n.len == 2: n.sons[1] = c.semConstExpr(c, n.sons[1]) let a = n.sons[1] if a.kind == nkPar: + # unpack the tuple var x = a.sons[0] var y = a.sons[1] if x.kind == nkExprColonExpr: x = x.sons[1] @@ -549,7 +551,7 @@ proc pragmaLine(c: PContext, n: PNode) = proc processPragma(c: PContext, n: PNode, i: int) = var it = n.sons[i] - if it.kind != nkExprColonExpr: invalidPragma(n) + if it.kind notin nkPragmaCallKinds and it.len == 2: invalidPragma(n) elif it.sons[0].kind != nkIdent: invalidPragma(n) elif it.sons[1].kind != nkIdent: invalidPragma(n) @@ -566,7 +568,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) = localError(x.info, errGenerated, "invalid type for raises/tags list") x.typ = t - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds and n.len == 2: let it = n.sons[1] if it.kind notin {nkCurly, nkBracket}: processExc(c, it) @@ -576,7 +578,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) = invalidPragma(n) proc pragmaLockStmt(c: PContext; it: PNode) = - if it.kind != nkExprColonExpr: + if it.kind notin nkPragmaCallKinds or it.len != 2: invalidPragma(it) else: let n = it[1] @@ -587,7 +589,7 @@ proc pragmaLockStmt(c: PContext; it: PNode) = n.sons[i] = c.semExpr(c, n.sons[i]) proc pragmaLocks(c: PContext, it: PNode): TLockLevel = - if it.kind != nkExprColonExpr: + if it.kind notin nkPragmaCallKinds or it.len != 2: invalidPragma(it) else: case it[1].kind @@ -604,7 +606,7 @@ proc pragmaLocks(c: PContext, it: PNode): TLockLevel = result = TLockLevel(x) proc typeBorrow(sym: PSym, n: PNode) = - if n.kind == nkExprColonExpr: + if n.kind in nkPragmaCallKinds and n.len == 2: let it = n.sons[1] if it.kind != nkAccQuoted: localError(n.info, "a type can only borrow `.` for now") @@ -624,7 +626,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) = if pragma.kind != nkBracket: localError(pragma.info, "list of key:value pairs expected"); return for n in pragma: - if n.kind in {nkExprColonExpr, nkExprEqExpr}: + if n.kind in nkPragmaCallKinds and n.len == 2: let dest = qualifiedLookUp(c, n[1], {checkUndeclared}) if dest == nil or dest.kind in routineKinds: localError(n.info, warnUser, "the .deprecated pragma is unreliable for routines") @@ -638,7 +640,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) = localError(n.info, "key:value pair expected") proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym = - if it.kind != nkExprColonExpr: + if it.kind notin nkPragmaCallKinds or it.len != 2: invalidPragma(it); return let n = it[1] if n.kind == nkSym: @@ -655,13 +657,36 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym = else: result = qualifiedLookUp(c, n, {checkUndeclared}) +proc semCustomPragma(c: PContext, n: PNode): PNode = + assert(n.kind in nkPragmaCallKinds + {nkIdent}) + + if n.kind == nkIdent: + result = newTree(nkCall, n) + elif n.kind == nkExprColonExpr: + # pragma: arg -> pragma(arg) + result = newTree(nkCall, n[0], n[1]) + else: + result = n + + result = c.semOverloadedCall(c, result, n, {skTemplate}, {}) + if sfCustomPragma notin result[0].sym.flags: + invalidPragma(n) + + if n.kind == nkIdent: + result = result[0] + elif n.kind == nkExprColonExpr: + result.kind = n.kind # pragma(arg) -> pragma: arg + proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, validPragmas: TSpecialWords): bool = var it = n.sons[i] - var key = if it.kind == nkExprColonExpr: it.sons[0] else: it + var key = if it.kind in nkPragmaCallKinds and it.len > 1: it.sons[0] else: it if key.kind == nkBracketExpr: processNote(c, it) return + elif key.kind notin nkIdentKinds: + n.sons[i] = semCustomPragma(c, it) + return let ident = considerQuotedIdent(key) var userPragma = strTableGet(c.userPragmas, ident) if userPragma != nil: @@ -785,7 +810,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wExplain: sym.flags.incl sfExplain of wDeprecated: - if it.kind == nkExprColonExpr: deprecatedStmt(c, it) + if it.kind in nkPragmaCallKinds: deprecatedStmt(c, it) elif sym != nil: incl(sym.flags, sfDeprecated) else: incl(c.module.flags, sfDeprecated) of wVarargs: @@ -864,8 +889,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, result = true of wPop: processPop(c, it) of wPragma: - processPragma(c, n, i) - result = true + if not sym.isNil and sym.kind == skTemplate: + sym.flags.incl sfCustomPragma + else: + processPragma(c, n, i) + result = true of wDiscardable: noVal(it) if sym != nil: incl(sym.flags, sfDiscardable) @@ -939,7 +967,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, elif sym.typ == nil: invalidPragma(it) else: sym.typ.lockLevel = pragmaLocks(c, it) of wBitsize: - if sym == nil or sym.kind != skField or it.kind != nkExprColonExpr: + if sym == nil or sym.kind != skField: invalidPragma(it) else: sym.bitsize = expectIntLit(c, it) @@ -957,7 +985,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, if sym == nil: invalidPragma(it) else: magicsys.registerNimScriptSymbol(sym) of wInjectStmt: - if it.kind != nkExprColonExpr: + if it.kind notin nkPragmaCallKinds or it.len != 2: localError(it.info, errExprExpected) else: it.sons[1] = c.semExpr(c, it.sons[1]) @@ -968,10 +996,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, else: localError(it.info, "'experimental' pragma only valid as toplevel statement") of wThis: - if it.kind == nkExprColonExpr: + if it.kind in nkPragmaCallKinds and it.len == 2: c.selfName = considerQuotedIdent(it[1]) - else: + elif it.kind == nkIdent or it.len == 1: c.selfName = getIdent("self") + else: + localError(it.info, "'this' pragma is allowed to have zero or one arguments") of wNoRewrite: noVal(it) of wBase: @@ -987,7 +1017,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, else: sym.flags.incl sfUsed of wLiftLocals: discard else: invalidPragma(it) - else: invalidPragma(it) + else: + n.sons[i] = semCustomPragma(c, it) + proc implicitPragmas*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) = @@ -1015,7 +1047,7 @@ proc hasPragma*(n: PNode, pragma: TSpecialWord): bool = return false for p in n.sons: - var key = if p.kind == nkExprColonExpr: p[0] else: p + var key = if p.kind in nkPragmaCallKinds and p.len > 1: p[0] else: p if key.kind == nkIdent and whichKeyword(key.ident) == pragma: return true diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 62489bd365..e737f7676d 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -951,7 +951,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = else: result = semMacroExpr(c, n, n, s, flags) of skTemplate: - if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0: + if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or + sfCustomPragma in sym.flags: markUsed(n.info, s, c.graph.usageSym) styleCheckUse(n.info, s) result = newSymNode(s, n.info) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 096cf99deb..ccddabcbec 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1153,6 +1153,9 @@ proc semProcAnnotation(c: PContext, prc: PNode; else: localError(prc.info, errOnlyACallOpCanBeDelegator) continue + elif sfCustomPragma in m.flags: + continue # semantic check for custom pragma happens later in semProcAux + # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and # let the semantic checker deal with it: var x = newNodeI(nkCall, n.info) diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index f90dff8f1a..454dadec09 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -608,7 +608,10 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = popOwner(c) s.ast = n result = n - if n.sons[bodyPos].kind == nkEmpty: + if sfCustomPragma in s.flags: + if n.sons[bodyPos].kind != nkEmpty: + localError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s) + elif n.sons[bodyPos].kind == nkEmpty: localError(n.info, errImplOfXexpected, s.name.s) var proto = searchForProc(c, c.currentScope, s) if proto == nil: diff --git a/doc/manual/pragmas.txt b/doc/manual/pragmas.txt index 835b6909da..cd26a9448b 100644 --- a/doc/manual/pragmas.txt +++ b/doc/manual/pragmas.txt @@ -1087,3 +1087,70 @@ In the above example, providing the -d flag causes the symbol ``FooBar`` to be overwritten at compile time, printing out 42. If the ``-d:FooBar=42`` were to be omitted, the default value of 5 would be used. + + +Custom annotations +------------------ +It is possible to define custom typed pragmas. Custom pragmas do not effect +code generation directly, but their presence can be detected by macros. +Custom pragmas are defined using templates annotated with pragma ``pragma``: + +.. code-block:: nim + template dbTable(name: string, table_space: string = nil) {.pragma.} + template dbKey(name: string = nil, primary_key: bool = false) {.pragma.} + template dbForeignKey(t: typedesc) {.pragma.} + template dbIgnore {.pragma.} + + +Consider stylized example of possible Object Relation Mapping (ORM) implementation: + +.. code-block:: nim + const tblspace {.strdefine.} = "dev" # switch for dev, test and prod environments + + type + User {.dbTable("users", tblspace).} = object + id {.dbKey(primary_key = true).}: int + name {.dbKey"full_name".}: string + is_cached {.dbIgnore.}: bool + age: int + + UserProfile {.dbTable("profiles", tblspace).} = object + id {.dbKey(primary_key = true).}: int + user_id {.dbForeignKey: User.}: int + read_access: bool + write_access: bool + admin_acess: bool + +In this example custom pragmas are used to describe how Nim objects are +mapped to the schema of the relational database. Custom pragmas can have +zero or more arguments. In order to pass multiple arguments use one of +template call syntaxes. All arguments are typed and follow standard +overload resolution rules for templates. Therefore, it is possible to have +default values for arguments, pass by name, varargs, etc. + +Custom pragmas can be used in all locations where ordinary pragmas can be +specified. It is possible to annotate procs, templates, type and variable +definitions, statements, etc. + +Macros module includes helpers which can be used to simplify custom pragma +access `hasCustomPragma`, `getCustomPragmaVal`. Please consult macros module +documentation for details. These macros are no magic, they don't do anything +you cannot do yourself by walking AST object representation. + +More examples with custom pragmas: + - Better serialization/deserialization control: + + .. code-block:: nim + type MyObj = object + a {.dontSerialize.}: int + b {.defaultDeserialize: 5.}: int + c {.serializationKey: "_c".}: string + + - Adopting type for gui inspector in a game engine: + + .. code-block:: nim + type MyComponent = object + position {.editable, animatable.}: Vector3 + alpha {.editRange: [0.0..1.0], animatable.}: float32 + + diff --git a/lib/core/macros.nim b/lib/core/macros.nim index b08a2198e1..ed9c304fe0 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -130,6 +130,7 @@ const nnkLiterals* = {nnkCharLit..nnkNilLit} nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, nnkCallStrLit} + nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit} proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated.} ## constructs an identifier from the string `s` @@ -1213,6 +1214,59 @@ macro expandMacros*(body: typed): untyped = result = getAst(inner(body)) echo result.toStrLit +proc customPragmaNode(n: NimNode): NimNode = + expectKind(n, {nnkSym, nnkDotExpr}) + if n.kind == nnkSym: + let sym = n.symbol.getImpl() + sym.expectRoutine() + result = sym.pragma + elif n.kind == nnkDotExpr: + let typDef = getImpl(getTypeInst(n[0]).symbol) + typDef.expectKind(nnkTypeDef) + typDef[2].expectKind(nnkObjectTy) + let recList = typDef[2][2] + for identDefs in recList: + for i in 0 .. identDefs.len - 3: + if identDefs[i].kind == nnkPragmaExpr and + identDefs[i][0].kind == nnkIdent and $identDefs[i][0] == $n[1]: + return identDefs[i][1] + +macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` + ## has custom pragma `cp`. + ## + ## .. code-block:: nim + ## template myAttr() {.pragma.} + ## type + ## MyObj = object + ## myField {.myAttr.}: int + ## var o: MyObj + ## assert(o.myField.hasCustomPragma(myAttr) == 0) + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if (p.kind == nnkSym and p == cp) or + (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp): + return newLit(true) + return newLit(false) + +macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to value of custom pragma `cp` of expression `n` which is expected + ## to be `nnkDotExpr`. + ## + ## .. code-block:: nim + ## template serializationKey(key: string) {.pragma.} + ## type + ## MyObj = object + ## myField {.serializationKey: "mf".}: int + ## var o: MyObj + ## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf") + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: + return p[1] + return newEmptyNode() + + when not defined(booting): template emit*(e: static[string]): untyped {.deprecated.} = ## accepts a single string argument and treats it as nim code diff --git a/tests/pragmas/custom_pragma.nim b/tests/pragmas/custom_pragma.nim new file mode 100644 index 0000000000..9e8f51deba --- /dev/null +++ b/tests/pragmas/custom_pragma.nim @@ -0,0 +1,5 @@ +# imported by tcustom_pragmas to test scoping + +template serializationKey*(s: string) {.pragma.} +template defaultValue*(V: typed) {.pragma.} +template alternativeKey*(s: string = nil, V: typed) {.pragma.} \ No newline at end of file diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim new file mode 100644 index 0000000000..a2380522ff --- /dev/null +++ b/tests/pragmas/tcustom_pragma.nim @@ -0,0 +1,43 @@ +import macros + +block: + template myAttr() {.pragma.} + + proc myProc():int {.myAttr.} = 2 + const myAttrIdx = myProc.hasCustomPragma(myAttr) + static: + assert(myAttrIdx) + +block: + template myAttr(a: string) {.pragma.} + + type MyObj = object + myField1, myField2 {.myAttr: "hi".}: int + var o: MyObj + static: + assert o.myField2.hasCustomPragma(myAttr) + assert(not o.myField1.hasCustomPragma(myAttr)) + +import custom_pragma +block: # A bit more advanced case + type + Subfield = object + c {.serializationKey: "cc".}: float + + MySerializable = object + a {.serializationKey"asdf", defaultValue: 5.} : int + b {.custom_pragma.defaultValue"hello".} : int + field: Subfield + d {.alternativeKey("df", 5).}: float + e {.alternativeKey(V = 5).}: seq[bool] + + var s: MySerializable + + const aDefVal = s.a.getCustomPragmaVal(defaultValue) + static: assert(aDefVal == 5) + + const aSerKey = s.a.getCustomPragmaVal(serializationKey) + static: assert(aSerKey == "asdf") + + const cSerKey = getCustomPragmaVal(s.field.c, serializationKey) + static: assert(cSerKey == "cc") From b6b99da08ff52095a6ab409aa2fa91d381179ab9 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Tue, 9 Jan 2018 17:48:27 +0300 Subject: [PATCH 168/200] Windows: fix a eraseLine bug (#7044) --- lib/pure/terminal.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index f15cee66ac..d9600a3a1d 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -391,12 +391,11 @@ proc eraseLine*(f: File) = origin.X = 0'i16 if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) - var ht: DWORD = scrbuf.dwSize.Y - origin.Y var wt: DWORD = scrbuf.dwSize.X - origin.X - if fillConsoleOutputCharacter(h, ' ', ht*wt, + if fillConsoleOutputCharacter(h, ' ', wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if fillConsoleOutputAttribute(h, scrbuf.wAttributes, ht * wt, + if fillConsoleOutputAttribute(h, scrbuf.wAttributes, wt, scrbuf.dwCursorPosition, addr(numwrote)) == 0: raiseOSError(osLastError()) else: From a54430ea2dc7d39670e0451a04d3b9f349ae8072 Mon Sep 17 00:00:00 2001 From: sleepyqt Date: Wed, 10 Jan 2018 01:02:09 +0300 Subject: [PATCH 169/200] Fix struct packing for VCC. (#7049) "#pragma pack(1)" sets current alligment without pushing into stack, so "#pragma pack(pop)" causing stack underflow. --- compiler/ccgtypes.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 24d3a0dfb4..8a60143fb7 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -496,7 +496,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode, if hasAttribute in CC[cCompiler].props: add(unionBody, "struct __attribute__((__packed__)){" ) else: - addf(unionBody, "#pragma pack(1)$nstruct{", []) + addf(unionBody, "#pragma pack(push, 1)$nstruct{", []) add(unionBody, a) addf(unionBody, "} $1;$n", [sname]) if tfPacked in rectype.flags and hasAttribute notin CC[cCompiler].props: @@ -551,7 +551,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, if hasAttribute in CC[cCompiler].props: result = structOrUnion(typ) & " __attribute__((__packed__))" else: - result = "#pragma pack(1)" & tnl & structOrUnion(typ) + result = "#pragma pack(push, 1)" & tnl & structOrUnion(typ) else: result = structOrUnion(typ) From 002840edff7de99b88a723dace1ceb770c00bfbf Mon Sep 17 00:00:00 2001 From: cooldome Date: Tue, 9 Jan 2018 22:29:52 +0000 Subject: [PATCH 170/200] Add warning 4809 to ignore list --- lib/nimbase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index 31075bbd2f..69699a2a4b 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -70,7 +70,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297) +# pragma warning(disable: 4710 4711 4774 4800 4809 4820 4996 4090 4297) #endif /* ------------------------------------------------------------------------- */ From 13e9f8aac1b8636c6a45b07c72fc27c1b311e824 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 10 Jan 2018 10:01:33 +0100 Subject: [PATCH 171/200] make tfragment_gc more robust for Appveyor --- tests/fragmentation/tfragment_gc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 92367a6dd9..3ebed23932 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -20,7 +20,7 @@ let total = getTotalMem() # Concrete values on Win64: 58.152MiB / 188.285MiB -let occupiedOk = occ < 64 * 1024 * 1024 +let occupiedOk = occ < 80 * 1024 * 1024 if not occupiedOk: echo "occupied ", formatSize(occ) echo "occupied ok: ", occupiedOk From a1016245cc9dd217222c2d03d875cac4314c16f1 Mon Sep 17 00:00:00 2001 From: rrenderr Date: Wed, 10 Jan 2018 13:41:29 +0200 Subject: [PATCH 172/200] - fixed lock of a program when proc echo is called on Android 5.x.x (#7054) --- lib/system/sysio.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 4348ffbb50..f638b299ce 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -406,7 +406,8 @@ proc setStdIoUnbuffered() = when declared(stdout): proc echoBinSafe(args: openArray[string]) {.compilerProc.} = - when not defined(windows): + # flockfile deadlocks some versions of Android 5.x.x + when not defined(windows) and not defined(android): proc flockfile(f: File) {.importc, noDecl.} proc funlockfile(f: File) {.importc, noDecl.} flockfile(stdout) @@ -415,7 +416,7 @@ when declared(stdout): const linefeed = "\n" # can be 1 or more chars discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) discard c_fflush(stdout) - when not defined(windows): + when not defined(windows) and not defined(android): funlockfile(stdout) {.pop.} From 2aebb8ed7ee3f442e478ec96e5af4c23957f6bb9 Mon Sep 17 00:00:00 2001 From: cooldome Date: Thu, 11 Jan 2018 00:57:20 +0000 Subject: [PATCH 173/200] Fix for isssue in parseBiggestFloat #7060 (#7061) --- lib/system/sysstr.nim | 7 ++++--- tests/float/tfloat4.nim | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 4c5f3d9a15..514223af21 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -386,8 +386,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, kdigits, fdigits = 0 exponent: int integer: uint64 - fraction: uint64 - frac_exponent= 0 + frac_exponent = 0 exp_sign = 1 first_digit = -1 has_sign = false @@ -480,7 +479,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if integer is representable in 53 bits: fast path # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) - if kdigits + fdigits <= 16 and first_digit <= 8: + let digits = kdigits + fdigits + if digits <= 15 or (digits <= 16 and first_digit <= 8): # max float power of ten with set bits above the 53th bit is 10^22 if abs_exponent <= 22: if exp_negative: @@ -504,6 +504,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, result = i - start i = start # re-parse without error checking, any error should be handled by the code above. + if s[i] == '.': i.inc while s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) diff --git a/tests/float/tfloat4.nim b/tests/float/tfloat4.nim index 559c8aacad..68df56be8c 100644 --- a/tests/float/tfloat4.nim +++ b/tests/float/tfloat4.nim @@ -48,5 +48,11 @@ doAssert "2.71828182845904523536028747".parseFloat == 2.71828182845904523536028747 doAssert 0.00097656250000000021684043449710088680149056017398834228515625 == "0.00097656250000000021684043449710088680149056017398834228515625".parseFloat +doAssert 0.00998333 == ".00998333".parseFloat +doAssert 0.00128333 == ".00128333".parseFloat +doAssert 999999999999999.0 == "999999999999999.0".parseFloat +doAssert 9999999999999999.0 == "9999999999999999.0".parseFloat +doAssert 0.999999999999999 == ".999999999999999".parseFloat +doAssert 0.9999999999999999 == ".9999999999999999".parseFloat echo("passed all tests.") From d0a9fac36252a135dba6304b9f7ccc18c899c267 Mon Sep 17 00:00:00 2001 From: jcosborn Date: Wed, 10 Jan 2018 19:01:51 -0600 Subject: [PATCH 174/200] avoid creating temporary in genObjConstr if possible (#7032) --- compiler/ccgexprs.nim | 48 +++++++++++++++++++++++------------ tests/objects/tobjconstr.nim | 7 ++++- tests/objects/tobjconstr2.nim | 7 +++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 0a312b4c72..bb9a9f6c60 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1202,18 +1202,30 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = # we skip this step here: if not p.module.compileToCpp: if handleConstExpr(p, e, d): return - var tmp: TLoc var t = e.typ.skipTypes(abstractInst) - getTemp(p, t, tmp) let isRef = t.kind == tyRef - var r = rdLoc(tmp) - if isRef: - rawGenNew(p, tmp, nil) - t = t.lastSon.skipTypes(abstractInst) - r = "(*$1)" % [r] - gcUsage(e) + + # check if we need to construct the object in a temporary + var useTemp = + isRef or + (d.k notin {locTemp,locLocalVar,locGlobalVar,locParam,locField}) or + (isPartOf(d.lode, e) != arNo) + + var tmp: TLoc + var r: Rope + if useTemp: + getTemp(p, t, tmp) + r = rdLoc(tmp) + if isRef: + rawGenNew(p, tmp, nil) + t = t.lastSon.skipTypes(abstractInst) + r = "(*$1)" % [r] + gcUsage(e) + else: + constructLoc(p, tmp) else: - constructLoc(p, tmp) + resetLoc(p, d) + r = rdLoc(d) discard getTypeDesc(p.module, t) let ty = getUniqueType(t) for i in 1 ..< e.len: @@ -1227,15 +1239,19 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = genFieldCheck(p, it.sons[2], r, field) add(tmp2.r, ".") add(tmp2.r, field.loc.r) - tmp2.k = locTemp + if useTemp: + tmp2.k = locTemp + tmp2.storage = if isRef: OnHeap else: OnStack + else: + tmp2.k = d.k + tmp2.storage = if isRef: OnHeap else: d.storage tmp2.lode = it.sons[1] - tmp2.storage = if isRef: OnHeap else: OnStack expr(p, it.sons[1], tmp2) - - if d.k == locNone: - d = tmp - else: - genAssignment(p, d, tmp, {}) + if useTemp: + if d.k == locNone: + d = tmp + else: + genAssignment(p, d, tmp, {}) proc lhsDoesAlias(a, b: PNode): bool = for y in b: diff --git a/tests/objects/tobjconstr.nim b/tests/objects/tobjconstr.nim index b7da176aae..d1f3c8bdb7 100644 --- a/tests/objects/tobjconstr.nim +++ b/tests/objects/tobjconstr.nim @@ -15,7 +15,8 @@ discard """ (y: 678, x: 123) (y: 678, x: 123) (y: 0, x: 123) -(y: 678, x: 123)''' +(y: 678, x: 123) +(y: 123, x: 678)''' """ type @@ -75,3 +76,7 @@ when true: echo b # (y: 0, x: 123) b=B(y: 678, x: 123) echo b # (y: 678, x: 123) + b=B(y: b.x, x: b.y) + echo b # (y: 123, x: 678) + +GC_fullCollect() diff --git a/tests/objects/tobjconstr2.nim b/tests/objects/tobjconstr2.nim index f6805190bb..6253edab00 100644 --- a/tests/objects/tobjconstr2.nim +++ b/tests/objects/tobjconstr2.nim @@ -1,3 +1,8 @@ +discard """ + output: '''42 +Foo''' +""" + type TFoo{.exportc.} = object x:int @@ -48,3 +53,5 @@ type NamedGraphic = object of Graphic2 var ngr = NamedGraphic(kind: Koo, radius: 6.9, name: "Foo") echo ngr.name + +GC_fullCollect() From e98a2051ce22c1f72c662e0c3e37597cbfaad6b0 Mon Sep 17 00:00:00 2001 From: oskca Date: Wed, 10 Jan 2018 23:20:18 +0800 Subject: [PATCH 175/200] check ERROR_NO_MORE_FILES to prevent walkDir[Rec] to quit prematurely --- lib/pure/os.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c18d032891..689fc8d4a6 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -672,7 +672,7 @@ template walkCommon(pattern: string, filter) = if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or pattern[dotPos+1] == '*': yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: break + if findNextFile(res, f) == 0'i32 and getLastError() == 18: break # ERROR_NO_MORE_FILES=18 else: # here we use glob var f: Glob @@ -782,7 +782,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) yield (k, xx) - if findNextFile(h, f) == 0'i32: break + if findNextFile(h, f) == 0'i32 and getLastError() == 18: break # ERROR_NO_MORE_FILES=18 else: var d = opendir(dir) if d != nil: From 495331bf20d5c3147290f0393581c4bd7292c89c Mon Sep 17 00:00:00 2001 From: oskca Date: Thu, 11 Jan 2018 13:40:25 +0800 Subject: [PATCH 176/200] raiseOSError to indicate the failling of findNextFile in walkDir[Rec] --- lib/pure/os.nim | 10 ++++++++-- lib/windows/winlean.nim | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 689fc8d4a6..1e32245373 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -672,7 +672,10 @@ template walkCommon(pattern: string, filter) = if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or pattern[dotPos+1] == '*': yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32 and getLastError() == 18: break # ERROR_NO_MORE_FILES=18 + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode, "findNextFile failed") else: # here we use glob var f: Glob @@ -782,7 +785,10 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) yield (k, xx) - if findNextFile(h, f) == 0'i32 and getLastError() == 18: break # ERROR_NO_MORE_FILES=18 + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode, "findNextFile failed") else: var d = opendir(dir) if d != nil: diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index a833377e5e..bd6e58a10f 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -686,6 +686,7 @@ const ERROR_FILE_NOT_FOUND* = 2 ERROR_PATH_NOT_FOUND* = 3 ERROR_ACCESS_DENIED* = 5 + ERROR_NO_MORE_FILES* = 18 ERROR_HANDLE_EOF* = 38 ERROR_BAD_ARGUMENTS* = 165 From 852e1d3da17c3a616a8e6d8cc5e3c711f527b52e Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Thu, 11 Jan 2018 11:50:13 +0200 Subject: [PATCH 177/200] logging: don't crash on nil strings --- lib/pure/logging.nim | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 830820fd13..751fc0e8dd 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -107,9 +107,14 @@ var proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string = ## Format a log message using the ``frmt`` format string, ``level`` and varargs. ## See the module documentation for the format string syntax. + const nilString = "nil" + var msgLen = 0 for arg in args: - msgLen += arg.len + if arg.isNil: + msgLen += nilString.len + else: + msgLen += arg.len result = newStringOfCap(frmt.len + msgLen + 20) var i = 0 while i < frmt.len: @@ -136,7 +141,10 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str of "levelname": result.add(LevelNames[level]) else: discard for arg in args: - result.add(arg) + if arg.isNil: + result.add(nilString) + else: + result.add(arg) method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {. raises: [Exception], gcsafe, @@ -361,3 +369,6 @@ when not defined(testing) and isMainModule: addHandler(L) for i in 0 .. 25: info("hello", i) + + var nilString: string + info "hello ", nilString From c9c44a4eb9d294ca1a0fe36e964536431b198035 Mon Sep 17 00:00:00 2001 From: oskca Date: Thu, 11 Jan 2018 19:42:19 +0800 Subject: [PATCH 178/200] correct type for raiseOSError --- lib/pure/os.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1e32245373..9d6dc4f809 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -675,7 +675,7 @@ template walkCommon(pattern: string, filter) = if findNextFile(res, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode, "findNextFile failed") + else: raiseOSError(errCode.OSErrorCode, "findNextFile failed") else: # here we use glob var f: Glob @@ -785,10 +785,10 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) yield (k, xx) - if findNextFile(res, f) == 0'i32: + if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode, "findNextFile failed") + else: raiseOSError(errCode.OSErrorCode, "findNextFile failed") else: var d = opendir(dir) if d != nil: From 8da96bc72bb17e9adddee52cb95393f4d6deef29 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 11 Jan 2018 12:16:34 +0000 Subject: [PATCH 179/200] Remove additionalInfo in OSError in findNextFile --- lib/pure/os.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 9d6dc4f809..a5db4ed22a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -675,7 +675,7 @@ template walkCommon(pattern: string, filter) = if findNextFile(res, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode, "findNextFile failed") + else: raiseOSError(errCode.OSErrorCode) else: # here we use glob var f: Glob @@ -788,7 +788,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode, "findNextFile failed") + else: raiseOSError(errCode.OSErrorCode) else: var d = opendir(dir) if d != nil: From d5cd8e6f71f640ec7b6d9feb369576c9b89de5b8 Mon Sep 17 00:00:00 2001 From: grazil <24716172+grazil@users.noreply.github.com> Date: Thu, 11 Jan 2018 14:16:48 +0100 Subject: [PATCH 180/200] add missing methods in js backend --- lib/js/dom.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index aa7f5d8396..6434785025 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -405,7 +405,7 @@ type # EventTarget "methods" proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), options: AddEventListenerOptions) - +proc removeEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) # Window "methods" proc alert*(w: Window, msg: cstring) @@ -507,6 +507,7 @@ proc replace*(loc: Location, s: cstring) proc back*(h: History) proc forward*(h: History) proc go*(h: History, pagesToJump: int) +proc pushState*(h: History, stateObject, title, url: cstring) # Navigator "methods" proc javaEnabled*(h: Navigator): bool From b7713859ab110cca3f785bd602f329dc82cc3190 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 11 Jan 2018 16:15:53 +0000 Subject: [PATCH 181/200] Use regex to match output of tasync_traceback. --- tests/async/tasync_traceback.nim | 129 +++++++++++++++++-------------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 08f7e7317d..8584cc9814 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -1,61 +1,6 @@ discard """ exitcode: 0 disabled: "windows" - output: ''' -b failure -Async traceback: - tasync_traceback.nim(97) tasync_traceback - asyncmacro.nim(395) a - asyncmacro.nim(34) a_continue - ## Resumes an async procedure - tasync_traceback.nim(95) aIter - asyncmacro.nim(395) b - asyncmacro.nim(34) b_continue - ## Resumes an async procedure - tasync_traceback.nim(92) bIter - #[ - tasync_traceback.nim(97) tasync_traceback - asyncmacro.nim(395) a - asyncmacro.nim(43) a_continue - ## Resumes an async procedure - asyncfutures.nim(211) callback= - asyncfutures.nim(190) addCallback - asyncfutures.nim(53) callSoon - asyncmacro.nim(34) a_continue - ## Resumes an async procedure - asyncmacro.nim(0) aIter - asyncfutures.nim(304) read - ]# -Exception message: b failure -Exception type: - -bar failure -Async traceback: - tasync_traceback.nim(113) tasync_traceback - asyncdispatch.nim(1492) waitFor - asyncdispatch.nim(1496) poll - ## Processes asynchronous completion events - asyncdispatch.nim(1262) runOnce - asyncdispatch.nim(183) processPendingCallbacks - ## Executes pending callbacks - asyncmacro.nim(34) bar_continue - ## Resumes an async procedure - tasync_traceback.nim(108) barIter - #[ - tasync_traceback.nim(113) tasync_traceback - asyncdispatch.nim(1492) waitFor - asyncdispatch.nim(1496) poll - ## Processes asynchronous completion events - asyncdispatch.nim(1262) runOnce - asyncdispatch.nim(183) processPendingCallbacks - ## Executes pending callbacks - asyncmacro.nim(34) foo_continue - ## Resumes an async procedure - asyncmacro.nim(0) fooIter - asyncfutures.nim(304) read - ]# -Exception message: bar failure -Exception type:''' """ import asyncdispatch @@ -87,6 +32,8 @@ import asyncdispatch # tasync_traceback.nim(21) a # tasync_traceback.nim(18) b +var result = "" + proc b(): Future[int] {.async.} = if true: raise newException(OSError, "b failure") @@ -98,8 +45,8 @@ let aFut = a() try: discard waitFor aFut except Exception as exc: - echo exc.msg -echo() + result.add(exc.msg & "\n") +result.add("\n") # From #6803 proc bar(): Future[string] {.async.} = @@ -110,7 +57,69 @@ proc bar(): Future[string] {.async.} = proc foo(): Future[string] {.async.} = return await bar() try: - echo waitFor(foo()) + result.add(waitFor(foo()) & "\n") except Exception as exc: - echo exc.msg -echo() \ No newline at end of file + result.add(exc.msg & "\n") +result.add("\n") + +# Use re to parse the result +import re +const expected = """ +b failure +Async traceback: + tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback + asyncmacro\.nim\(\d+?\)\s+?a + asyncmacro\.nim\(\d+?\)\s+?a_continue + ## Resumes an async procedure + tasync_traceback\.nim\(\d+?\)\s+?aIter + asyncmacro\.nim\(\d+?\)\s+?b + asyncmacro\.nim\(\d+?\)\s+?b_continue + ## Resumes an async procedure + tasync_traceback\.nim\(\d+?\)\s+?bIter + #\[ + tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback + asyncmacro\.nim\(\d+?\)\s+?a + asyncmacro\.nim\(\d+?\)\s+?a_continue + ## Resumes an async procedure + asyncmacro\.nim\(\d+?\)\s+?aIter + asyncfutures\.nim\(\d+?\)\s+?read + \]# +Exception message: b failure +Exception type: + +bar failure +Async traceback: + tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback + asyncdispatch\.nim\(\d+?\)\s+?waitFor + asyncdispatch\.nim\(\d+?\)\s+?poll + ## Processes asynchronous completion events + asyncdispatch\.nim\(\d+?\)\s+?runOnce + asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks + ## Executes pending callbacks + asyncmacro\.nim\(\d+?\)\s+?bar_continue + ## Resumes an async procedure + tasync_traceback\.nim\(\d+?\)\s+?barIter + #\[ + tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback + asyncdispatch\.nim\(\d+?\)\s+?waitFor + asyncdispatch\.nim\(\d+?\)\s+?poll + ## Processes asynchronous completion events + asyncdispatch\.nim\(\d+?\)\s+?runOnce + asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks + ## Executes pending callbacks + asyncmacro\.nim\(\d+?\)\s+?foo_continue + ## Resumes an async procedure + asyncmacro\.nim\(\d+?\)\s+?fooIter + asyncfutures\.nim\(\d+?\)\s+?read + \]# +Exception message: bar failure +Exception type: +""" + +if result.match(re(expected)): + echo("Matched") +else: + echo("Not matched!") + echo() + echo(result) + quit(QuitFailure) From 72d9485e8e0b839c3832ecb3b949e116fe9ed062 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 11 Jan 2018 16:52:14 +0000 Subject: [PATCH 182/200] Fix tasync_traceback test. --- tests/async/tasync_traceback.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 8584cc9814..e4c8a67b34 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -1,6 +1,7 @@ discard """ exitcode: 0 disabled: "windows" + output: "Matched" """ import asyncdispatch From df73d412ba219bc02d76aff89e95f85b33411ffa Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 11 Jan 2018 17:58:26 +0100 Subject: [PATCH 183/200] introduce --symbolFiles:v2 as the next attempt to bring symbol files to Nim --- compiler/commands.nim | 1 + compiler/options.nim | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 2d9f769597..18b328fd6d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -603,6 +603,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "off": gSymbolFiles = disabledSf of "writeonly": gSymbolFiles = writeOnlySf of "readonly": gSymbolFiles = readOnlySf + of "v2": gSymbolFiles = v2Sf else: localError(info, errOnOrOffExpectedButXFound, arg) of "skipcfg": expectNoArg(switch, arg, pass, info) diff --git a/compiler/options.nim b/compiler/options.nim index 0732e49896..8112f26b14 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -148,7 +148,7 @@ var type SymbolFilesOption* = enum - disabledSf, enabledSf, writeOnlySf, readOnlySf + disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf var gSymbolFiles*: SymbolFilesOption From fb8def869c0a372dd943297a981970adef59bb4d Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 11 Jan 2018 18:05:14 +0100 Subject: [PATCH 184/200] rename strformat.fmt to `%` as it works better with backslash escape sequences; refs #6958 --- lib/pure/strformat.nim | 214 ++++++++++++++++++------------------ tests/stdlib/tfrexp1.nim | 2 +- tests/stdlib/tstrformat.nim | 4 +- 3 files changed, 110 insertions(+), 110 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index d1dedb625c..0673c4fa89 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -15,27 +15,27 @@ Examples: .. code-block:: nim - doAssert fmt"""{"abc":>4}""" == " abc" - doAssert fmt"""{"abc":<4}""" == "abc " + doAssert %"""{"abc":>4}""" == " abc" + doAssert %"""{"abc":<4}""" == "abc " - doAssert fmt"{-12345:08}" == "-0012345" - doAssert fmt"{-1:3}" == " -1" - doAssert fmt"{-1:03}" == "-01" - doAssert fmt"{16:#X}" == "0x10" + doAssert %"{-12345:08}" == "-0012345" + doAssert %"{-1:3}" == " -1" + doAssert %"{-1:03}" == "-01" + doAssert %"{16:#X}" == "0x10" - doAssert fmt"{123.456}" == "123.456" - doAssert fmt"{123.456:>9.3f}" == " 123.456" - doAssert fmt"{123.456:9.3f}" == " 123.456" - doAssert fmt"{123.456:9.4f}" == " 123.4560" - doAssert fmt"{123.456:>9.0f}" == " 123." - doAssert fmt"{123.456:<9.4f}" == "123.4560 " + doAssert %"{123.456}" == "123.456" + doAssert %"{123.456:>9.3f}" == " 123.456" + doAssert %"{123.456:9.3f}" == " 123.456" + doAssert %"{123.456:9.4f}" == " 123.4560" + doAssert %"{123.456:>9.0f}" == " 123." + doAssert %"{123.456:<9.4f}" == "123.4560 " - doAssert fmt"{123.456:e}" == "1.234560e+02" - doAssert fmt"{123.456:>13e}" == " 1.234560e+02" - doAssert fmt"{123.456:13e}" == " 1.234560e+02" + doAssert %"{123.456:e}" == "1.234560e+02" + doAssert %"{123.456:>13e}" == " 1.234560e+02" + doAssert %"{123.456:13e}" == " 1.234560e+02" -An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: +An expression like ``%"{key} is {value:arg} {{z}}"`` is transformed into: .. code-block:: nim var temp = newStringOfCap(educatedCapGuess) @@ -48,13 +48,13 @@ An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: Parts of the string that are enclosed in the curly braces are interpreted as Nim code, to escape an ``{`` or ``}`` double it. -``fmt`` delegates most of the work to an open overloaded set +``%`` delegates most of the work to an open overloaded set of ``format`` procs. The required signature for a type ``T`` that supports formatting is usually ``proc format(x: T; result: var string)`` for efficiency but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are used as the fallback implementation. -This is the concrete lookup algorithm that ``fmt`` uses: +This is the concrete lookup algorithm that ``%`` uses: .. code-block:: nim @@ -69,7 +69,7 @@ This is the concrete lookup algorithm that ``fmt`` uses: The subexpression after the colon -(``arg`` in ``fmt"{key} is {value:arg} {{z}}"``) is an optional argument +(``arg`` in ``%"{key} is {value:arg} {{z}}"``) is an optional argument passed to ``format``. If an optional argument is present the following lookup algorithm is used: @@ -226,8 +226,8 @@ template callFormatOption(res, arg, option) {.dirty.} = else: format($arg, option, res) -macro fmt*(pattern: string{lit}): untyped = - ## For a specification of the ``fmt`` macro, see the module level documentation. +macro `%`*(pattern: string{lit}): untyped = + ## For a specification of the ``%`` macro, see the module level documentation. runnableExamples: template check(actual, expected: string) = doAssert actual == expected @@ -236,113 +236,113 @@ macro fmt*(pattern: string{lit}): untyped = # Basic tests let s = "string" - check fmt"{0} {s}", "0 string" - check fmt"{s[0..2].toUpperAscii}", "STR" - check fmt"{-10:04}", "-010" - check fmt"{-10:<04}", "-010" - check fmt"{-10:>04}", "-010" - check fmt"0x{10:02X}", "0x0A" + check %"{0} {s}", "0 string" + check %"{s[0..2].toUpperAscii}", "STR" + check %"{-10:04}", "-010" + check %"{-10:<04}", "-010" + check %"{-10:>04}", "-010" + check %"0x{10:02X}", "0x0A" - check fmt"{10:#04X}", "0x0A" + check %"{10:#04X}", "0x0A" - check fmt"""{"test":#>5}""", "#test" - check fmt"""{"test":>5}""", " test" + check %"""{"test":#>5}""", "#test" + check %"""{"test":>5}""", " test" - check fmt"""{"test":#^7}""", "#test##" + check %"""{"test":#^7}""", "#test##" - check fmt"""{"test": <5}""", "test " - check fmt"""{"test":<5}""", "test " - check fmt"{1f:.3f}", "1.000" - check fmt"Hello, {s}!", "Hello, string!" + check %"""{"test": <5}""", "test " + check %"""{"test":<5}""", "test " + check %"{1f:.3f}", "1.000" + check %"Hello, {s}!", "Hello, string!" # Tests for identifers without parenthesis - check fmt"{s} works{s}", "string worksstring" - check fmt"{s:>7}", " string" - doAssert(not compiles(fmt"{s_works}")) # parsed as identifier `s_works` + check %"{s} works{s}", "string worksstring" + check %"{s:>7}", " string" + doAssert(not compiles(%"{s_works}")) # parsed as identifier `s_works` # Misc general tests - check fmt"{{}}", "{}" - check fmt"{0}%", "0%" - check fmt"{0}%asdf", "0%asdf" - check fmt("\n{\"\\n\"}\n"), "\n\n\n" - check fmt"""{"abc"}s""", "abcs" + check %"{{}}", "{}" + check %"{0}%", "0%" + check %"{0}%asdf", "0%asdf" + check %("\n{\"\\n\"}\n"), "\n\n\n" + check %"""{"abc"}s""", "abcs" # String tests - check fmt"""{"abc"}""", "abc" - check fmt"""{"abc":>4}""", " abc" - check fmt"""{"abc":<4}""", "abc " - check fmt"""{"":>4}""", " " - check fmt"""{"":<4}""", " " + check %"""{"abc"}""", "abc" + check %"""{"abc":>4}""", " abc" + check %"""{"abc":<4}""", "abc " + check %"""{"":>4}""", " " + check %"""{"":<4}""", " " # Int tests - check fmt"{12345}", "12345" - check fmt"{ - 12345}", "-12345" - check fmt"{12345:6}", " 12345" - check fmt"{12345:>6}", " 12345" - check fmt"{12345:4}", "12345" - check fmt"{12345:08}", "00012345" - check fmt"{-12345:08}", "-0012345" - check fmt"{0:0}", "0" - check fmt"{0:02}", "00" - check fmt"{-1:3}", " -1" - check fmt"{-1:03}", "-01" - check fmt"{10}", "10" - check fmt"{16:#X}", "0x10" - check fmt"{16:^#7X}", " 0x10 " - check fmt"{16:^+#7X}", " +0x10 " + check %"{12345}", "12345" + check %"{ - 12345}", "-12345" + check %"{12345:6}", " 12345" + check %"{12345:>6}", " 12345" + check %"{12345:4}", "12345" + check %"{12345:08}", "00012345" + check %"{-12345:08}", "-0012345" + check %"{0:0}", "0" + check %"{0:02}", "00" + check %"{-1:3}", " -1" + check %"{-1:03}", "-01" + check %"{10}", "10" + check %"{16:#X}", "0x10" + check %"{16:^#7X}", " 0x10 " + check %"{16:^+#7X}", " +0x10 " # Hex tests - check fmt"{0:x}", "0" - check fmt"{-0:x}", "0" - check fmt"{255:x}", "ff" - check fmt"{255:X}", "FF" - check fmt"{-255:x}", "-ff" - check fmt"{-255:X}", "-FF" - check fmt"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" - check fmt"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check fmt"{255:4x}", " ff" - check fmt"{255:04x}", "00ff" - check fmt"{-255:4x}", " -ff" - check fmt"{-255:04x}", "-0ff" + check %"{0:x}", "0" + check %"{-0:x}", "0" + check %"{255:x}", "ff" + check %"{255:X}", "FF" + check %"{-255:x}", "-ff" + check %"{-255:X}", "-FF" + check %"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check %"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check %"{255:4x}", " ff" + check %"{255:04x}", "00ff" + check %"{-255:4x}", " -ff" + check %"{-255:04x}", "-0ff" # Float tests - check fmt"{123.456}", "123.456" - check fmt"{-123.456}", "-123.456" - check fmt"{123.456:.3f}", "123.456" - check fmt"{123.456:+.3f}", "+123.456" - check fmt"{-123.456:+.3f}", "-123.456" - check fmt"{-123.456:.3f}", "-123.456" - check fmt"{123.456:1g}", "123.456" - check fmt"{123.456:.1f}", "123.5" - check fmt"{123.456:.0f}", "123." - #check fmt"{123.456:.0f}", "123." - check fmt"{123.456:>9.3f}", " 123.456" - check fmt"{123.456:9.3f}", " 123.456" - check fmt"{123.456:>9.4f}", " 123.4560" - check fmt"{123.456:>9.0f}", " 123." - check fmt"{123.456:<9.4f}", "123.4560 " + check %"{123.456}", "123.456" + check %"{-123.456}", "-123.456" + check %"{123.456:.3f}", "123.456" + check %"{123.456:+.3f}", "+123.456" + check %"{-123.456:+.3f}", "-123.456" + check %"{-123.456:.3f}", "-123.456" + check %"{123.456:1g}", "123.456" + check %"{123.456:.1f}", "123.5" + check %"{123.456:.0f}", "123." + #check %"{123.456:.0f}", "123." + check %"{123.456:>9.3f}", " 123.456" + check %"{123.456:9.3f}", " 123.456" + check %"{123.456:>9.4f}", " 123.4560" + check %"{123.456:>9.0f}", " 123." + check %"{123.456:<9.4f}", "123.4560 " # Float (scientific) tests - check fmt"{123.456:e}", "1.234560e+02" - check fmt"{123.456:>13e}", " 1.234560e+02" - check fmt"{123.456:<13e}", "1.234560e+02 " - check fmt"{123.456:.1e}", "1.2e+02" - check fmt"{123.456:.2e}", "1.23e+02" - check fmt"{123.456:.3e}", "1.235e+02" + check %"{123.456:e}", "1.234560e+02" + check %"{123.456:>13e}", " 1.234560e+02" + check %"{123.456:<13e}", "1.234560e+02 " + check %"{123.456:.1e}", "1.2e+02" + check %"{123.456:.2e}", "1.23e+02" + check %"{123.456:.3e}", "1.235e+02" # Note: times.format adheres to the format protocol. Test that this # works: import times var nullTime: DateTime - check fmt"{nullTime:yyyy-mm-dd}", "0000-00-00" + check %"{nullTime:yyyy-mm-dd}", "0000-00-00" # Unicode string tests - check fmt"""{"αβγ"}""", "αβγ" - check fmt"""{"αβγ":>5}""", " αβγ" - check fmt"""{"αβγ":<5}""", "αβγ " - check fmt"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" - check fmt"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + check %"""{"αβγ"}""", "αβγ" + check %"""{"αβγ":>5}""", " αβγ" + check %"""{"αβγ":<5}""", "αβγ " + check %"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check %"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " # Invalid unicode sequences should be handled as plain strings. # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 let invalidUtf8 = [ @@ -351,10 +351,10 @@ macro fmt*(pattern: string{lit}): untyped = "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" ] for s in invalidUtf8: - check fmt"{s:>5}", repeat(" ", 5-s.len) & s + check %"{s:>5}", repeat(" ", 5-s.len) & s if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "fmt only works with string literals", pattern + error "% only works with string literals", pattern let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") @@ -560,7 +560,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; proc format*(value: SomeInteger; specifier: string; res: var string) = ## Standard format implementation for ``SomeInteger``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``%`` macro. let spec = parseStandardFormatSpecifier(specifier) var radix = 10 case spec.typ @@ -577,7 +577,7 @@ proc format*(value: SomeInteger; specifier: string; res: var string) = proc format*(value: SomeReal; specifier: string; res: var string) = ## Standard format implementation for ``SomeReal``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``%`` macro. let spec = parseStandardFormatSpecifier(specifier) var fmode = ffDefault @@ -609,7 +609,7 @@ proc format*(value: SomeReal; specifier: string; res: var string) = proc format*(value: string; specifier: string; res: var string) = ## Standard format implementation for ``string``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``%`` macro. let spec = parseStandardFormatSpecifier(specifier) case spec.typ of 's', '\0': discard diff --git a/tests/stdlib/tfrexp1.nim b/tests/stdlib/tfrexp1.nim index c6bb2b38cc..caed71250f 100644 --- a/tests/stdlib/tfrexp1.nim +++ b/tests/stdlib/tfrexp1.nim @@ -22,7 +22,7 @@ proc frexp_test(lo, hi, step: float64) = doAssert(abs(rslt - x) < eps) when manualTest: - echo fmt("x: {x:10.3f} exp: {exp:4d} frac: {frac:24.20f} check: {$(abs(rslt - x) < eps):-5s} {rslt: 9.3f}") + echo %("x: {x:10.3f} exp: {exp:4d} frac: {frac:24.20f} check: {$(abs(rslt - x) < eps):-5s} {rslt: 9.3f}") x += step when manualTest: diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim index 4e5c614a78..0c56a27f96 100644 --- a/tests/stdlib/tstrformat.nim +++ b/tests/stdlib/tstrformat.nim @@ -9,5 +9,5 @@ type Obj = object proc `$`(o: Obj): string = "foobar" var o: Obj -doAssert fmt"{o}" == "foobar" -doAssert fmt"{o:10}" == "foobar " \ No newline at end of file +doAssert %"{o}" == "foobar" +doAssert %"{o:10}" == "foobar " \ No newline at end of file From 4181e1940d9abaa8d39f247262db5c88322108c9 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 11 Jan 2018 20:46:44 +0000 Subject: [PATCH 185/200] recv with a timeout of -1 shouldn't wait on all data. --- lib/pure/net.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 083e70c8da..9215a18128 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1145,7 +1145,11 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. data.setLen(size) - result = recv(socket, cstring(data), size, timeout) + result = + if timeout == -1: + recv(socket, cstring(data), size) + else: + recv(socket, cstring(data), size, timeout) if result < 0: data.setLen(0) let lastError = getSocketError(socket) From cfc0a58417e4b6967fbb334e8d6556dccc1efad3 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Thu, 11 Jan 2018 23:02:42 +0200 Subject: [PATCH 186/200] Fixed crash in ssl httpclient --- lib/pure/httpclient.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 54a8498fa5..aefcff37a7 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -923,8 +923,14 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] if chunkSize <= 0: discard await recvFull(client, 2, client.timeout, false) # Skip \c\L break - discard await recvFull(client, chunkSize, client.timeout, true) - discard await recvFull(client, 2, client.timeout, false) # Skip \c\L + var bytesRead = await recvFull(client, chunkSize, client.timeout, true) + if bytesRead != chunkSize: + httpError("Server terminated connection prematurely") + + bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L + if bytesRead != 2: + httpError("Server terminated connection prematurely") + # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 @@ -965,7 +971,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": while true: let recvLen = await client.recvFull(4000, client.timeout, true) - if recvLen == 0: + if recvLen != 4000: client.close() break From 66634fe1a0cf4956a82315299b4fc43b855b3ab5 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 12 Jan 2018 01:27:24 +0100 Subject: [PATCH 187/200] strformat: introduce 'fmt' as an alias for '%'; ensure overloading resolution produces a clash between strformat.'%' and json.'%' --- lib/pure/strformat.nim | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 0673c4fa89..c771343c32 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -226,7 +226,7 @@ template callFormatOption(res, arg, option) {.dirty.} = else: format($arg, option, res) -macro `%`*(pattern: string{lit}): untyped = +macro `%`*(pattern: string): untyped = ## For a specification of the ``%`` macro, see the module level documentation. runnableExamples: template check(actual, expected: string) = @@ -407,6 +407,18 @@ macro `%`*(pattern: string{lit}): untyped = when defined(debugFmtDsl): echo repr result +template fmt*(pattern: string): untyped = + ## An alias for ``%``. Helps to avoid conflicts with ``json``'s ``%`` operator. + ## **Examples:** + ## + ## .. code-block:: nim + ## import json + ## import strformat except `%` + ## + ## let example = "oh, look no conflicts anymore" + ## echo fmt"{example}" + %pattern + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: @@ -618,3 +630,8 @@ proc format*(value: string; specifier: string; res: var string) = "invalid type in format string for string, expected 's', but got " & spec.typ) res.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + +when isMainModule: + import json + + doAssert fmt"{'a'} {'b'}" == "a b" From e7e0648829199531c9665f5168f5eec98c646854 Mon Sep 17 00:00:00 2001 From: smt Date: Fri, 12 Jan 2018 13:57:58 +0000 Subject: [PATCH 188/200] Update two links in tutorial to compiler usage docs with updated page anchors Seems like these class ids on the page were renamed to reflect their hierarchy but the tutorial links weren't pointing to them, this should fix that --- doc/tut1.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 9e6f1ab3c2..f935e79354 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -41,7 +41,7 @@ Save this code to the file "greetings.nim". Now compile and run it:: nim compile --run greetings.nim -With the ``--run`` `switch `_ Nim +With the ``--run`` `switch `_ Nim executes the file automatically after compilation. You can give your program command line arguments by appending them after the filename:: @@ -58,7 +58,7 @@ To compile a release version use:: By default the Nim compiler generates a large amount of runtime checks aiming for your debugging pleasure. With ``-d:release`` these checks are `turned off and optimizations are turned on -`_. +`_. Though it should be pretty obvious what the program does, I will explain the syntax: statements which are not indented are executed when the program From f1bf65f66e8bc3acb0afea4699a48c514189d355 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 12 Jan 2018 15:08:02 +0100 Subject: [PATCH 189/200] gc.nim: Add a gcAssert to enforce the no heap sharing restrictions --- lib/system/gc.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index dac06119d9..a0c943c350 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -639,6 +639,7 @@ when useMarkForDebug or useBackupGc: while gch.tempStack.len > 0: dec gch.tempStack.len var d = gch.tempStack.d[gch.tempStack.len] + gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected!" if not containsOrIncl(gch.marked, d): forAllChildren(d, waMarkPrecise) From 38fde80b35d866ca36db680498c0f32936369c7e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 13 Jan 2018 10:57:51 +0100 Subject: [PATCH 190/200] strformat: fixes new 'fmt' template --- lib/pure/strformat.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index c771343c32..62b095cb1f 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -417,6 +417,7 @@ template fmt*(pattern: string): untyped = ## ## let example = "oh, look no conflicts anymore" ## echo fmt"{example}" + bind `%` %pattern proc mkDigit(v: int, typ: char): string {.inline.} = From 6db1b492e12490be976c3aeda3a54eddf59cdc04 Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Sun, 14 Jan 2018 09:22:48 -0500 Subject: [PATCH 191/200] Include target in tester nimcache hash (#7053) --- tests/testament/categories.nim | 11 ++++++----- tests/testament/tester.nim | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 42e19d3dd0..06decfa3c7 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -17,11 +17,12 @@ const rodfilesDir = "tests/rodfiles" proc delNimCache(filename, options: string) = - let dir = nimcacheDir(filename, options) - try: - removeDir(dir) - except OSError: - echo "[Warning] could not delete: ", dir + for target in low(TTarget)..high(TTarget): + let dir = nimcacheDir(filename, options, target) + try: + removeDir(dir) + except OSError: + echo "[Warning] could not delete: ", dir proc runRodFiles(r: var TResults, cat: Category, options: string) = template test(filename: string, clearCacheFirst=false) = diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 870f9f8656..918de881ff 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -71,13 +71,14 @@ proc getFileDir(filename: string): string = if not result.isAbsolute(): result = getCurrentDir() / result -proc nimcacheDir(filename, options: string): string = +proc nimcacheDir(filename, options: string, target: TTarget): string = ## Give each test a private nimcache dir so they don't clobber each other's. - return "nimcache" / (filename & '_' & options.getMD5) + let hashInput = options & $target + return "nimcache" / (filename & '_' & hashInput.getMD5) proc callCompiler(cmdTemplate, filename, options: string, target: TTarget, extraOptions=""): TSpec = - let nimcache = nimcacheDir(filename, options) + let nimcache = nimcacheDir(filename, options, target) let options = options & " --nimCache:" & nimcache.quoteShell & extraOptions let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, @@ -231,7 +232,7 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarg proc generatedFile(test: TTest, target: TTarget): string = let (_, name, _) = test.name.splitFile let ext = targetToExt[target] - result = nimcacheDir(test.name, test.options) / + result = nimcacheDir(test.name, test.options, target) / (if target == targetJS: "" else: "compiler_") & name.changeFileExt(ext) @@ -334,7 +335,7 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = var exeFile: string if isJsTarget: let (_, file, _) = splitFile(tname) - exeFile = nimcacheDir(test.name, test.options) / file & ".js" + exeFile = nimcacheDir(test.name, test.options, target) / file & ".js" else: exeFile = changeFileExt(tname, ExeExt) From f71f9f83c2de79d57e89a7b6c3d66225f42c1482 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 14 Jan 2018 17:34:27 +0100 Subject: [PATCH 192/200] GC improvements; distinguish between thread local and globals in the marking step --- compiler/ccgstmts.nim | 8 ++- lib/system/alloc.nim | 8 +-- lib/system/gc.nim | 117 ++++++++++++++++----------------------- lib/system/gc2.nim | 24 +++----- lib/system/gc_common.nim | 24 ++++++++ lib/system/gc_ms.nim | 23 ++------ tests/gc/gctest.nim | 52 ++++++++++------- tests/gc/gctest.nim.cfg | 1 + 8 files changed, 129 insertions(+), 128 deletions(-) create mode 100644 tests/gc/gctest.nim.cfg diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 36816cc2c0..c9642b74d3 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -21,8 +21,12 @@ proc registerGcRoot(p: BProc, v: PSym) = # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) let prc = genTraverseProcForGlobal(p.module, v, v.info) - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterGlobalMarker($1);$n", [prc]) + if sfThread in v.flags: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "#nimRegisterThreadLocalMarker($1);$n", [prc]) + else: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "#nimRegisterGlobalMarker($1);$n", [prc]) proc isAssignedImmediately(n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index e274e8e0cd..46f75396b2 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -813,13 +813,13 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = zeroMem(result, size) proc dealloc(allocator: var MemRegion, p: pointer) = - sysAssert(p != nil, "dealloc 0") + sysAssert(p != nil, "dealloc: p is nil") var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) - sysAssert(x != nil, "dealloc 1") + sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") - sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2") + sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") rawDealloc(allocator, x) - sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3") + sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible") track("dealloc", p, 0) proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = diff --git a/lib/system/gc.nim b/lib/system/gc.nim index a0c943c350..cd5c6870dc 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -21,10 +21,6 @@ const # reaches this threshold # this seems to be a good value withRealTime = defined(useRealtimeGC) - useMarkForDebug = defined(gcGenerational) - useBackupGc = true # use a simple M&S GC to collect - # cycles instead of the complex - # algorithm when withRealTime and not declared(getTicks): include "system/timers" @@ -92,14 +88,12 @@ type maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 region: MemRegion # garbage collected region stat: GcStat - when useMarkForDebug or useBackupGc: - marked: CellSet - additionalRoots: CellSeq # dummy roots for GC_ref/unref + marked: CellSet + additionalRoots: CellSeq # dummy roots for GC_ref/unref when hasThreadSupport: toDispose: SharedList[pointer] + isMainThread: bool -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, - TGcStat: GcStat].} var gch {.rtlThreadVar.}: GcHeap @@ -314,27 +308,11 @@ proc initGC() = init(gch.zct) init(gch.tempStack) init(gch.decStack) - when useMarkForDebug or useBackupGc: - init(gch.marked) - init(gch.additionalRoots) + init(gch.marked) + init(gch.additionalRoots) when hasThreadSupport: init(gch.toDispose) - -when useMarkForDebug or useBackupGc: - type - GlobalMarkerProc = proc () {.nimcall, benign.} - {.deprecated: [TGlobalMarkerProc: GlobalMarkerProc].} - var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - - proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 + gch.isMainThread = true proc cellsetReset(s: var CellSet) = deinit(s) @@ -377,10 +355,10 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = else: discard proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: 4" + gcAssert(cell != nil, "forAllChildren: cell is nil") + gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: pointer not part of the heap") + gcAssert(cell.typ != nil, "forAllChildren: cell.typ is nil") + gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) @@ -623,30 +601,31 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -when useBackupGc: - proc sweep(gch: var GcHeap) = - for x in allObjects(gch.region): - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - if c notin gch.marked: freeCyclicCell(gch, c) +proc sweep(gch: var GcHeap) = + for x in allObjects(gch.region): + if isCell(x): + # cast to PCell is correct here: + var c = cast[PCell](x) + if c notin gch.marked: freeCyclicCell(gch, c) -when useMarkForDebug or useBackupGc: - proc markS(gch: var GcHeap, c: PCell) = - incl(gch.marked, c) - gcAssert gch.tempStack.len == 0, "stack not empty!" - forAllChildren(c, waMarkPrecise) - while gch.tempStack.len > 0: - dec gch.tempStack.len - var d = gch.tempStack.d[gch.tempStack.len] - gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected!" - if not containsOrIncl(gch.marked, d): - forAllChildren(d, waMarkPrecise) +proc markS(gch: var GcHeap, c: PCell) = + gcAssert isAllocatedPtr(gch.region, c), "markS: foreign heap root detected A!" + incl(gch.marked, c) + gcAssert gch.tempStack.len == 0, "stack not empty!" + forAllChildren(c, waMarkPrecise) + while gch.tempStack.len > 0: + dec gch.tempStack.len + var d = gch.tempStack.d[gch.tempStack.len] + gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected B!" + if not containsOrIncl(gch.marked, d): + forAllChildren(d, waMarkPrecise) - proc markGlobals(gch: var GcHeap) = +proc markGlobals(gch: var GcHeap) = + if gch.isMainThread: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - let d = gch.additionalRoots.d - for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i]) + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() + let d = gch.additionalRoots.d + for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i]) when logGC: var @@ -690,16 +669,15 @@ proc doOperation(p: pointer, op: WalkOp) = of waPush: add(gch.tempStack, c) of waMarkGlobal: - when useMarkForDebug or useBackupGc: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - markS(gch, c) - else: + when hasThreadSupport: + # could point to a cell which we don't own and don't want to touch/trace + # XXX: This should not be required anymore! + if isAllocatedPtr(gch.region, c): markS(gch, c) + else: + markS(gch, c) of waMarkPrecise: - when useMarkForDebug or useBackupGc: - add(gch.tempStack, c) + add(gch.tempStack, c) #of waDebug: debugGraph(c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = @@ -713,14 +691,13 @@ proc collectCycles(gch: var GcHeap) = nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) - when useBackupGc: - cellsetReset(gch.marked) - var d = gch.decStack.d - for i in 0..gch.decStack.len-1: - sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" - markS(gch, d[i]) - markGlobals(gch) - sweep(gch) + cellsetReset(gch.marked) + var d = gch.decStack.d + for i in 0..gch.decStack.len-1: + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -861,7 +838,7 @@ proc collectCT(gch: var GcHeap) = if (gch.zct.len >= stackMarkCosts or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and gch.recGcLock == 0: - when useMarkForDebug: + when false: prepareForInteriorPointerChecking(gch.region) cellsetReset(gch.marked) markForDebug(gch) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index d57a01dc75..cd90c6d622 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -104,6 +104,7 @@ type pDumpHeapFile: pointer # File that is used for GC_dumpHeap when hasThreadSupport: toDispose: SharedList[pointer] + isMainThread: bool var gch {.rtlThreadVar.}: GcHeap @@ -134,6 +135,7 @@ proc initGC() = init(gch.greyStack) when hasThreadSupport: init(gch.toDispose) + gch.isMainThread = true # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only @@ -284,20 +286,6 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = if not isOnStack(dest): markGrey(s) dest[] = src -type - GlobalMarkerProc = proc () {.nimcall, benign.} -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) case n.kind @@ -492,7 +480,9 @@ proc GC_dumpHeap*(file: File) = c_fprintf(file, "onstack %p\n", d[i]) else: c_fprintf(file, "onstack_invalid %p\n", d[i]) - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.isMainThread: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() while true: let x = allObjectsAsProc(gch.region, addr spaceIter) if spaceIter.state < 0: break @@ -579,7 +569,9 @@ proc markIncremental(gch: var GcHeap): bool = result = true proc markGlobals(gch: var GcHeap) = - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.isMainThread: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() proc doOperation(p: pointer, op: WalkOp) = if p == nil: return diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 484a4db9a0..f0abd918a2 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -393,3 +393,27 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = zeroMem(addr gch.region, sizeof(gch.region)) if allowGcAfterwards: initGC() + +type + GlobalMarkerProc = proc () {.nimcall, benign.} +var + globalMarkersLen: int + globalMarkers: array[0.. 3499, GlobalMarkerProc] + threadLocalMarkersLen: int + threadLocalMarkers: array[0.. 3499, GlobalMarkerProc] + +proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = + if globalMarkersLen <= high(globalMarkers): + globalMarkers[globalMarkersLen] = markerProc + inc globalMarkersLen + else: + echo "[GC] cannot register global variable; too many global variables" + quit 1 + +proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = + if threadLocalMarkersLen <= high(threadLocalMarkers): + threadLocalMarkers[threadLocalMarkersLen] = markerProc + inc threadLocalMarkersLen + else: + echo "[GC] cannot register thread local variable; too many thread local variables" + quit 1 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 5fc48d848b..0754f2cfee 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -40,8 +40,6 @@ type # A ref type can have a finalizer that is called before the object's # storage is freed. - GlobalMarkerProc = proc () {.nimcall, benign.} - GcStat = object collections: int # number of performed full collections maxThreshold: int # max threshold that has been set @@ -75,9 +73,9 @@ type stat: GcStat when hasThreadSupport: toDispose: SharedList[pointer] + isMainThread: bool additionalRoots: CellSeq # dummy roots for GC_ref/unref -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, - TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} + var gch {.rtlThreadVar.}: GcHeap @@ -119,18 +117,6 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} = proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = result = 0 -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - # this that has to equals zero, otherwise we have to round up UnitsPerPage: when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} @@ -234,6 +220,7 @@ proc initGC() = init(gch.marked) when hasThreadSupport: init(gch.toDispose) + gch.isMainThread = true proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -450,7 +437,9 @@ when false: quit 1 proc markGlobals(gch: var GcHeap) = - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.isMainThread: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() let d = gch.additionalRoots.d for i in 0 .. gch.additionalRoots.len-1: mark(gch, d[i]) diff --git a/tests/gc/gctest.nim b/tests/gc/gctest.nim index f5c81f033a..6d7cf9985d 100644 --- a/tests/gc/gctest.nim +++ b/tests/gc/gctest.nim @@ -179,24 +179,38 @@ proc main() = write(stdout, "done!\n") var - father: TBNode - s: string -s = "" -s = "" -writeLine(stdout, repr(caseTree())) -father.t.data = @["ha", "lets", "stress", "it"] -father.t.data = @["ha", "lets", "stress", "it"] -var t = buildTree() -write(stdout, repr(t[])) -buildBTree(father) -write(stdout, repr(father)) + father {.threadvar.}: TBNode + s {.threadvar.}: string -write(stdout, "starting main...\n") -main() + fatherAsGlobal: TBNode -GC_fullCollect() -# the M&S GC fails with this call and it's unclear why. Definitely something -# we need to fix! -GC_fullCollect() -writeLine(stdout, GC_getStatistics()) -write(stdout, "finished\n") +proc start = + s = "" + s = "" + writeLine(stdout, repr(caseTree())) + father.t.data = @["ha", "lets", "stress", "it"] + father.t.data = @["ha", "lets", "stress", "it"] + var t = buildTree() + write(stdout, repr(t[])) + buildBTree(father) + write(stdout, repr(father)) + + write(stdout, "starting main...\n") + main() + + GC_fullCollect() + # the M&S GC fails with this call and it's unclear why. Definitely something + # we need to fix! + #GC_fullCollect() + writeLine(stdout, GC_getStatistics()) + write(stdout, "finished\n") + +#fatherAsGlobal.t.data = @["ha", "lets", "stress", "it"] +#var tg = buildTree() +#buildBTree(fatherAsGlobal) + +var thr: array[8, Thread[void]] +for i in low(thr)..high(thr): + createThread(thr[i], start) +joinThreads(thr) +start() diff --git a/tests/gc/gctest.nim.cfg b/tests/gc/gctest.nim.cfg new file mode 100644 index 0000000000..aed303eef8 --- /dev/null +++ b/tests/gc/gctest.nim.cfg @@ -0,0 +1 @@ +--threads:on From a2b7fcdb4ddf55122f158d54a5364df48187f752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Renaud=20Ch=C3=A9nard?= <24716172+grazil@users.noreply.github.com> Date: Sun, 14 Jan 2018 23:12:59 +0100 Subject: [PATCH 193/200] Use generic for 'stateObject' in func 'pushState' --- lib/js/dom.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 6434785025..55692d47dd 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -507,7 +507,7 @@ proc replace*(loc: Location, s: cstring) proc back*(h: History) proc forward*(h: History) proc go*(h: History, pagesToJump: int) -proc pushState*(h: History, stateObject, title, url: cstring) +proc pushState*[T](h: History, stateObject: T, title, url: cstring) # Navigator "methods" proc javaEnabled*(h: Navigator): bool From 9a60eae631e9f778d3920f3a66de0e726e687eef Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 14 Jan 2018 23:49:53 +0100 Subject: [PATCH 194/200] fixes #7078 --- lib/pure/strformat.nim | 222 ++++++++++++++++++------------------ tests/stdlib/tfrexp1.nim | 2 +- tests/stdlib/tstrformat.nim | 4 +- 3 files changed, 114 insertions(+), 114 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 62b095cb1f..ba21f894f8 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -15,27 +15,27 @@ Examples: .. code-block:: nim - doAssert %"""{"abc":>4}""" == " abc" - doAssert %"""{"abc":<4}""" == "abc " + doAssert &"""{"abc":>4}""" == " abc" + doAssert &"""{"abc":<4}""" == "abc " - doAssert %"{-12345:08}" == "-0012345" - doAssert %"{-1:3}" == " -1" - doAssert %"{-1:03}" == "-01" - doAssert %"{16:#X}" == "0x10" + doAssert &"{-12345:08}" == "-0012345" + doAssert &"{-1:3}" == " -1" + doAssert &"{-1:03}" == "-01" + doAssert &"{16:#X}" == "0x10" - doAssert %"{123.456}" == "123.456" - doAssert %"{123.456:>9.3f}" == " 123.456" - doAssert %"{123.456:9.3f}" == " 123.456" - doAssert %"{123.456:9.4f}" == " 123.4560" - doAssert %"{123.456:>9.0f}" == " 123." - doAssert %"{123.456:<9.4f}" == "123.4560 " + doAssert &"{123.456}" == "123.456" + doAssert &"{123.456:>9.3f}" == " 123.456" + doAssert &"{123.456:9.3f}" == " 123.456" + doAssert &"{123.456:9.4f}" == " 123.4560" + doAssert &"{123.456:>9.0f}" == " 123." + doAssert &"{123.456:<9.4f}" == "123.4560 " - doAssert %"{123.456:e}" == "1.234560e+02" - doAssert %"{123.456:>13e}" == " 1.234560e+02" - doAssert %"{123.456:13e}" == " 1.234560e+02" + doAssert &"{123.456:e}" == "1.234560e+02" + doAssert &"{123.456:>13e}" == " 1.234560e+02" + doAssert &"{123.456:13e}" == " 1.234560e+02" -An expression like ``%"{key} is {value:arg} {{z}}"`` is transformed into: +An expression like ``&"{key} is {value:arg} {{z}}"`` is transformed into: .. code-block:: nim var temp = newStringOfCap(educatedCapGuess) @@ -48,13 +48,13 @@ An expression like ``%"{key} is {value:arg} {{z}}"`` is transformed into: Parts of the string that are enclosed in the curly braces are interpreted as Nim code, to escape an ``{`` or ``}`` double it. -``%`` delegates most of the work to an open overloaded set +``&`` delegates most of the work to an open overloaded set of ``format`` procs. The required signature for a type ``T`` that supports formatting is usually ``proc format(x: T; result: var string)`` for efficiency but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are used as the fallback implementation. -This is the concrete lookup algorithm that ``%`` uses: +This is the concrete lookup algorithm that ``&`` uses: .. code-block:: nim @@ -69,7 +69,7 @@ This is the concrete lookup algorithm that ``%`` uses: The subexpression after the colon -(``arg`` in ``%"{key} is {value:arg} {{z}}"``) is an optional argument +(``arg`` in ``&"{key} is {value:arg} {{z}}"``) is an optional argument passed to ``format``. If an optional argument is present the following lookup algorithm is used: @@ -226,8 +226,8 @@ template callFormatOption(res, arg, option) {.dirty.} = else: format($arg, option, res) -macro `%`*(pattern: string): untyped = - ## For a specification of the ``%`` macro, see the module level documentation. +macro `&`*(pattern: string): untyped = + ## For a specification of the ``&`` macro, see the module level documentation. runnableExamples: template check(actual, expected: string) = doAssert actual == expected @@ -236,113 +236,113 @@ macro `%`*(pattern: string): untyped = # Basic tests let s = "string" - check %"{0} {s}", "0 string" - check %"{s[0..2].toUpperAscii}", "STR" - check %"{-10:04}", "-010" - check %"{-10:<04}", "-010" - check %"{-10:>04}", "-010" - check %"0x{10:02X}", "0x0A" + check &"{0} {s}", "0 string" + check &"{s[0..2].toUpperAscii}", "STR" + check &"{-10:04}", "-010" + check &"{-10:<04}", "-010" + check &"{-10:>04}", "-010" + check &"0x{10:02X}", "0x0A" - check %"{10:#04X}", "0x0A" + check &"{10:#04X}", "0x0A" - check %"""{"test":#>5}""", "#test" - check %"""{"test":>5}""", " test" + check &"""{"test":#>5}""", "#test" + check &"""{"test":>5}""", " test" - check %"""{"test":#^7}""", "#test##" + check &"""{"test":#^7}""", "#test##" - check %"""{"test": <5}""", "test " - check %"""{"test":<5}""", "test " - check %"{1f:.3f}", "1.000" - check %"Hello, {s}!", "Hello, string!" + check &"""{"test": <5}""", "test " + check &"""{"test":<5}""", "test " + check &"{1f:.3f}", "1.000" + check &"Hello, {s}!", "Hello, string!" # Tests for identifers without parenthesis - check %"{s} works{s}", "string worksstring" - check %"{s:>7}", " string" - doAssert(not compiles(%"{s_works}")) # parsed as identifier `s_works` + check &"{s} works{s}", "string worksstring" + check &"{s:>7}", " string" + doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works` # Misc general tests - check %"{{}}", "{}" - check %"{0}%", "0%" - check %"{0}%asdf", "0%asdf" - check %("\n{\"\\n\"}\n"), "\n\n\n" - check %"""{"abc"}s""", "abcs" + check &"{{}}", "{}" + check &"{0}%", "0%" + check &"{0}%asdf", "0%asdf" + check &("\n{\"\\n\"}\n"), "\n\n\n" + check &"""{"abc"}s""", "abcs" # String tests - check %"""{"abc"}""", "abc" - check %"""{"abc":>4}""", " abc" - check %"""{"abc":<4}""", "abc " - check %"""{"":>4}""", " " - check %"""{"":<4}""", " " + check &"""{"abc"}""", "abc" + check &"""{"abc":>4}""", " abc" + check &"""{"abc":<4}""", "abc " + check &"""{"":>4}""", " " + check &"""{"":<4}""", " " # Int tests - check %"{12345}", "12345" - check %"{ - 12345}", "-12345" - check %"{12345:6}", " 12345" - check %"{12345:>6}", " 12345" - check %"{12345:4}", "12345" - check %"{12345:08}", "00012345" - check %"{-12345:08}", "-0012345" - check %"{0:0}", "0" - check %"{0:02}", "00" - check %"{-1:3}", " -1" - check %"{-1:03}", "-01" - check %"{10}", "10" - check %"{16:#X}", "0x10" - check %"{16:^#7X}", " 0x10 " - check %"{16:^+#7X}", " +0x10 " + check &"{12345}", "12345" + check &"{ - 12345}", "-12345" + check &"{12345:6}", " 12345" + check &"{12345:>6}", " 12345" + check &"{12345:4}", "12345" + check &"{12345:08}", "00012345" + check &"{-12345:08}", "-0012345" + check &"{0:0}", "0" + check &"{0:02}", "00" + check &"{-1:3}", " -1" + check &"{-1:03}", "-01" + check &"{10}", "10" + check &"{16:#X}", "0x10" + check &"{16:^#7X}", " 0x10 " + check &"{16:^+#7X}", " +0x10 " # Hex tests - check %"{0:x}", "0" - check %"{-0:x}", "0" - check %"{255:x}", "ff" - check %"{255:X}", "FF" - check %"{-255:x}", "-ff" - check %"{-255:X}", "-FF" - check %"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" - check %"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check %"{255:4x}", " ff" - check %"{255:04x}", "00ff" - check %"{-255:4x}", " -ff" - check %"{-255:04x}", "-0ff" + check &"{0:x}", "0" + check &"{-0:x}", "0" + check &"{255:x}", "ff" + check &"{255:X}", "FF" + check &"{-255:x}", "-ff" + check &"{-255:X}", "-FF" + check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check &"{255:4x}", " ff" + check &"{255:04x}", "00ff" + check &"{-255:4x}", " -ff" + check &"{-255:04x}", "-0ff" # Float tests - check %"{123.456}", "123.456" - check %"{-123.456}", "-123.456" - check %"{123.456:.3f}", "123.456" - check %"{123.456:+.3f}", "+123.456" - check %"{-123.456:+.3f}", "-123.456" - check %"{-123.456:.3f}", "-123.456" - check %"{123.456:1g}", "123.456" - check %"{123.456:.1f}", "123.5" - check %"{123.456:.0f}", "123." - #check %"{123.456:.0f}", "123." - check %"{123.456:>9.3f}", " 123.456" - check %"{123.456:9.3f}", " 123.456" - check %"{123.456:>9.4f}", " 123.4560" - check %"{123.456:>9.0f}", " 123." - check %"{123.456:<9.4f}", "123.4560 " + check &"{123.456}", "123.456" + check &"{-123.456}", "-123.456" + check &"{123.456:.3f}", "123.456" + check &"{123.456:+.3f}", "+123.456" + check &"{-123.456:+.3f}", "-123.456" + check &"{-123.456:.3f}", "-123.456" + check &"{123.456:1g}", "123.456" + check &"{123.456:.1f}", "123.5" + check &"{123.456:.0f}", "123." + #check &"{123.456:.0f}", "123." + check &"{123.456:>9.3f}", " 123.456" + check &"{123.456:9.3f}", " 123.456" + check &"{123.456:>9.4f}", " 123.4560" + check &"{123.456:>9.0f}", " 123." + check &"{123.456:<9.4f}", "123.4560 " # Float (scientific) tests - check %"{123.456:e}", "1.234560e+02" - check %"{123.456:>13e}", " 1.234560e+02" - check %"{123.456:<13e}", "1.234560e+02 " - check %"{123.456:.1e}", "1.2e+02" - check %"{123.456:.2e}", "1.23e+02" - check %"{123.456:.3e}", "1.235e+02" + check &"{123.456:e}", "1.234560e+02" + check &"{123.456:>13e}", " 1.234560e+02" + check &"{123.456:<13e}", "1.234560e+02 " + check &"{123.456:.1e}", "1.2e+02" + check &"{123.456:.2e}", "1.23e+02" + check &"{123.456:.3e}", "1.235e+02" # Note: times.format adheres to the format protocol. Test that this # works: import times var nullTime: DateTime - check %"{nullTime:yyyy-mm-dd}", "0000-00-00" + check &"{nullTime:yyyy-mm-dd}", "0000-00-00" # Unicode string tests - check %"""{"αβγ"}""", "αβγ" - check %"""{"αβγ":>5}""", " αβγ" - check %"""{"αβγ":<5}""", "αβγ " - check %"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" - check %"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + check &"""{"αβγ"}""", "αβγ" + check &"""{"αβγ":>5}""", " αβγ" + check &"""{"αβγ":<5}""", "αβγ " + check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " # Invalid unicode sequences should be handled as plain strings. # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 let invalidUtf8 = [ @@ -351,10 +351,10 @@ macro `%`*(pattern: string): untyped = "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" ] for s in invalidUtf8: - check %"{s:>5}", repeat(" ", 5-s.len) & s + check &"{s:>5}", repeat(" ", 5-s.len) & s if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "% only works with string literals", pattern + error "& only works with string literals", pattern let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") @@ -408,17 +408,17 @@ macro `%`*(pattern: string): untyped = echo repr result template fmt*(pattern: string): untyped = - ## An alias for ``%``. Helps to avoid conflicts with ``json``'s ``%`` operator. + ## An alias for ``&``. ## **Examples:** ## ## .. code-block:: nim ## import json - ## import strformat except `%` + ## import strformat except `&` ## ## let example = "oh, look no conflicts anymore" ## echo fmt"{example}" - bind `%` - %pattern + bind `&` + &pattern proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) @@ -573,7 +573,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; proc format*(value: SomeInteger; specifier: string; res: var string) = ## Standard format implementation for ``SomeInteger``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``%`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) var radix = 10 case spec.typ @@ -590,7 +590,7 @@ proc format*(value: SomeInteger; specifier: string; res: var string) = proc format*(value: SomeReal; specifier: string; res: var string) = ## Standard format implementation for ``SomeReal``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``%`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) var fmode = ffDefault @@ -622,7 +622,7 @@ proc format*(value: SomeReal; specifier: string; res: var string) = proc format*(value: string; specifier: string; res: var string) = ## Standard format implementation for ``string``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``%`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) case spec.typ of 's', '\0': discard diff --git a/tests/stdlib/tfrexp1.nim b/tests/stdlib/tfrexp1.nim index caed71250f..c6bb2b38cc 100644 --- a/tests/stdlib/tfrexp1.nim +++ b/tests/stdlib/tfrexp1.nim @@ -22,7 +22,7 @@ proc frexp_test(lo, hi, step: float64) = doAssert(abs(rslt - x) < eps) when manualTest: - echo %("x: {x:10.3f} exp: {exp:4d} frac: {frac:24.20f} check: {$(abs(rslt - x) < eps):-5s} {rslt: 9.3f}") + echo fmt("x: {x:10.3f} exp: {exp:4d} frac: {frac:24.20f} check: {$(abs(rslt - x) < eps):-5s} {rslt: 9.3f}") x += step when manualTest: diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim index 0c56a27f96..4e5c614a78 100644 --- a/tests/stdlib/tstrformat.nim +++ b/tests/stdlib/tstrformat.nim @@ -9,5 +9,5 @@ type Obj = object proc `$`(o: Obj): string = "foobar" var o: Obj -doAssert %"{o}" == "foobar" -doAssert %"{o:10}" == "foobar " \ No newline at end of file +doAssert fmt"{o}" == "foobar" +doAssert fmt"{o:10}" == "foobar " \ No newline at end of file From 2d907ac334659a06de8c956c788cfe64ee0d240d Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 15 Jan 2018 00:09:11 +0100 Subject: [PATCH 195/200] make tests green again --- tests/concepts/t3330.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/concepts/t3330.nim b/tests/concepts/t3330.nim index 722c0a0e0a..bf8bffc7d4 100644 --- a/tests/concepts/t3330.nim +++ b/tests/concepts/t3330.nim @@ -6,9 +6,9 @@ but expected one of: proc test(foo: Foo[int]) t3330.nim(25, 8) Hint: Non-matching candidates for add(k, string, T) proc add(x: var string; y: string) -proc add(result: var string; x: float) proc add(x: var string; y: char) proc add(result: var string; x: int64) +proc add(result: var string; x: float) proc add(x: var string; y: cstring) proc add[T](x: var seq[T]; y: openArray[T]) proc add[T](x: var seq[T]; y: T) From 24a6583fa76289bfd725b61b9bd5effad7f5765a Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 15 Jan 2018 00:48:52 +0100 Subject: [PATCH 196/200] hardened gctest --- tests/gc/gctest.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/gc/gctest.nim b/tests/gc/gctest.nim index 6d7cf9985d..7f5260200c 100644 --- a/tests/gc/gctest.nim +++ b/tests/gc/gctest.nim @@ -205,9 +205,9 @@ proc start = writeLine(stdout, GC_getStatistics()) write(stdout, "finished\n") -#fatherAsGlobal.t.data = @["ha", "lets", "stress", "it"] -#var tg = buildTree() -#buildBTree(fatherAsGlobal) +fatherAsGlobal.t.data = @["ha", "lets", "stress", "it"] +var tg = buildTree() +buildBTree(fatherAsGlobal) var thr: array[8, Thread[void]] for i in low(thr)..high(thr): From f1089db1755c1e8cc5bcf965cac64014005cac3a Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 15 Jan 2018 17:41:05 +0100 Subject: [PATCH 197/200] GC: enable precise global/thread local storage tracing --- lib/system/gc.nim | 33 +++++++++------------- lib/system/gc2.nim | 61 ++++++++++++++++++---------------------- lib/system/gc_common.nim | 1 + lib/system/gc_ms.nim | 15 ++++------ 4 files changed, 47 insertions(+), 63 deletions(-) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index cd5c6870dc..66d49ce1bc 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -92,7 +92,7 @@ type additionalRoots: CellSeq # dummy roots for GC_ref/unref when hasThreadSupport: toDispose: SharedList[pointer] - isMainThread: bool + gcThreadId: int var gch {.rtlThreadVar.}: GcHeap @@ -159,12 +159,12 @@ when defined(logGC): if not c.typ.name.isNil: typName = c.typ.name - when leakDetector: - c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n", - msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line) - else: - c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; color=%ld\n", - msg, c, kind, typName, c.refcount shr rcShift, c.color) + when leakDetector: + c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n", + msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line) + else: + c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; thread=%ld\n", + msg, c, kind, typName, c.refcount shr rcShift, gch.gcThreadId) template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) @@ -312,7 +312,8 @@ proc initGC() = init(gch.additionalRoots) when hasThreadSupport: init(gch.toDispose) - gch.isMainThread = true + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc cellsetReset(s: var CellSet) = deinit(s) @@ -459,7 +460,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "rawNewObj end") @@ -506,7 +507,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) zeroMem(result, size) sysAssert(allocInv(gch.region), "newObjRC1 end") @@ -576,7 +577,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "growObj end") when defined(memProfiler): nimProfile(newsize-oldsize) @@ -621,7 +622,7 @@ proc markS(gch: var GcHeap, c: PCell) = forAllChildren(d, waMarkPrecise) proc markGlobals(gch: var GcHeap) = - if gch.isMainThread: + if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() let d = gch.additionalRoots.d @@ -669,13 +670,7 @@ proc doOperation(p: pointer, op: WalkOp) = of waPush: add(gch.tempStack, c) of waMarkGlobal: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - # XXX: This should not be required anymore! - if isAllocatedPtr(gch.region, c): - markS(gch, c) - else: - markS(gch, c) + markS(gch, c) of waMarkPrecise: add(gch.tempStack, c) #of waDebug: debugGraph(c) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index cd90c6d622..ca2a35f601 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -104,7 +104,7 @@ type pDumpHeapFile: pointer # File that is used for GC_dumpHeap when hasThreadSupport: toDispose: SharedList[pointer] - isMainThread: bool + gcThreadId: int var gch {.rtlThreadVar.}: GcHeap @@ -120,23 +120,6 @@ template release(gch: GcHeap) = when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) -proc initGC() = - when not defined(useNimRtl): - gch.red = (1-gch.black) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.completedCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.additionalRoots) - init(gch.greyStack) - when hasThreadSupport: - init(gch.toDispose) - gch.isMainThread = true - # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only # reachable from them. However, when we are sweeping, they have to @@ -342,6 +325,24 @@ proc gcInvariant*() = include gc_common +proc initGC() = + when not defined(useNimRtl): + gch.red = (1-gch.black) + gch.cycleThreshold = InitialCycleThreshold + gch.stat.stackScans = 0 + gch.stat.completedCollections = 0 + gch.stat.maxThreshold = 0 + gch.stat.maxStackSize = 0 + gch.stat.maxStackCells = 0 + gch.stat.cycleTableSize = 0 + # init the rt + init(gch.additionalRoots) + init(gch.greyStack) + when hasThreadSupport: + init(gch.toDispose) + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") + proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 sysAssert(allocInv(gch.region), "rawNewObj begin") @@ -480,7 +481,7 @@ proc GC_dumpHeap*(file: File) = c_fprintf(file, "onstack %p\n", d[i]) else: c_fprintf(file, "onstack_invalid %p\n", d[i]) - if gch.isMainThread: + if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() while true: @@ -569,7 +570,7 @@ proc markIncremental(gch: var GcHeap): bool = result = true proc markGlobals(gch: var GcHeap) = - if gch.isMainThread: + if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() @@ -591,22 +592,14 @@ proc doOperation(p: pointer, op: WalkOp) = markRoot(gch, c) else: dumpRoot(gch, c) - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): handleRoot() - else: - #gcAssert(isAllocatedPtr(gch.region, c), "doOperation: waMarkGlobal") - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") - handleRoot() + handleRoot() discard allocInv(gch.region) of waMarkGrey: - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") + when false: + if not isAllocatedPtr(gch.region, c): + c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) + #GC_dumpHeap() + sysAssert(false, "wtf") if c.color == 1-gch.black: c.setColor(rcGrey) add(gch.greyStack, c) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index f0abd918a2..ad37df0e0c 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -401,6 +401,7 @@ var globalMarkers: array[0.. 3499, GlobalMarkerProc] threadLocalMarkersLen: int threadLocalMarkers: array[0.. 3499, GlobalMarkerProc] + gHeapidGenerator: int proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = if globalMarkersLen <= high(globalMarkers): diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 0754f2cfee..101185dfbd 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -73,7 +73,7 @@ type stat: GcStat when hasThreadSupport: toDispose: SharedList[pointer] - isMainThread: bool + gcThreadId: int additionalRoots: CellSeq # dummy roots for GC_ref/unref var @@ -220,7 +220,8 @@ proc initGC() = init(gch.marked) when hasThreadSupport: init(gch.toDispose) - gch.isMainThread = true + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -394,13 +395,7 @@ proc doOperation(p: pointer, op: WalkOp) = var c: PCell = usrToCell(p) gcAssert(c != nil, "doOperation: 1") case op - of waMarkGlobal: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - mark(gch, c) - else: - mark(gch, c) + of waMarkGlobal: mark(gch, c) of waMarkPrecise: add(gch.tempStack, c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = @@ -437,7 +432,7 @@ when false: quit 1 proc markGlobals(gch: var GcHeap) = - if gch.isMainThread: + if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() let d = gch.additionalRoots.d From 1661062ebf147e55e29a0bfb8fc62dc7959b8614 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 11 Jan 2018 20:47:07 +0000 Subject: [PATCH 198/200] Raise assertion error when attempting to use closed socket. --- lib/pure/asyncnet.nim | 2 ++ lib/pure/net.nim | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 5be457d2a2..93399bb409 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -277,6 +277,7 @@ template readInto(buf: pointer, size: int, socket: AsyncSocket, flags: set[SocketFlag]): int = ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that ## this is a template and not a proc. + assert(not socket.closed, "Cannot `recv` on a closed socket") var res = 0 if socket.isSsl: when defineSsl: @@ -403,6 +404,7 @@ proc send*(socket: AsyncSocket, buf: pointer, size: int, ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all ## data has been sent. assert socket != nil + assert(not socket.closed, "Cannot `send` on a closed socket") if socket.isSsl: when defineSsl: sslLoop(socket, flags, diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 9215a18128..f348b7c51b 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -868,6 +868,7 @@ proc close*(socket: Socket) = socket.sslHandle = nil socket.fd.close() + socket.fd = osInvalidSocket when defined(posix): from posix import TCP_NODELAY @@ -1005,15 +1006,25 @@ proc select(readfd: Socket, timeout = 500): int = var fds = @[readfd.fd] result = select(fds, timeout) +proc isClosed(socket: Socket): bool = + socket.fd == osInvalidSocket + +proc uniRecv(socket: Socket, buffer: pointer, size, flags: cint): int = + ## Handles SSL and non-ssl recv in a nice package. + ## + ## In particular handles the case where socket has been closed properly + ## for both SSL and non-ssl. + result = 0 + assert(not socket.isClosed, "Cannot `recv` on a closed socket") + when defineSsl: + if socket.isSsl: + return SSLRead(socket.sslHandle, buffer, size) + + return recv(socket.fd, buffer, size, flags) + proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defineSsl: - if socket.isSSL: - result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + result = uniRecv(socket, addr(socket.buffer), socket.buffer.high, flags) if result < 0: # Save it in case it gets reset (the Nim codegen occasionally may call # Win API functions which reset it). @@ -1059,16 +1070,16 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] else: when defineSsl: if socket.isSSL: - if socket.sslHasPeekChar: + if socket.sslHasPeekChar: # TODO: Merge this peek char mess into uniRecv copyMem(data, addr(socket.sslPeekChar), 1) socket.sslHasPeekChar = false if size-1 > 0: var d = cast[cstring](data) - result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 + result = uniRecv(socket, addr(d[1]), cint(size-1), 0'i32) + 1 else: result = 1 else: - result = SSLRead(socket.sslHandle, data, size) + result = uniRecv(socket, data, size.cint, 0'i32) else: result = recv(socket.fd, data, size.cint, 0'i32) else: @@ -1186,7 +1197,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: - result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) + result = uniRecv(socket, addr(socket.sslPeekChar), 1, 0'i32) socket.sslHasPeekChar = true c = socket.sslPeekChar @@ -1320,6 +1331,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. + assert(not socket.isClosed, "Cannot `send` on a closed socket") when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -1364,6 +1376,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, ## which is defined below. ## ## **Note:** This proc is not available for SSL sockets. + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") var aiList = getAddrInfo(address, port, af) # try all possibilities: From b38f6d49b7a795a46b83bcafda25b41b41b1ba8f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 16 Jan 2018 11:06:22 +0100 Subject: [PATCH 199/200] travis: attempt to make tests green again --- tests/testament/categories.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 06decfa3c7..5c845fc5c4 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -148,7 +148,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcbench" test "gcleak" test "gcleak2" - test "gctest" + testWithoutBoehm "gctest" testWithNone "gctest" test "gcleak3" test "gcleak4" From 399c5e38b7bf7a0998ca9ed4bce57c7092d59229 Mon Sep 17 00:00:00 2001 From: jcosborn Date: Tue, 16 Jan 2018 18:32:40 -0600 Subject: [PATCH 200/200] don't make optNilCheck default to on for now (#7058) * don't make optNilCheck default to on for now * add conditional symbol nimHasNilChecks --- compiler/condsyms.nim | 1 + config/nim.cfg | 3 +++ 2 files changed, 4 insertions(+) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index a52214e734..0be2899be0 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -112,3 +112,4 @@ proc initDefines*() = defineSymbol("nimNewRoof") defineSymbol("nimHasRunnableExamples") defineSymbol("nimNewDot") + defineSymbol("nimHasNilChecks") diff --git a/config/nim.cfg b/config/nim.cfg index a146c4ebf4..9fddce90a5 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -59,6 +59,9 @@ path="$lib/pure" debugger:off line_dir:off dead_code_elim:on + @if nimHasNilChecks: + nilchecks:off + @end @end @if release: