From bcbe317d177e9a22856e8dd6c0ad1e20436b913a Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Sun, 30 Dec 2018 23:04:41 +0530 Subject: [PATCH 001/350] Before showing deprecated warning, check whether enum field was marked deprecated or the whole enum type (#10135) --- compiler/suggest.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index f149327ac9..d3ce461724 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -464,7 +464,7 @@ proc extractPragma(s: PSym): PNode = proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) = let pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s) let name = - if s.kind == skEnumField: "enum '" & s.owner.name.s & "' which contains field '" & s.name.s & "'" + if s.kind == skEnumField and sfDeprecated notin s.flags: "enum '" & s.owner.name.s & "' which contains field '" & s.name.s & "'" else: s.name.s if pragmaNode != nil: for it in pragmaNode: @@ -490,7 +490,7 @@ proc markUsed(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) = if s.kind == skEnumField and s.owner != nil: incl(s.owner.flags, sfUsed) if sfDeprecated in s.owner.flags: - incl(s.flags, sfDeprecated) + warnAboutDeprecated(conf, info, s) if {sfDeprecated, sfError} * s.flags != {}: if sfDeprecated in s.flags: warnAboutDeprecated(conf, info, s) if sfError in s.flags: userError(conf, info, s) From e87910197a209d02cf69bba3aacd9766723aeb37 Mon Sep 17 00:00:00 2001 From: deech Date: Mon, 31 Dec 2018 07:41:24 -0600 Subject: [PATCH 002/350] Check there are no side effects before optimizing away compile time expressions. (#9934) --- compiler/semexprs.nim | 20 +++++++------ compiler/semstmts.nim | 2 ++ tests/vm/tcompiletimesideeffects.nim | 42 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 tests/vm/tcompiletimesideeffects.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ddec457a1a..45e259b652 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -710,16 +710,20 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = let a = getConstExpr(c.module, n.sons[i], c.graph) if a == nil: return n call.add(a) + #echo "NOW evaluating at compile time: ", call.renderTree - if sfCompileTime in callee.flags: - result = evalStaticExpr(c.module, c.graph, call, c.p.owner) - if result.isNil: - localError(c.config, n.info, errCannotInterpretNodeX % renderTree(call)) - else: result = fixupTypeAfterEval(c, result, n) + if c.inStaticContext == 0 or sfNoSideEffect in callee.flags: + if sfCompileTime in callee.flags: + result = evalStaticExpr(c.module, c.graph, call, c.p.owner) + if result.isNil: + localError(c.config, n.info, errCannotInterpretNodeX % renderTree(call)) + else: result = fixupTypeAfterEval(c, result, n) + else: + result = evalConstExpr(c.module, c.graph, call) + if result.isNil: result = n + else: result = fixupTypeAfterEval(c, result, n) else: - result = evalConstExpr(c.module, c.graph, call) - if result.isNil: result = n - else: result = fixupTypeAfterEval(c, result, n) + result = n #if result != n: # echo "SUCCESS evaluated at compile time: ", call.renderTree diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 04991193cd..f60e2556d2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -548,6 +548,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) + inc c.inStaticContext for i in countup(0, sonsLen(n) - 1): var a = n.sons[i] if c.config.cmd == cmdIdeTools: suggestStmt(c, a) @@ -607,6 +608,7 @@ proc semConst(c: PContext, n: PNode): PNode = v.ast = def[j] b.sons[j] = newSymNode(v) addSon(result,b) + dec c.inStaticContext include semfields diff --git a/tests/vm/tcompiletimesideeffects.nim b/tests/vm/tcompiletimesideeffects.nim new file mode 100644 index 0000000000..4cd57b3bd6 --- /dev/null +++ b/tests/vm/tcompiletimesideeffects.nim @@ -0,0 +1,42 @@ +discard """ + output: +''' +@[0, 1, 2] +@[3, 4, 5] +@[0, 1, 2] +3 +4 +''' +""" + +template runNTimes(n: int, f : untyped) : untyped = + var accum: seq[type(f)] + for i in 0..n-1: + accum.add(f) + accum + +var state {.compileTime.} : int = 0 +proc fill(): int {.compileTime.} = + result = state + inc state + +# invoke fill() at compile time as a compile time expression +const C1 = runNTimes(3, fill()) +echo C1 + +# invoke fill() at compile time as a set of compile time statements +const C2 = + block: + runNTimes(3, fill()) +echo C2 + +# invoke fill() at compile time after a compile time reset of state +const C3 = + block: + state = 0 + runNTimes(3, fill()) +echo C3 + +# evaluate fill() at compile time and use the results at runtime +echo fill() +echo fill() From 7ac1fc81fd119dba3739e05a89ad63fcee405ac1 Mon Sep 17 00:00:00 2001 From: c-blake Date: Mon, 31 Dec 2018 08:52:51 -0500 Subject: [PATCH 003/350] Resolve things raised in https://github.com/nim-lang/Nim/issues/10081 ? (#10084) * Resolve things raised in https://github.com/nim-lang/Nim/issues/10081 ? CDF is a standard ident in all things related to random numbers/sampling, and full words "cumulativeDistributionFunction" would be silly long, in this case, IMO. We use lowercase `cdf` to make it not look like a type, remove all looping from `sample` letting callers do it. Besides just side-stepping any `sampleSize` name choice, callers may want to filter out samples anyway which this makes slightly simpler. Also add two variants of `cumsum`, value return and in-place update distinguished by the var-ness of the first argument. Add tests for `int` and `float` for both `cumsum` and the new `sample`. (The sample tests exercise the value return mode of `cumsum`.) Functionality pre-this-PR `sample(a, w)` is now the almost as simple `for i in 0.. `cumsummed` to honor NEP1 style. Re-instate `cumsum` as the in-place transformation. Test both in `tests/stdlib/tmath.nim` and use `cumsummed` in the example code for sample since that's a simpler example. * Fix requests from https://github.com/nim-lang/Nim/pull/10084 : example in lib/pure/math.nim and comment whitespace in lib/pure/random.nim --- lib/pure/math.nim | 18 +++++++++++++ lib/pure/random.nim | 39 ++++++++++++++--------------- tests/stdlib/tmath.nim | 57 +++++++++++++++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index ee32772b1b..72abae2653 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -163,6 +163,24 @@ proc prod*[T](x: openArray[T]): T {.noSideEffect.} = result = 1.T for i in items(x): result = result * i +proc cumsummed*[T](x: openArray[T]): seq[T] = + ## Return cumulative aka prefix summation of ``x``. + ## + ## .. code-block:: nim + ## var x = [1, 2, 3, 4] + ## echo x.cumsummed # [1, 3, 6, 10] + result.setLen(x.len) + result[0] = x[0] + for i in 1 ..< x.len: result[i] = result[i-1] + x[i] + +proc cumsum*[T](x: var openArray[T]) = + ## Transforms ``x`` in-place into its cumulative aka prefix summation. + ## + ## .. code-block:: nim + ## var x = [1, 2, 3, 4] + ## x.cumsum; echo x # [1, 3, 6, 10] + for i in 1 ..< x.len: x[i] = x[i-1] + x[i] + {.push noSideEffect.} when not defined(JS): # C proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "".} diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 26e6740ea6..ee728ad4aa 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -175,27 +175,26 @@ proc sample*[T](a: openArray[T]): T = ## returns a random element from openArray ``a`` using non-thread-safe state. result = a[rand(a.low..a.high)] -proc sample*[T, U](r: var Rand; a: openArray[T], w: openArray[U], n=1): seq[T] = - ## Return a sample (with replacement) of size ``n`` from elements of ``a`` - ## according to convertible-to-``float``, not necessarily normalized, and - ## non-negative weights ``w``. Uses state in ``r``. Must have sum ``w > 0.0``. - assert(w.len == a.len) - var cdf = newSeq[float](a.len) # The *unnormalized* CDF - var tot = 0.0 # Unnormalized is fine if we sample up to tot - for i, w in w: - assert(w >= 0) - tot += float(w) - cdf[i] = tot - assert(tot > 0.0) # Need at least one non-zero weight - for i in 0 ..< n: - result.add(a[cdf.upperBound(r.rand(tot))]) - -proc sample*[T, U](a: openArray[T], w: openArray[U], n=1): seq[T] = - ## Return a sample (with replacement) of size ``n`` from elements of ``a`` - ## according to convertible-to-``float``, not necessarily normalized, and - ## non-negative weights ``w``. Uses default non-thread-safe state. - state.sample(a, w, n) +proc sample*[T, U](r: var Rand; a: openArray[T], cdf: openArray[U]): T = + ## Sample one element from openArray ``a`` when it has cumulative distribution + ## function (CDF) ``cdf`` (not necessarily normalized, any type of elements + ## convertible to ``float``). Uses state in ``r``. E.g.: + ## + ## .. code-block:: nim + ## let val = [ "a", "b", "c", "d" ] # some values + ## var cnt = [1, 2, 3, 4] # histogram of counts + ## echo r.sample(val, cnt.cumsummed) # echo a sample + assert(cdf.len == a.len) # Two basic sanity checks. + assert(float(cdf[^1]) > 0.0) + #While we could check cdf[i-1] <= cdf[i] for i in 1..cdf.len, that could get + #awfully expensive even in debugging modes. + let u = r.rand(float(cdf[^1])) + a[cdf.upperBound(U(u))] +proc sample*[T, U](a: openArray[T], cdf: openArray[U]): T = + ## Like ``sample(var Rand; openArray[T], openArray[U])``, but uses default + ## non-thread-safe state. + state.sample(a, cdf) proc initRand*(seed: int64): Rand = ## Creates a new ``Rand`` state from ``seed``. diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 7c1851e7a4..544fa8dc09 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -4,6 +4,8 @@ discard """ [Suite] random float +[Suite] cumsum + [Suite] random sample [Suite] ^ @@ -74,29 +76,66 @@ suite "random float": var rand2:float = random(1000000.0) check rand1 != rand2 +suite "cumsum": + test "cumsum int seq return": + let counts = [ 1, 2, 3, 4 ] + check counts.cumsummed == [ 1, 3, 6, 10 ] + + test "cumsum float seq return": + let counts = [ 1.0, 2.0, 3.0, 4.0 ] + check counts.cumsummed == [ 1.0, 3.0, 6.0, 10.0 ] + + test "cumsum int in-place": + var counts = [ 1, 2, 3, 4 ] + counts.cumsum + check counts == [ 1, 3, 6, 10 ] + + test "cumsum float in-place": + var counts = [ 1.0, 2.0, 3.0, 4.0 ] + counts.cumsum + check counts == [ 1.0, 3.0, 6.0, 10.0 ] + suite "random sample": - test "non-uniform array sample": + test "non-uniform array sample unnormalized int CDF": let values = [ 10, 20, 30, 40, 50 ] # values - let weight = [ 4, 3, 2, 1, 0 ] # weights aka unnormalized probabilities - let weightSum = 10.0 # sum of weights + let counts = [ 4, 3, 2, 1, 0 ] # weights aka unnormalized probabilities var histo = initCountTable[int]() - for v in sample(values, weight, 5000): - histo.inc(v) - check histo.len == 4 # number of non-zero in `weight` + let cdf = counts.cumsummed # unnormalized CDF + for i in 0 ..< 5000: + histo.inc(sample(values, cdf)) + check histo.len == 4 # number of non-zero in `counts` # Any one bin is a binomial random var for n samples, each with prob p of # adding a count to k; E[k]=p*n, Var k=p*(1-p)*n, approximately Normal for # big n. So, P(abs(k - p*n)/sqrt(p*(1-p)*n))>3.0) =~ 0.0027, while # P(wholeTestFails) =~ 1 - P(binPasses)^4 =~ 1 - (1-0.0027)^4 =~ 0.01. - for i, w in weight: - if w == 0: + for i, c in counts: + if c == 0: check values[i] notin histo continue - let p = float(w) / float(weightSum) + let p = float(c) / float(cdf[^1]) let n = 5000.0 let expected = p * n let stdDev = sqrt(n * p * (1.0 - p)) check abs(float(histo[values[i]]) - expected) <= 3.0 * stdDev + test "non-uniform array sample normalized float CDF": + let values = [ 10, 20, 30, 40, 50 ] # values + let counts = [ 0.4, 0.3, 0.2, 0.1, 0 ] # probabilities + var histo = initCountTable[int]() + let cdf = counts.cumsummed # normalized CDF + for i in 0 ..< 5000: + histo.inc(sample(values, cdf)) + check histo.len == 4 # number of non-zero in ``counts`` + for i, c in counts: + if c == 0: + check values[i] notin histo + continue + let p = float(c) / float(cdf[^1]) + let n = 5000.0 + let expected = p * n + let stdDev = sqrt(n * p * (1.0 - p)) + # NOTE: like unnormalized int CDF test, P(wholeTestFails) =~ 0.01. + check abs(float(histo[values[i]]) - expected) <= 3.0 * stdDev suite "^": test "compiles for valid types": From ab72d68ec80300ebd0629c5174a78c73f28fc729 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 31 Dec 2018 06:42:01 -0800 Subject: [PATCH 004/350] fix off by 1 error in `col` shown by toFileLineCol (#10138) * fix off by 1 error in `col` shown by toFileLineCol * fix test failures --- compiler/msgs.nim | 2 +- tests/discard/tneedsdiscard.nim | 2 +- tests/method/tmapper.nim | 2 +- tests/modules/tmismatchedvisibility.nim | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 7e6b67cbec..18c196085a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -208,7 +208,7 @@ proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} = result = toFilename(conf, info) & ":" & $info.line proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} = - result = toFilename(conf, info) & "(" & $info.line & ", " & $info.col & ")" + result = toFilename(conf, info) & "(" & $info.line & ", " & $(info.col+1) & ")" proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info) diff --git a/tests/discard/tneedsdiscard.nim b/tests/discard/tneedsdiscard.nim index 7d2997b3ff..d9483947f4 100644 --- a/tests/discard/tneedsdiscard.nim +++ b/tests/discard/tneedsdiscard.nim @@ -1,5 +1,5 @@ discard """ - errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded; start of expression here: tneedsdiscard.nim(7, 2)''' + errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded; start of expression here: tneedsdiscard.nim(7, 3)''' line: 10 """ diff --git a/tests/method/tmapper.nim b/tests/method/tmapper.nim index a5d03f7003..9162d0eec6 100644 --- a/tests/method/tmapper.nim +++ b/tests/method/tmapper.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tmapper.nim(22, 7)" + errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tmapper.nim(22, 8)" line: 25 """ diff --git a/tests/modules/tmismatchedvisibility.nim b/tests/modules/tmismatchedvisibility.nim index 4bf244807e..fd582b571e 100644 --- a/tests/modules/tmismatchedvisibility.nim +++ b/tests/modules/tmismatchedvisibility.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "public implementation 'tmismatchedvisibility.foo(a: int)[declared in tmismatchedvisibility.nim(6, 5)]' has non-public forward declaration in " + errormsg: "public implementation 'tmismatchedvisibility.foo(a: int)[declared in tmismatchedvisibility.nim(6, 6)]' has non-public forward declaration in " line: 8 """ From 7c90e22ddd354692d259c5620578f194757c20e1 Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 31 Dec 2018 21:57:09 +0000 Subject: [PATCH 005/350] fixes #10148 (#10149) * fixes #10148 * fix a typo --- compiler/cgen.nim | 31 ++++++++++++++++++------------- tests/cpp/t10148.nim | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 tests/cpp/t10148.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 3d76be2540..e4f16f4ed3 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1103,9 +1103,8 @@ proc genMainProc(m: BModule) = "}$N$N" & "void PreMain(void) {$N" & "\tvoid (*volatile inner)(void);$N" & - "$1" & "\tinner = PreMainInner;$N" & - "$4$5" & + "$1" & "\t(*inner)();$N" & "}$N$N" @@ -1222,12 +1221,7 @@ proc genMainProc(m: BModule) = else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N") inc(m.labels) appcg(m, m.s[cfsProcs], PreMainBody, [ - m.g.mainDatInit, m.g.breakpoints, m.g.otherModsInit, - if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone: - ropecg(m, "\t#initThreadVarsEmulation();$N") - else: - "".rope, - initStackBottomCall]) + m.g.mainDatInit, m.g.breakpoints, m.g.otherModsInit]) appcg(m, m.s[cfsProcs], nimMain, [m.g.mainModInit, initStackBottomCall, rope(m.labels)]) @@ -1256,21 +1250,32 @@ proc getInitName(m: PSym): Rope = proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") + proc registerModuleToMain(g: BModuleList; m: BModule) = + if m.s[cfsDatInitProc].len > 0: + let datInit = m.module.getDatInitName + addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) + addf(g.mainDatInit, "\t$1();$N", [datInit]) + + # Initialization of TLS and GC should be done in between + # systemDatInit and systemInit calls if any + if sfSystemModule in m.module.flags: + if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone: + add(g.mainDatInit, ropecg(m, "\t#initThreadVarsEmulation();$N")) + if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: + add(g.mainDatInit, ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")) + if m.s[cfsInitProc].len > 0: let init = m.module.getInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.module.flags: add(g.mainModInit, initCall) + elif sfSystemModule in m.module.flags: + add(g.mainDatInit, initCall) # systemInit must called right after systemDatInit if any else: add(g.otherModsInit, initCall) - if m.s[cfsDatInitProc].len > 0: - let datInit = m.module.getDatInitName - addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) - addf(g.mainDatInit, "\t$1();$N", [datInit]) - proc genDatInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate diff --git a/tests/cpp/t10148.nim b/tests/cpp/t10148.nim new file mode 100644 index 0000000000..e8dd3098fe --- /dev/null +++ b/tests/cpp/t10148.nim @@ -0,0 +1,29 @@ +discard """ + output: '''Expected successful exit''' + joinable: false +""" + +import os + +proc another_proc: string = + ## trigger many GC allocations + var x = @[""] + for i in 0..100: + x.add $i + result = "not_existent_path" + +proc findlib2: string = + let path = getEnv("MYLIB2_DOES_NOT_EXIST_PATH") + let another_path = another_proc() + GC_fullCollect() + + if path.len > 0 and dirExists(path): + path / "alib_does_not_matter.dll" + elif fileExists(another_path): + another_path + else: + quit("Expected successful exit", 0) + +proc imported_func*(a: cint): cstring {.importc, dynlib: findlib2().} + +echo imported_func(0) From e7fa8f3443ebe03d74fb20b7f4d0096756211463 Mon Sep 17 00:00:00 2001 From: jiro Date: Tue, 1 Jan 2019 17:56:59 +0900 Subject: [PATCH 006/350] Add Example code to Documentation comment of wordWrap proc. (#10146) --- lib/std/wordwrap.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim index c7898b3398..4b0dc4417f 100644 --- a/lib/std/wordwrap.nim +++ b/lib/std/wordwrap.nim @@ -24,6 +24,11 @@ proc wrapWords*(s: string, maxLineWidth = 80, seps: set[char] = Whitespace, newLine = "\n"): string {.noSideEffect.} = ## Word wraps `s`. + runnableExamples: + doAssert "12345678901234567890".wrapWords() == "12345678901234567890" + doAssert "123456789012345678901234567890".wrapWords(20) == "12345678901234567890\n1234567890" + doAssert "Hello Bob. Hello John.".wrapWords(13, false) == "Hello Bob.\nHello John." + doAssert "Hello Bob. Hello John.".wrapWords(13, true, {';'}) == "Hello Bob. He\nllo John." result = newStringOfCap(s.len + s.len shr 6) var spaceLeft = maxLineWidth var lastSep = "" From 9faad7591e37fa074544206d1de17e59f56bd576 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Tue, 1 Jan 2019 18:20:48 +0530 Subject: [PATCH 007/350] Deprecate gc v2 (#10151) * Deprecate gc v2 * warnDeprecated now has custom messages --- compiler/commands.nim | 4 ++-- compiler/importer.nim | 4 ++-- compiler/lineinfos.nim | 2 +- compiler/lookups.nim | 2 +- compiler/modulepaths.nim | 2 +- compiler/parser.nim | 4 ++-- compiler/pragmas.nim | 12 ++++++------ compiler/semexprs.nim | 2 +- compiler/semstmts.nim | 2 +- compiler/semtypes.nim | 2 +- compiler/suggest.nim | 4 ++-- doc/advopt.txt | 2 +- 12 files changed, 21 insertions(+), 21 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 5893791cc8..8b73884e8f 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -224,7 +224,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo case arg.normalize of "boehm": result = conf.selectedGC == gcBoehm of "refc": result = conf.selectedGC == gcRefc - of "v2": result = conf.selectedGC == gcV2 + of "v2": result = false of "markandsweep": result = conf.selectedGC == gcMarkAndSweep of "generational": result = false of "destructors": result = conf.selectedGC == gcDestructors @@ -442,7 +442,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "refc": conf.selectedGC = gcRefc of "v2": - conf.selectedGC = gcV2 + message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc") of "markandsweep": conf.selectedGC = gcMarkAndSweep defineSymbol(conf.symbols, "gcmarkandsweep") diff --git a/compiler/importer.nim b/compiler/importer.nim index 118d26d806..eef0c9bb98 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -162,9 +162,9 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = localError(c.config, n.info, "A module cannot import itself") if sfDeprecated in result.flags: if result.constraint != nil: - message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s) + message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated") else: - message(c.config, n.info, warnDeprecated, result.name.s) + message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated") suggestSym(c.config, n.info, result, c.graph.usageSym, false) importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index b1ecf779ee..21ce44406e 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -65,7 +65,7 @@ const warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored", warnXIsNeverRead: "'$1' is never read", warnXmightNotBeenInit: "'$1' might not have been initialized", - warnDeprecated: "$1 is deprecated", + warnDeprecated: "$1", warnConfigDeprecated: "config file '$1' is deprecated", warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)", warnUnknownMagic: "unknown magic '$1' might crash the compiler", diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 11a7415058..d4959db12d 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -89,7 +89,7 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym = prettybase.replaceDeprecated(conf, n.info, s, result) else: message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " & - s.name.s) + s.name.s & " is deprecated") proc localSearchInScope*(c: PContext, s: PIdent): PSym = result = strTableGet(c.currentScope.symbols, s) diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim index f0718c4ebb..9e27a2d7d5 100644 --- a/compiler/modulepaths.nim +++ b/compiler/modulepaths.nim @@ -147,7 +147,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string = # hacky way to implement 'x / y /../ z': result = renderTree(n, {renderNoComments}).replace(" ") of nkDotExpr: - localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths") + localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated") result = renderTree(n, {renderNoComments}).replace(".", "/") of nkImportAs: result = getModuleName(conf, n.sons[0]) diff --git a/compiler/parser.nim b/compiler/parser.nim index 163ad64556..afc49c0d37 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1916,7 +1916,7 @@ proc parseObject(p: var TParser): PNode = getTok(p) if p.tok.tokType == tkCurlyDotLe and p.validInd: # Deprecated since v0.20.0 - parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas") + parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas is deprecated") addSon(result, parsePragma(p)) else: addSon(result, p.emptyNode) @@ -2008,7 +2008,7 @@ proc parseTypeDef(p: var TParser): PNode = if p.tok.tokType == tkBracketLe and p.validInd: if not noPragmaYet: # Deprecated since v0.20.0 - parMessage(p, warnDeprecated, "pragma before generic parameter list") + parMessage(p, warnDeprecated, "pragma before generic parameter list is deprecated") genericParam = parseGenericParamList(p) else: genericParam = p.emptyNode diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 58f64f7b03..39b58d0b18 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -242,7 +242,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) = # deprecated as of 0.18.1 message(c.config, n.info, warnDeprecated, "use {.experimental: \"codeReordering.\".} instead; " & - (if flag == sfNoForward: "{.noForward.}" else: "{.reorder.}")) + (if flag == sfNoForward: "{.noForward.}" else: "{.reorder.}") & " is deprecated") proc processCallConv(c: PContext, n: PNode) = if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent: @@ -447,14 +447,14 @@ proc processPop(c: PContext, n: PNode) = proc processDefine(c: PContext, n: PNode) = if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): defineSymbol(c.config.symbols, n[1].ident.s) - message(c.config, n.info, warnDeprecated, "define") + message(c.config, n.info, warnDeprecated, "define is deprecated") else: invalidPragma(c, n) proc processUndef(c: PContext, n: PNode) = if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): undefSymbol(c.config.symbols, n[1].ident.s) - message(c.config, n.info, warnDeprecated, "undef") + message(c.config, n.info, warnDeprecated, "undef is deprecated") else: invalidPragma(c, n) @@ -784,7 +784,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, if sym.kind in {skTemplate, skMacro}: incl(sym.flags, sfImmediate) incl(sym.flags, sfAllUntyped) - message(c.config, n.info, warnDeprecated, "use 'untyped' parameters instead; immediate") + message(c.config, n.info, warnDeprecated, "use 'untyped' parameters instead; immediate is deprecated") else: invalidPragma(c, it) of wDirty: if sym.kind == skTemplate: incl(sym.flags, sfDirty) @@ -1098,10 +1098,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wThis: if it.kind in nkPragmaCallKinds and it.len == 2: c.selfName = considerQuotedIdent(c, it[1]) - message(c.config, n.info, warnDeprecated, "the '.this' pragma") + message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") elif it.kind == nkIdent or it.len == 1: c.selfName = getIdent(c.cache, "self") - message(c.config, n.info, warnDeprecated, "the '.this' pragma") + message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") else: localError(c.config, it.info, "'this' pragma is allowed to have zero or one arguments") of wNoRewrite: diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 45e259b652..19d008557f 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2434,7 +2434,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result.kind = nkCall result = semExpr(c, result, flags) of nkBind: - message(c.config, n.info, warnDeprecated, "bind") + message(c.config, n.info, warnDeprecated, "bind is deprecated") result = semExpr(c, n.sons[0], flags) of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy: if c.matchedConcept != nil and n.len == 1: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f60e2556d2..59eb231628 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1582,7 +1582,7 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) = foundObj = true x.methods.add((col,s)) if not foundObj: - message(c.config, n.info, warnDeprecated, "generic method not attachable to object type") + message(c.config, n.info, warnDeprecated, "generic method not attachable to object type is deprecated") else: # why check for the body? bug #2400 has none. Checking for sfForward makes # no sense either. diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 200b247ca0..d1ea54eca7 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -210,7 +210,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = tyError, tyObject}: message c.config, n[i].info, errGenerated, "region needs to be an object type" else: - message(c.config, n.info, warnDeprecated, "region for pointer types") + message(c.config, n.info, warnDeprecated, "region for pointer types is deprecated") addSonSkipIntLit(result, region) addSonSkipIntLit(result, t) if tfPartial in result.flags: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index d3ce461724..144b86224f 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -470,9 +470,9 @@ proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) = for it in pragmaNode: if whichPragma(it) == wDeprecated and it.safeLen == 2 and it[1].kind in {nkStrLit..nkTripleStrLit}: - message(conf, info, warnDeprecated, it[1].strVal & "; " & name) + message(conf, info, warnDeprecated, it[1].strVal & "; " & name & " is deprecated") return - message(conf, info, warnDeprecated, name) + message(conf, info, warnDeprecated, name & " is deprecated") proc userError(conf: ConfigRef; info: TLineInfo; s: PSym) = let pragmaNode = extractPragma(s) diff --git a/doc/advopt.txt b/doc/advopt.txt index 7445068c1e..75260baad3 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -82,7 +82,7 @@ Advanced options: --skipUserCfg do not read the user's configuration file --skipParentCfg do not read the parent dirs' configuration files --skipProjCfg do not read the project's configuration file - --gc:refc|v2|markAndSweep|boehm|go|none|regions + --gc:refc|markAndSweep|boehm|go|none|regions select the GC to use; default is 'refc' --index:on|off turn index file generation on|off --putenv:key=value set an environment variable From 6ed9676af5ae17fa15551a00f1ad5f3be109233f Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Tue, 1 Jan 2019 23:19:44 +0530 Subject: [PATCH 008/350] Show field not initialized warning only when `notnil` used (#10155) --- compiler/semtypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index d1ea54eca7..f4936a71aa 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -141,7 +141,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil): wrongRedefinition(c, e.info, e.name.s, conflict.info) inc(counter) - if not hasNull: incl(result.flags, tfNeedsInit) + if tfNotNil in e.typ.flags and not hasNull: incl(result.flags, tfNeedsInit) proc semSet(c: PContext, n: PNode, prev: PType): PType = result = newOrPrevType(tySet, prev, c) From 31b8bc7866e8b024daeb0be153144e3398933559 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Tue, 1 Jan 2019 19:49:29 +0100 Subject: [PATCH 009/350] Add link to tutorial part III to docs.rst (#10157) [ci skip] --- doc/docs.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/docs.rst b/doc/docs.rst index cd1a058537..4a69bd69aa 100644 --- a/doc/docs.rst +++ b/doc/docs.rst @@ -6,6 +6,9 @@ The documentation consists of several documents: - | `Tutorial (part II) `_ | The Nim tutorial part two deals with the advanced language constructs. +- | `Tutorial (part III) `_ + | The Nim tutorial part three about Nim's macro system. + - | `Language Manual `_ | The Nim manual is a draft that will evolve into a proper specification. From a1e268e3dccdde4df9b11a0ee87971e1143fbb43 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 2 Jan 2019 01:26:40 -0800 Subject: [PATCH 010/350] [cleanup] remove dead code compiler.options.mergeConfigs (#10165) --- compiler/options.nim | 78 -------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 49d2f74049..448741ddb7 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -253,84 +253,6 @@ template depConfigFields*(fn) {.dirty.} = fn(globalOptions) fn(selectedGC) -proc mergeConfigs*(dest, src: ConfigRef; mergeSymbols: bool) = - template merge[T: enum](a, b: T) = - a = b - template merge[T](a, b: set[T]) = - a = a + b - template merge(a, b: int) = - inc a, b - template merge[T](a, b: seq[T]) = - for bb in b: a.add b - template merge(a, b: string) = - a = b - template merge[T: AbsoluteFile|AbsoluteDir](a, b: T) = - if a.isEmpty and not b.isEmpty: a = b - - template merge[T](a, b: HashSet[T]) = - for bb in b: a.incl b - template merge(a, b: StringTableRef) = - for k, v in b: a[k] = v - template merge[T: object](a, b: T) = - a = b - - template m(field) = - merge(dest.field, src.field) - - m target - m options - m globalOptions - m cmd - m selectedGC - dest.verbosity = src.verbosity - m numberOfProcessors - m evalExpr - m symbolFiles - m cppDefines - m headerFile - m features - m arguments - m ideCmd - m cCompiler - m enableNotes - m disableNotes - m foreignPackageNotes - m notes - m errorCounter - m hintCounter - m warnCounter - m errorMax - m configVars - if mergeSymbols: - m symbols - m projectName - m projectPath - m projectFull - m searchPaths - m lazyPaths - m outFile - m prefixDir - m libpath - m nimcacheDir - m dllOverrides - m moduleOverrides - m command - m commandArgs - m implicitImports - m implicitIncludes - m docSeeSrcUrl - m cIncludes - m cLibs - m cLinkedLibs - m externalToLink - m linkOptionsCmd - m compileOptionsCmd - m linkOptions - m compileOptions - m ccompilerpath - m toCompile - m cppCustomNamespace - const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel} const From 9fb8c3d965ebf760c0636838eec1217ee7be7caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Wed, 2 Jan 2019 17:39:53 +0100 Subject: [PATCH 011/350] Add {.noReturn.} to system.raiseAssert (#10161) --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index b9f86f5493..cab8a91aa7 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3927,7 +3927,7 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ template currentSourcePath*: string = instantiationInfo(-1, true).filename ## returns the full file-system path of the current source -proc raiseAssert*(msg: string) {.noinline.} = +proc raiseAssert*(msg: string) {.noinline, noReturn.} = sysFatal(AssertionError, msg) proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = From 7c5ae008874e7bb76214bca96267bcb400a723d1 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Thu, 3 Jan 2019 00:31:06 +0530 Subject: [PATCH 012/350] exportc is now not allowed for type aliases (#9979) --- compiler/semstmts.nim | 2 ++ tests/pragmas/t5149.nim | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/pragmas/t5149.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 59eb231628..39f200ba86 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1101,6 +1101,8 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = #debug s.typ s.ast = a popOwner(c) + if sfExportc in s.flags and s.typ.kind == tyAlias: + localError(c.config, name.info, "{.exportc.} not allowed for type aliases") let aa = a.sons[2] if aa.kind in {nkRefTy, nkPtrTy} and aa.len == 1 and aa.sons[0].kind == nkObjectTy: diff --git a/tests/pragmas/t5149.nim b/tests/pragmas/t5149.nim new file mode 100644 index 0000000000..2d242a8d5f --- /dev/null +++ b/tests/pragmas/t5149.nim @@ -0,0 +1,12 @@ +discard """ + errormsg: "{.exportc.} not allowed for type aliases" + line: 9 +""" + +type + X* = object + a: int + Y* {.exportc.} = X + +proc impl*(x: X) = + echo "it works" From e6cbf9e79014e2fb6851796adda251eed1abcfd5 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 3 Jan 2019 23:45:42 +0100 Subject: [PATCH 013/350] koch.nim: speed up tests for the 'testinstall' target for the nightly builds (these tests are also covered by the other CIs) --- koch.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index 9596ce21fa..3082dba8f2 100644 --- a/koch.nim +++ b/koch.nim @@ -475,8 +475,7 @@ proc testUnixInstall(cmdLineRest: string) = execCleanPath("./koch --latest tools") # check the tests work: putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH") - execCleanPath("./koch tests", destDir / "bin") - #execCleanPath("./koch tests cat newconfig", destDir / "bin") + execCleanPath("./koch tests cat megatest", destDir / "bin") else: echo "Version check: failure" finally: From 77166ba795b03d4536f8efb0f225ba3ffc8531be Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 4 Jan 2019 09:25:58 +0100 Subject: [PATCH 014/350] [backport] correctly document `toInt`, fixes #2764 [ci skip] (#10176) --- lib/system.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index cab8a91aa7..9fa889b117 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1799,14 +1799,16 @@ proc toBiggestFloat*(i: BiggestInt): BiggestFloat {. proc toInt*(f: float): int {. magic: "ToInt", noSideEffect, importc: "toInt".} ## converts a floating point number `f` into an ``int``. Conversion - ## rounds `f` if it does not contain an integer value. If the conversion - ## fails (because `f` is infinite for example), `ValueError` is raised. + ## rounds `f` if it does not contain an integer value. + ## Note that some floating point numbers (e.g. infinity) cannot be + ## accurately converted. proc toBiggestInt*(f: BiggestFloat): BiggestInt {. magic: "ToBiggestInt", noSideEffect, importc: "toBiggestInt".} ## converts a biggestfloat `f` into a ``biggestint``. Conversion - ## rounds `f` if it does not contain an integer value. If the conversion - ## fails (because `f` is infinite for example), `ValueError` is raised. + ## rounds `f` if it does not contain an integer value. + ## Note that some floating point numbers (e.g. infinity) cannot be + ## accurately converted. proc addQuitProc*(quitProc: proc() {.noconv.}) {. importc: "atexit", header: "".} From 086229459aa72f06d31a4f920653a82a74cfc6b0 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 4 Jan 2019 09:52:01 +0100 Subject: [PATCH 015/350] runnableExamples: compile and test them as a single file; fixes #9300 --- compiler/docgen.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 67f4108e17..0fe19c5bc5 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -40,6 +40,7 @@ type # already. See bug #3655 destFile*: AbsoluteFile thisDir*: AbsoluteDir + examples: string PDoc* = ref TDocumentor ## Alias to type less. @@ -378,6 +379,13 @@ proc testExample(d: PDoc; ex: PNode) = "_examples" & $d.exampleCounter & ".nim")) #let nimcache = outp.changeFileExt"" & "_nimcache" renderModule(ex, d.filename, outp.string, conf = d.conf) + d.examples.add "import r\"" & outp.string & "\"\n" + +proc runAllExamples(d: PDoc) = + let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples" + let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" & + "_examples.nim")) + writeFile(outp, d.examples) let backend = if isDefined(d.conf, "js"): "js" elif isDefined(d.conf, "cpp"): "cpp" elif isDefined(d.conf, "objc"): "objc" @@ -400,11 +408,9 @@ proc extractImports(n: PNode; result: PNode) = for i in 0.. Date: Fri, 4 Jan 2019 15:13:29 +0530 Subject: [PATCH 016/350] Undocument --genmapping (#10175) --- doc/advopt.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 75260baad3..5ddd064ef6 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -55,8 +55,6 @@ Advanced options: --clibdir:DIR modify the linker library search path --clib:LIBNAME link an additional C library (you should omit platform-specific extensions) - --genMapping generate a mapping file containing - (Nim, mangled) identifier pairs --project document the whole project (doc2) --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) From 69149a0e925d641c35e677a72b356e7f546e7e99 Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 4 Jan 2019 13:20:12 +0100 Subject: [PATCH 017/350] [backport] improve unicode docs, fixes #2353 (#10174) * as instructed in #2353, provides a short description why there are no specialized procs for seq[Rune] * adds several examples to better explain what some functions do * small fixes (double backticks, add missing dots, etc.) * use `rune` instead of "unicode characer" --- lib/pure/unicode.nim | 207 +++++++++++++++++++++++++------------------ 1 file changed, 122 insertions(+), 85 deletions(-) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 712cc46c80..27eabecc6d 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -8,6 +8,11 @@ # ## This module provides support to handle the Unicode UTF-8 encoding. +## +## There are no specialized ``insert``, ``delete``, ``add`` and ``contains`` +## procedures for ``seq[Rune]`` in this module because the generic variants +## of these procedures in the system module already work with it. + {.deadCodeElim: on.} # dce option deprecated @@ -15,7 +20,7 @@ include "system/inclrtl" type RuneImpl = int32 # underlying type of Rune - Rune* = distinct RuneImpl ## type that can hold any Unicode character + Rune* = distinct RuneImpl ## Unicode code point. Can hold any Unicode character. Rune16* = distinct int16 ## 16 bit Unicode character proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b) @@ -25,7 +30,12 @@ proc `==`*(a, b: Rune): bool = return int(a) == int(b) template ones(n: untyped): untyped = ((1 shl n)-1) proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = - ## Returns the number of Unicode characters of the string ``s`` + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + var i = 0 while i < len(s): if ord(s[i]) <=% 127: inc(i) @@ -38,7 +48,12 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = inc(result) proc runeLenAt*(s: string, i: Natural): int = - ## Returns the number of bytes the rune starting at ``s[i]`` takes + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + if ord(s[i]) <=% 127: result = 1 elif ord(s[i]) shr 5 == 0b110: result = 2 elif ord(s[i]) shr 4 == 0b1110: result = 3 @@ -50,7 +65,7 @@ proc runeLenAt*(s: string, i: Natural): int = const replRune = Rune(0xFFFD) template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = - ## Returns the Unicode character ``s[i]`` in ``result``. If ``doInc == true`` + ## Returns the rune ``s[i]`` in ``result``. If ``doInc == true`` ## ``i`` is incremented by the number of bytes that have been processed. bind ones if ord(s[i]) <=% 127: @@ -152,17 +167,21 @@ proc validateUtf8*(s: string): int = return -1 proc runeAt*(s: string, i: Natural): Rune = - ## Returns the unicode character in ``s`` at byte index ``i`` + ## Returns the rune in ``s`` at **byte index** ``i``. + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) fastRuneAt(s, i, result, false) template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = - ## Copies UTF-8 representation of `c` into the preallocated string `s` - ## starting at position `pos`. If `doInc == true`, `pos` is incremented + ## Copies UTF-8 representation of ``c`` into the preallocated string ``s`` + ## starting at position ``pos``. If ``doInc == true``, ``pos`` is incremented ## by the number of bytes that have been processed. ## - ## To be the most efficient, make sure `s` is preallocated - ## with an additional amount equal to the byte length of - ## `c`. + ## To be the most efficient, make sure ``s`` is preallocated + ## with an additional amount equal to the byte length of ``c``. var i = RuneImpl(c) if i <=% 127: s.setLen(pos+1) @@ -207,28 +226,39 @@ template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = discard # error, exception? proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = - ## Converts a rune into its UTF-8 representation + ## Converts a rune into its UTF-8 representation. + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1).toUTF8 == "ñ" + result = "" fastToUTF8Copy(c, result, 0, false) proc add*(s: var string; c: Rune) = + ## Adds a rune ``c`` to a string ``s``. + runnableExamples: + var s = "abc" + let c = "ä".runeAt(0) + s.add(c) + doAssert s == "abcä" + let pos = s.len fastToUTF8Copy(c, s, pos, false) proc `$`*(rune: Rune): string = - ## Converts a Rune to a string + ## An alias for `toUTF8 <#toUTF8%2CRune>`_. rune.toUTF8 proc `$`*(runes: seq[Rune]): string = - ## Converts a sequence of Runes to a string + ## Converts a sequence of Runes to a string. result = "" for rune in runes: result.add rune proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int = - ## Returns the byte position of unicode character - ## at position pos in s with an optional start byte position. - ## returns the special value -1 if it runs out of the string + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. ## ## Beware: This can lead to unoptimized code and slow execution! ## Most problems can be solved more efficiently by using an iterator @@ -244,7 +274,7 @@ proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int = return o proc runeAtPos*(s: string, pos: int): Rune = - ## Returns the unicode character at position pos + ## Returns the rune at position ``pos``. ## ## Beware: This can lead to unoptimized code and slow execution! ## Most problems can be solved more efficiently by using an iterator @@ -252,7 +282,7 @@ proc runeAtPos*(s: string, pos: int): Rune = fastRuneAt(s, runeOffset(s, pos), result, false) proc runeStrAtPos*(s: string, pos: Natural): string = - ## Returns the unicode character at position pos as UTF8 String + ## Returns the rune at position ``pos`` as UTF8 String. ## ## Beware: This can lead to unoptimized code and slow execution! ## Most problems can be solved more efficiently by using an iterator @@ -262,7 +292,7 @@ proc runeStrAtPos*(s: string, pos: Natural): string = proc runeReverseOffset*(s: string, rev:Positive): (int, int) = ## Returns a tuple with the the byte offset of the - ## unicode character at position ``rev`` in s counting + ## rune at position ``rev`` in ``s``, counting ## from the end (starting with 1) and the total ## number of runes in the string. Returns a negative value ## for offset if there are to few runes in the string to @@ -286,13 +316,21 @@ proc runeReverseOffset*(s: string, rev:Positive): (int, int) = return (-a, rev.int-a) return (x, -a+rev.int) -proc runeSubStr*(s: string, pos:int, len:int = int.high): string = - ## Returns the UTF-8 substring starting at codepoint pos - ## with len codepoints. If pos or len is negative they count from - ## the end of the string. If len is not given it means the longest +proc runeSubStr*(s: string, pos: int, len: int = int.high): string = + ## Returns the UTF-8 substring starting at codepoint ``pos`` + ## with ``len`` codepoints. If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest ## possible string. ## - ## (Needs some examples) + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: @@ -1321,7 +1359,7 @@ proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = return -1 proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` into lower case. This works for any Unicode character. + ## Converts ``c`` into lower case. This works for any rune. ## If possible, prefer ``toLower`` over ``toUpper``. var c = RuneImpl(c) var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) @@ -1333,7 +1371,7 @@ proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c) proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` into upper case. This works for any Unicode character. + ## Converts ``c`` into upper case. This works for any rune. ## If possible, prefer ``toLower`` over ``toUpper``. var c = RuneImpl(c) var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) @@ -1345,7 +1383,7 @@ proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c) proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` to title case + ## Converts ``c`` to title case. var c = RuneImpl(c) var p = binarySearch(c, toTitleSinglets, len(toTitleSinglets) div 2, 2) if p >= 0 and c == toTitleSinglets[p]: @@ -1353,7 +1391,7 @@ proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c) proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a lower case Unicode character. + ## Returns true iff ``c`` is a lower case rune. ## If possible, prefer ``isLower`` over ``isUpper``. var c = RuneImpl(c) # Note: toUpperRanges is correct here! @@ -1365,7 +1403,7 @@ proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = return true proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a upper case Unicode character. + ## Returns true iff ``c`` is a upper case rune. ## If possible, prefer ``isLower`` over ``isUpper``. var c = RuneImpl(c) # Note: toLowerRanges is correct here! @@ -1377,7 +1415,7 @@ proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = return true proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is an *alpha* Unicode character (i.e., a letter) + ## Returns true iff ``c`` is an *alpha* rune (i.e., a letter) if isUpper(c) or isLower(c): return true var c = RuneImpl(c) @@ -1389,18 +1427,18 @@ proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = return true proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode titlecase character + ## Returns true iff ``c`` is a Unicode titlecase character. return isUpper(c) and isLower(c) proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode whitespace character + ## Returns true iff ``c`` is a Unicode whitespace character. var c = RuneImpl(c) var p = binarySearch(c, spaceRanges, len(spaceRanges) div 2, 2) if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]: return true proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode combining character + ## Returns true iff ``c`` is a Unicode combining character. var c = RuneImpl(c) # Optimized to return false immediately for ASCII @@ -1424,12 +1462,12 @@ template runeCheck(s, runeProc) = proc isAlpha*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nuc$1Str".} = - ## Returns true iff `s` contains all alphabetic unicode characters. + ## Returns true iff ``s`` contains all alphabetic runes. runeCheck(s, isAlpha) proc isSpace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nuc$1Str".} = - ## Returns true iff `s` contains all whitespace unicode characters. + ## Returns true iff ``s`` contains all whitespace runes. runeCheck(s, isWhiteSpace) template runeCaseCheck(s, runeProc, skipNonAlpha) = @@ -1459,7 +1497,7 @@ proc isLower*(s: string, skipNonAlpha: bool): bool {. ## Checks whether ``s`` is lower case. ## ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## runes in ``s`` are lower case. Returns false if none of the + ## runes in ``s`` are lower case. Returns false if none of the ## runes in ``s`` are alphabetical. ## ## If ``skipNonAlpha`` is false, returns true only if all runes in @@ -1474,7 +1512,7 @@ proc isUpper*(s: string, skipNonAlpha: bool): bool {. ## Checks whether ``s`` is upper case. ## ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## runes in ``s`` are upper case. Returns false if none of the + ## runes in ``s`` are upper case. Returns false if none of the ## runes in ``s`` are alphabetical. ## ## If ``skipNonAlpha`` is false, returns true only if all runes in @@ -1485,7 +1523,7 @@ proc isUpper*(s: string, skipNonAlpha: bool): bool {. runeCaseCheck(s, isUpper, skipNonAlpha) template convertRune(s, runeProc) = - ## Convert runes in `s` using `runeProc` as the converter. + ## Convert runes in ``s`` using ``runeProc`` as the converter. result = newString(len(s)) var @@ -1502,20 +1540,20 @@ template convertRune(s, runeProc) = proc toUpper*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1Str".} = - ## Converts `s` into upper-case unicode characters. + ## Converts ``s`` into upper-case runes. convertRune(s, toUpper) proc toLower*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1Str".} = - ## Converts `s` into lower-case unicode characters. + ## Converts ``s`` into lower-case runes. convertRune(s, toLower) proc swapCase*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1".} = - ## Swaps the case of unicode characters in `s` + ## Swaps the case of runes in ``s``. ## - ## Returns a new string such that the cases of all unicode characters - ## are swapped if possible + ## Returns a new string such that the cases of all runes + ## are swapped if possible. var i = 0 @@ -1538,7 +1576,7 @@ proc swapCase*(s: string): string {.noSideEffect, procvar, proc capitalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1".} = - ## Converts the first character of `s` into an upper-case unicode character. + ## Converts the first character of ``s`` into an upper-case rune. if len(s) == 0: return s @@ -1552,10 +1590,10 @@ proc capitalize*(s: string): string {.noSideEffect, procvar, proc translate*(s: string, replacements: proc(key: string): string): string {. rtl, extern: "nuc$1".} = - ## Translates words in a string using the `replacements` proc to substitute - ## words inside `s` with their replacements + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. ## - ## `replacements` is any proc that takes a word and returns + ## ``replacements`` is any proc that takes a word and returns ## a new word to fill it's place. # Allocate memory for the new string based on the old one. @@ -1601,10 +1639,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. proc title*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1".} = - ## Converts `s` to a unicode title. + ## Converts ``s`` to a unicode title. ## ## Returns a new string such that the first character - ## in each word inside `s` is capitalized + ## in each word inside ``s`` is capitalized. var i = 0 @@ -1631,10 +1669,10 @@ proc title*(s: string): string {.noSideEffect, procvar, proc isTitle*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nuc$1Str", deprecated: "Deprecated since version 0.20 since its semantics are unclear".}= - ## Checks whether or not `s` is a unicode title. + ## Checks whether or not ``s`` is a unicode title. ## - ## Returns true if the first character in each word inside `s` - ## are upper case and there is at least one character in `s`. + ## Returns true if the first character in each word inside ``s`` + ## are upper case and there is at least one character in ``s``. if s.len == 0: return false @@ -1656,7 +1694,7 @@ proc isTitle*(s: string): bool {.noSideEffect, procvar, firstRune = true iterator runes*(s: string): Rune = - ## Iterates over any unicode character of the string ``s`` returning runes + ## Iterates over any rune of the string ``s`` returning runes. var i = 0 result: Rune @@ -1665,7 +1703,7 @@ iterator runes*(s: string): Rune = yield result iterator utf8*(s: string): string = - ## Iterates over any unicode character of the string ``s`` returning utf8 values + ## Iterates over any rune of the string ``s`` returning utf8 values. var o = 0 while o < s.len: let n = runeLenAt(s, o) @@ -1673,7 +1711,7 @@ iterator utf8*(s: string): string = o += n proc toRunes*(s: string): seq[Rune] = - ## Obtains a sequence containing the Runes in ``s`` + ## Obtains a sequence containing the Runes in ``s``. result = newSeq[Rune]() for r in s.runes: result.add(r) @@ -1696,15 +1734,14 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = result = a.len - b.len proc reversed*(s: string): string = - ## Returns the reverse of ``s``, interpreting it as Unicode characters. - ## Unicode combining characters are correctly interpreted as well: - ## - ## .. code-block:: nim - ## - ## assert reversed("Reverse this!") == "!siht esreveR" - ## assert reversed("先秦兩漢") == "漢兩秦先" - ## assert reversed("as⃝df̅") == "f̅ds⃝a" - ## assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + ## Returns the reverse of ``s``, interpreting it as runes. + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + var i = 0 lastI = 0 @@ -1731,7 +1768,7 @@ proc reversed*(s: string): string = reverseUntil(len(s)) proc graphemeLen*(s: string; i: Natural): Natural = - ## The number of bytes belonging to 's[i]' including following combining + ## The number of bytes belonging to ``s[i]`` including following combining ## characters. var j = i.int var r, r2: Rune @@ -1744,7 +1781,7 @@ proc graphemeLen*(s: string; i: Natural): Natural = result = j-i proc lastRune*(s: string; last: int): (Rune, int) = - ## length of the last rune in 's[0..last]'. Returns the rune and its length + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): result = (Rune(s[last]), 1) @@ -1778,7 +1815,7 @@ proc stringHasSep(s: string, index: int, sep: Rune): bool = return sep == rune template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) = - ## Common code for split procedures + ## Common code for split procedures. var last = 0 splits = maxsplit @@ -1801,9 +1838,9 @@ template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) = iterator split*(s: string, seps: openarray[Rune] = unicodeSpaces, maxsplit: int = -1): string = - ## Splits the unicode string `s` into substrings using a group of separators. + ## Splits the unicode string ``s`` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. + ## Substrings are separated by a substring containing only ``seps``. ## ## .. code-block:: nim ## for word in split("this\lis an\texample"): @@ -1844,7 +1881,7 @@ iterator split*(s: string, seps: openarray[Rune] = unicodeSpaces, splitCommon(s, seps, maxsplit) iterator splitWhitespace*(s: string): string = - ## Splits a unicode string at whitespace runes + ## Splits a unicode string at whitespace runes. splitCommon(s, unicodeSpaces, -1) template accResult(iter: untyped) = @@ -1858,9 +1895,9 @@ proc splitWhitespace*(s: string): seq[string] {.noSideEffect, accResult(splitWhitespace(s)) iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = - ## Splits the unicode string `s` into substrings using a single separator. + ## Splits the unicode string ``s`` into substrings using a single separator. ## - ## Substrings are separated by the rune `sep`. + ## Substrings are separated by the rune ``sep``. ## The code: ## ## .. code-block:: nim @@ -1898,11 +1935,11 @@ proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffec proc strip*(s: string, leading = true, trailing = true, runes: openarray[Rune] = unicodeSpaces): string {.noSideEffect, rtl, extern: "nucStrip".} = - ## Strips leading or trailing `runes` from `s` and returns + ## Strips leading or trailing ``runes`` from ``s`` and returns ## the resulting string. ## - ## If `leading` is true, leading `runes` are stripped. - ## If `trailing` is true, trailing `runes` are stripped. + ## If ``leading`` is true, leading ``runes`` are stripped. + ## If ``trailing`` is true, trailing ``runes`` are stripped. ## If both are false, the string is returned unchanged. var s_i = 0 ## starting index into string ``s`` @@ -1948,9 +1985,9 @@ proc strip*(s: string, leading = true, trailing = true, proc repeat*(c: Rune, count: Natural): string {.noSideEffect, rtl, extern: "nucRepeatRune".} = - ## Returns a string of `count` Runes `c`. + ## Returns a string of ``count`` Runes ``c``. ## - ## The returned string will have a rune-length of `count`. + ## The returned string will have a rune-length of ``count``. let s = $c result = newStringOfCap(count * s.len) for i in 0 ..< count: @@ -1958,11 +1995,11 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect, proc align*(s: string, count: Natural, padding = ' '.Rune): string {. noSideEffect, rtl, extern: "nucAlignString".} = - ## Aligns a unicode string `s` with `padding`, so that it has a rune-length - ## of `count`. + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. ## - ## `padding` characters (by default spaces) are added before `s` resulting in - ## right alignment. If ``s.runelen >= count``, no spaces are added and `s` is + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is ## returned unchanged. If you need to left align a string use the `alignLeft ## proc <#alignLeft>`_. runnableExamples: @@ -1985,11 +2022,11 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {. proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. noSideEffect.} = - ## Left-Aligns a unicode string `s` with `padding`, so that it has a - ## rune-length of `count`. + ## Left-Aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. ## - ## `padding` characters (by default spaces) are added after `s` resulting in - ## left alignment. If ``s.runelen >= count``, no spaces are added and `s` is + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is ## returned unchanged. If you need to right align a string use the `align ## proc <#align>`_. runnableExamples: From 5101b6befd55a7e32694a09a23dc818cb63dcbb6 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 4 Jan 2019 13:39:47 +0100 Subject: [PATCH 018/350] fixes #10188 --- compiler/docgen.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 0fe19c5bc5..1af9c06b8f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -382,6 +382,7 @@ proc testExample(d: PDoc; ex: PNode) = d.examples.add "import r\"" & outp.string & "\"\n" proc runAllExamples(d: PDoc) = + if d.examples.len == 0: return let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples" let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" & "_examples.nim")) From 319b46230cd96fe22a7d95b931e90ccf09783e61 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 4 Jan 2019 04:54:02 -0800 Subject: [PATCH 019/350] fix bug in doAssertRaises when exception==Exception (#10172) * fix bug in doAssertRaises when exception==Exception * add testcase for doAssertRaises --- lib/system.nim | 30 +++++++++++++++++++----------- tests/system/tsystem_misc.nim | 12 ++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 9fa889b117..d41953108d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -4353,7 +4353,7 @@ else: template runnableExamples*(body: untyped) = discard -template doAssertRaises*(exception, code: untyped): typed = +template doAssertRaises*(exception: typedesc, code: untyped): typed = ## Raises ``AssertionError`` if specified ``code`` does not raise the ## specified exception. Example: ## @@ -4361,16 +4361,24 @@ template doAssertRaises*(exception, code: untyped): typed = ## doAssertRaises(ValueError): ## raise newException(ValueError, "Hello World") var wrong = false - try: - if true: - code - wrong = true - except exception: - discard - except Exception as exc: - raiseAssert(astToStr(exception) & - " wasn't raised, another error was raised instead by:\n"& - astToStr(code)) + when Exception is exception: + try: + if true: + code + wrong = true + except Exception: + discard + else: + try: + if true: + code + wrong = true + except exception: + discard + except Exception as exc: + raiseAssert(astToStr(exception) & + " wasn't raised, another error was raised instead by:\n"& + astToStr(code)) if wrong: raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index 98bc3f4a32..f53a86e9a1 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -112,6 +112,18 @@ doAssertRaises(IndexError): foo(toOpenArray(arrNeg, -1, 0)) doAssertRaises(IndexError): foo(toOpenArray(arrNeg, -1, -3)) +doAssertRaises(Exception): + raise newException(Exception, "foo") + +block: + var didThrow = false + try: + doAssertRaises(IndexError): # should fail since it's wrong exception + raise newException(FieldError, "foo") + except AssertionError: + # ok, throwing was correct behavior + didThrow = true + doAssert didThrow type seqqType = ptr UncheckedArray[int] let qData = cast[seqqType](addr seqq[0]) From 4eee92a7a3741a2783b49dd74c11551cffc5518a Mon Sep 17 00:00:00 2001 From: Tristano Ajmone Date: Fri, 4 Jan 2019 14:37:07 +0100 Subject: [PATCH 020/350] Fix Typo in Compiler Guide (#10189) Change 'ableit' to 'albeit'. --- doc/nimc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index e1bf98ece4..6f6d38d665 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -353,7 +353,7 @@ Define Effect ``useFork`` Makes ``osproc`` use ``fork`` instead of ``posix_spawn``. ``useNimRtl`` Compile and link against ``nimrtl.dll``. ``useMalloc`` Makes Nim use C's `malloc`:idx: instead of Nim's - own memory manager, ableit prefixing each allocation with + own memory manager, albeit prefixing each allocation with its size to support clearing memory on reallocation. This only works with ``gc:none``. ``useRealtimeGC`` Enables support of Nim's GC for *soft* realtime From ba7d33b4e46f9eafb4980f7a2b4865c5914e0fcc Mon Sep 17 00:00:00 2001 From: rec <44084068+recloser@users.noreply.github.com> Date: Fri, 4 Jan 2019 15:05:03 +0100 Subject: [PATCH 021/350] Guard against null exception (#10162) --- lib/system/jssys.nim | 2 +- tests/js/test2.nim | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 8be19e5b84..d7718e4f4e 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -46,7 +46,7 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result[0] = x proc isNimException(): bool {.asmNoStackFrame.} = - asm "return `lastJSError`.m_type;" + asm "return `lastJSError` && `lastJSError`.m_type;" proc getCurrentException*(): ref Exception {.compilerRtl, benign.} = if isNimException(): result = cast[ref Exception](lastJSError) diff --git a/tests/js/test2.nim b/tests/js/test2.nim index 0bfb991397..9ecdbb35cf 100644 --- a/tests/js/test2.nim +++ b/tests/js/test2.nim @@ -9,6 +9,9 @@ js 3.14 # This file tests the JavaScript generator +doAssert getCurrentException() == nil +doAssert getCurrentExceptionMsg() == "" + # #335 proc foo() = var bar = "foo" From e9a192c36fb62331db0bbd308433e56fc771c352 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Fri, 4 Jan 2019 19:41:48 +0530 Subject: [PATCH 022/350] expandFilename on windows is now consistent with other platforms (#10154) --- compiler/options.nim | 11 +----- lib/pure/os.nim | 90 +++++++++++++++++++++++--------------------- 2 files changed, 49 insertions(+), 52 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 448741ddb7..3730d16f81 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -482,16 +482,7 @@ proc setDefaultLibpath*(conf: ConfigRef) = conf.libpath = AbsoluteDir parentNimLibPath proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile = - # on Windows, 'expandFilename' calls getFullPathName which doesn't do - # case corrections, so we have to use this convoluted way of retrieving - # the true filename (see tests/modules and Nimble uses 'import Uri' instead - # of 'import uri'): - when defined(windows): - result = AbsoluteFile path.string.expandFilename - for x in walkFiles(result.string): - return AbsoluteFile x - else: - result = AbsoluteFile path.string.expandFilename + result = AbsoluteFile path.string.expandFilename proc shortenDir*(conf: ConfigRef; dir: string): string {. deprecated: "use 'relativeTo' instead".} = diff --git a/lib/pure/os.nim b/lib/pure/os.nim index b6cef191b7..ab87101c2b 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -945,48 +945,6 @@ when not weirdTarget: raise newException(ValueError, "The specified root is not absolute: " & root) joinPath(root, path) -proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimScript.} = - ## Returns the full (`absolute`:idx:) path of an existing file `filename`, - ## raises OSError in case of an error. Follows symlinks. - when defined(windows): - var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break - else: - # according to Posix we don't need to allocate space for result pathname. - # But we need to free return value with free(3). - var r = realpath(filename, nil) - if r.isNil: - raiseOSError(osLastError()) - else: - result = $r - c_free(cast[pointer](r)) - proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [], noNimScript.} = ## Normalize a path. ## @@ -1425,6 +1383,54 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noNimScript ## notation is supported. walkCommon(pattern, isDir) +proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimScript.} = + ## Returns the full (`absolute`:idx:) path of an existing file `filename`, + ## raises OSError in case of an error. Follows symlinks. + when defined(windows): + var bufsize = MAX_PATH.int32 + when useWinUnicode: + var unused: WideCString = nil + var res = newWideCString("", bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break + else: + var unused: cstring = nil + result = newString(bufsize) + while true: + var L = getFullPathNameA(filename, bufsize, result, unused) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + result = newString(L) + bufsize = L + else: + setLen(result, L) + break + # getFullPathName doesn't do case corrections, so we have to use this convoluted + # way of retrieving the true filename + for x in walkFiles(result.string): + result = x + if not existsFile(result) and not existsDir(result): + raise newException(OSError, "file does not exist") + else: + # according to Posix we don't need to allocate space for result pathname. + # But we need to free return value with free(3). + var r = realpath(filename, nil) + if r.isNil: + raiseOSError(osLastError()) + else: + result = $r + c_free(cast[pointer](r)) + type PathComponent* = enum ## Enumeration specifying a path component. pcFile, ## path refers to a file From be0a4d1342b05815e25f9a51bda0cf199a6d439e Mon Sep 17 00:00:00 2001 From: alaviss Date: Fri, 4 Jan 2019 21:13:07 +0700 Subject: [PATCH 023/350] fix system.nim documentations (#10168) * system: fix nimGC_getStackBottom doc * system/helpers: avoid leaking docs to system --- lib/system.nim | 4 ++-- lib/system/helpers.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index d41953108d..f6987aa62d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2769,8 +2769,8 @@ when not defined(nimscript) and hasAlloc: when not defined(JS) and not defined(nimscript) and hasAlloc: proc nimGC_setStackBottom*(theStackBottom: pointer) {.compilerRtl, noinline, benign.} - ## Expands operating GC stack range to `theStackBottom`. Does nothing - ## if current stack bottom is already lower than `theStackBottom`. + ## Expands operating GC stack range to `theStackBottom`. Does nothing + ## if current stack bottom is already lower than `theStackBottom`. else: template GC_disable* = diff --git a/lib/system/helpers.nim b/lib/system/helpers.nim index fb1218684c..7b2b326790 100644 --- a/lib/system/helpers.nim +++ b/lib/system/helpers.nim @@ -1,5 +1,5 @@ -## helpers used system.nim and other modules, avoids code duplication while -## also minimizing symbols exposed in system.nim +# helpers used system.nim and other modules, avoids code duplication while +# also minimizing symbols exposed in system.nim # # TODO: move other things here that should not be exposed in system.nim From e5ca57d85c9187d4268c843b564b5123dc2e9960 Mon Sep 17 00:00:00 2001 From: Hugo Locurcio Date: Sat, 5 Jan 2019 01:00:38 +0100 Subject: [PATCH 024/350] Fix the tuple returned by `os.splitFile()` in documentation (#10197) --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ab87101c2b..96833056a0 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -336,7 +336,7 @@ proc searchExtPos*(path: string): int = proc splitFile*(path: string): tuple[dir, name, ext: string] {. noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into (dir, filename, extension). + ## Splits a filename into (dir, name, extension). ## `dir` does not end in `DirSep`. ## `extension` includes the leading dot. ## From ffea3fbfaed7aaf7c645bf254696ee98c1c840d9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jan 2019 07:01:10 -0800 Subject: [PATCH 025/350] * move up runnableExamples definition so can be used more in system.nim (#10196) * document that toInt, toBiggestInt round towards 0 and add runnableExamples * minor doc fixes --- lib/system.nim | 58 ++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index f6987aa62d..1d24019401 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -118,6 +118,22 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## # Do here programmer friendly expensive sanity checks. ## # Put here the normal code +when defined(nimHasRunnableExamples): + proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a ``runnableExamples`` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the ``##`` doc comment. As the last step of documentation + ## generation the examples are put into an ``$file_example.nim`` file, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. +else: + template runnableExamples*(body: untyped) = + discard + proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## Special compile-time procedure that checks whether `x` is ## declared. `x` has to be an identifier or a qualified identifier. @@ -1792,23 +1808,26 @@ proc toFloat*(i: int): float {. proc toBiggestFloat*(i: BiggestInt): BiggestFloat {. magic: "ToBiggestFloat", noSideEffect, importc: "toBiggestFloat".} - ## converts an biggestint `i` into a ``biggestfloat``. If the conversion + ## converts a biggestint `i` into a ``biggestfloat``. If the conversion ## fails, `ValueError` is raised. However, on most platforms the ## conversion cannot fail. proc toInt*(f: float): int {. - magic: "ToInt", noSideEffect, importc: "toInt".} + magic: "ToInt", noSideEffect, importc: "toInt".} = ## converts a floating point number `f` into an ``int``. Conversion - ## rounds `f` if it does not contain an integer value. - ## Note that some floating point numbers (e.g. infinity) cannot be - ## accurately converted. + ## rounds `f` half away from 0, see https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero + ## Note that some floating point numbers (e.g. infinity or even 1e19) + ## cannot be accurately converted. + runnableExamples: + doAssert toInt(0.49) == 0 + doAssert toInt(0.5) == 1 + doAssert toInt(-0.5) == -1 ## rounding is symmetrical proc toBiggestInt*(f: BiggestFloat): BiggestInt {. - magic: "ToBiggestInt", noSideEffect, importc: "toBiggestInt".} - ## converts a biggestfloat `f` into a ``biggestint``. Conversion - ## rounds `f` if it does not contain an integer value. - ## Note that some floating point numbers (e.g. infinity) cannot be - ## accurately converted. + magic: "ToBiggestInt", noSideEffect, importc: "toBiggestInt".} = + ## Same as `toInt` but for BiggestFloat to ``BiggestInt``. + runnableExamples: + doAssert toBiggestInt(0.49) == 0 proc addQuitProc*(quitProc: proc() {.noconv.}) {. importc: "atexit", header: "".} @@ -3084,7 +3103,7 @@ when not defined(JS): #and not defined(nimscript): proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my # Linux 64bit system. -- That's because the stack direction is the other - # way round. + # way around. when declared(nimGC_setStackBottom): var locals {.volatile.}: pointer locals = addr(locals) @@ -4336,23 +4355,6 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage): importc: "SetConsoleOutputCP".} discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage - -when defined(nimHasRunnableExamples): - proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} - ## A section you should use to mark `runnable example`:idx: code with. - ## - ## - In normal debug and release builds code within - ## a ``runnableExamples`` section is ignored. - ## - The documentation generator is aware of these examples and considers them - ## part of the ``##`` doc comment. As the last step of documentation - ## generation the examples are put into an ``$file_example.nim`` file, - ## compiled and tested. The collected examples are - ## put into their own module to ensure the examples do not refer to - ## non-exported symbols. -else: - template runnableExamples*(body: untyped) = - discard - template doAssertRaises*(exception: typedesc, code: untyped): typed = ## Raises ``AssertionError`` if specified ``code`` does not raise the ## specified exception. Example: From 4a720394bba39ce1e67d518b909cbb1c25f63d09 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jan 2019 07:07:09 -0800 Subject: [PATCH 026/350] testament megatest: we can now tell which test failed; helps debugging and prevents certain bugs, plus other fixes (#10089) * [testament] --verboseMegatest flag to make megatest compilation verbose * replace "tests" by testsDir * megatest's nimcache is now in same dir as other tests to avoid clobbering (eg when running tests from multiple Nim repos) --- .gitignore | 4 +++ testament/categories.nim | 57 ++++++++++++++++++++++++++++------------ testament/tester.nim | 9 ++++--- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index cbfb98a1be..132cd7f60c 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,7 @@ test.txt tweeter.db tweeter_test.db megatest.nim + +/outputExpected.txt +/outputGotten.txt + diff --git a/testament/categories.nim b/testament/categories.nim index 55b9e5ed27..4ff50af971 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -97,7 +97,7 @@ proc compileRodFiles(r: var TResults, cat: Category, options: string) = proc flagTests(r: var TResults, cat: Category, options: string) = # --genscript - const filename = "tests"/"flags"/"tgenscript" + const filename = testsDir/"flags"/"tgenscript" const genopts = " --genscript" let nimcache = nimcacheDir(filename, genopts, targetC) testSpec r, makeTest(filename, genopts, cat) @@ -356,7 +356,7 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) = ] for i, test in tests: - let filename = "tests" / test.addFileExt("nim") + let filename = testsDir / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed." # Run the tests. @@ -425,15 +425,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = let contents = readFile(testFile).string var testObj = makeTest(testFile, options, cat) #[ - TODO: + todo: this logic is fragile: false positives (if appears in a comment), or false negatives, eg `when defined(osx) and isMainModule`. - Instead of fixing this, a much better way, is to extend - https://github.com/nim-lang/Nim/issues/9581 to stdlib modules as follows: - * add these to megatest - * patch compiler so `isMainModule` is true when -d:isMainModuleIsAlwaysTrue - That'll give speedup benefit, and we don't have to patch stdlib files. + Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 + for a much better way. ]# if "when isMainModule" notin contents: testObj.spec.action = actionCompile @@ -534,7 +531,7 @@ proc `&.?`(a, b: string): string = result = if b.startswith(a): b else: a & b proc processSingleTest(r: var TResults, cat: Category, options, test: string) = - let test = "tests" & DirSep &.? cat.string / test + let test = testsDir &.? cat.string / test let target = if cat.string.normalize == "js": targetJS else: targetC if existsFile(test): testSpec r, makeTest(test, options, cat), {target} @@ -559,6 +556,7 @@ proc isJoinableSpec(spec: TSpec): bool = (spec.targets == {} or spec.targets == {targetC}) proc norm(s: var string) = + # equivalent of s/\n+/\n/g (could use a single pass over input if needed) while true: let tmp = s.replace("\n\n", "\n") if tmp == s: break @@ -566,9 +564,13 @@ proc norm(s: var string) = s = s.strip proc isTestFile*(file: string): bool = - let (dir, name, ext) = splitFile(file) + let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") +proc quoted(a: string): string = + # todo: consider moving to system.nim + result.addQuoted(a) + proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = ## returs a list of tests that have problems var specs: seq[TSpec] = @[] @@ -582,6 +584,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = if isJoinableSpec(spec): specs.add spec + proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp=cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: @@ -591,19 +595,36 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = return var megatest: string - for runSpec in specs: - megatest.add "import r\"" - megatest.add runSpec.file - megatest.add "\"\n" + #[ + TODO(minor): + get from Nim cmd + put outputGotten.txt, outputGotten.txt, megatest.nim there too + delete upon completion, maybe + ]# + var outDir = nimcacheDir(testsDir / "megatest", "", targetC) + const marker = "megatest:processing: " + + for i, runSpec in specs: + let file = runSpec.file + let file2 = outDir / ("megatest_" & $i & ".nim") + # `include` didn't work with `trecmod2.nim`, so using `import` + let code = "echo \"" & marker & "\", " & quoted(file) & "\n" + createDir(file2.parentDir) + writeFile(file2, code) + megatest.add "import " & quoted(file2) & "\n" + megatest.add "import " & quoted(file) & "\n" writeFile("megatest.nim", megatest) - const args = ["c", "-d:testing", "--listCmd", "megatest.nim"] - var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "") + let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "megatest.nim"] + proc onStdout(line: string) = echo line + var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "", + onStdout = if verboseMegatest: onStdout else: nil) if exitCode != 0: echo buf quit("megatest compilation failed") + # Could also use onStdout here. (buf, exitCode) = execCmdEx("./megatest") if exitCode != 0: echo buf @@ -613,6 +634,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile("outputGotten.txt", buf) var outputExpected = "" for i, runSpec in specs: + outputExpected.add marker & runSpec.file & "\n" outputExpected.add runSpec.output.strip outputExpected.add '\n' norm outputExpected @@ -621,6 +643,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile("outputExpected.txt", outputExpected) discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") echo "output different!" + # outputGotten.txt, outputExpected.txt not removed on purpose for debugging. quit 1 else: echo "output OK" @@ -686,7 +709,7 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string, var testsRun = 0 var files: seq[string] - for file in walkDirRec("tests" & DirSep &.? cat.string): + for file in walkDirRec(testsDir &.? cat.string): if isTestFile(file): files.add file files.sort # give reproducible order diff --git a/testament/tester.nim b/testament/tester.nim index 9e5fe58307..1867871c55 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -17,8 +17,10 @@ import var useColors = true var backendLogging = true var simulate = false +var verboseMegatest = false # very verbose but can be useful const + testsDir = "tests" & DirSep resultsFile = "testresults.html" #jsonFile = "testresults.json" # not used Usage = """Usage: @@ -34,6 +36,7 @@ Arguments: arguments are passed to the compiler Options: --print also print results to the console + --verboseMegatest log to stdout megatetest compilation --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests --targets:"c c++ js objc" run tests for specified targets (default: all) @@ -83,7 +86,7 @@ proc getFileDir(filename: string): string = if not result.isAbsolute(): result = getCurrentDir() / result -proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string): tuple[ +proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string, onStdout: proc(line: string) = nil): tuple[ output: TaintedString, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = @@ -103,6 +106,7 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp if outp.readLine(line): result[0].string.add(line.string) result[0].string.add("\n") + if onStdout != nil: onStdout(line.string) else: result[1] = peekExitCode(p) if result[1] != -1: break @@ -517,8 +521,6 @@ else: include categories -const testsDir = "tests" & DirSep - proc main() = os.putenv "NIMTEST_COLOR", "never" os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" @@ -533,6 +535,7 @@ proc main() = while p.kind == cmdLongoption: case p.key.string.normalize of "print", "verbose": optPrintResults = true + of "verbosemegatest": verboseMegatest = true of "failing": optFailing = true of "pedantic": discard "now always enabled" of "targets": From f3bd3691e796f4be5d522b4d0d1c64e30f985e68 Mon Sep 17 00:00:00 2001 From: Tristano Ajmone Date: Sun, 6 Jan 2019 10:45:24 +0100 Subject: [PATCH 027/350] Minor Fixes to Manual Wording (#10214) --- doc/manual.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 4a1ad9d5e6..f79811002d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1587,7 +1587,7 @@ details like this when mixing garbage collected data with unmanaged memory. Not nil annotation ------------------ -All types for that ``nil`` is a valid value can be annotated to +All types for which ``nil`` is a valid value can be annotated to exclude ``nil`` as a valid value with the ``not nil`` annotation: .. code-block:: nim @@ -1664,12 +1664,12 @@ Nim supports these `calling conventions`:idx:\: and another one for the pointer to implicitly passed environment. `stdcall`:idx: - This the stdcall convention as specified by Microsoft. The generated C + This is the stdcall convention as specified by Microsoft. The generated C procedure is declared with the ``__stdcall`` keyword. `cdecl`:idx: The cdecl convention means that a procedure shall use the same convention - as the C compiler. Under windows the generated C procedure is declared with + as the C compiler. Under Windows the generated C procedure is declared with the ``__cdecl`` keyword. `safecall`:idx: From 2fa35126b09e3487fbb82328238e59b9a4dd6d4c Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 5 Jan 2019 21:12:53 +0000 Subject: [PATCH 028/350] Fix getAddrInfo, add IPPROTO_ICMPV6 Closes #10198 --- lib/posix/posix_linux_amd64_consts.nim | 1 + lib/posix/posix_nintendoswitch_consts.nim | 1 + lib/posix/posix_other_consts.nim | 1 + lib/pure/nativesockets.nim | 7 +++-- tests/stdlib/tgetaddrinfo.nim | 36 +++++++++++++++++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/stdlib/tgetaddrinfo.nim diff --git a/lib/posix/posix_linux_amd64_consts.nim b/lib/posix/posix_linux_amd64_consts.nim index c23005b1ea..dfbfe7f647 100644 --- a/lib/posix/posix_linux_amd64_consts.nim +++ b/lib/posix/posix_linux_amd64_consts.nim @@ -295,6 +295,7 @@ const IF_NAMESIZE* = cint(16) const IPPROTO_IP* = cint(0) const IPPROTO_IPV6* = cint(41) const IPPROTO_ICMP* = cint(1) +const IPPROTO_ICMPV6* = cint(58) const IPPROTO_RAW* = cint(255) const IPPROTO_TCP* = cint(6) const IPPROTO_UDP* = cint(17) diff --git a/lib/posix/posix_nintendoswitch_consts.nim b/lib/posix/posix_nintendoswitch_consts.nim index f0c0dd717e..1e782d92e7 100644 --- a/lib/posix/posix_nintendoswitch_consts.nim +++ b/lib/posix/posix_nintendoswitch_consts.nim @@ -237,6 +237,7 @@ const IF_NAMESIZE* = cint(16) const IPPROTO_IP* = cint(0) const IPPROTO_IPV6* = cint(41) const IPPROTO_ICMP* = cint(1) +const IPPROTO_ICMPV6* = cint(58) const IPPROTO_RAW* = cint(255) const IPPROTO_TCP* = cint(6) const IPPROTO_UDP* = cint(17) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 2b4b709411..cd5199078d 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -302,6 +302,7 @@ var IF_NAMESIZE* {.importc: "IF_NAMESIZE", header: "".}: cint var IPPROTO_IP* {.importc: "IPPROTO_IP", header: "".}: cint var IPPROTO_IPV6* {.importc: "IPPROTO_IPV6", header: "".}: cint var IPPROTO_ICMP* {.importc: "IPPROTO_ICMP", header: "".}: cint +var IPPROTO_ICMPV6* {.importc: "IPPROTO_ICMPV6", header: "".}: cint var IPPROTO_RAW* {.importc: "IPPROTO_RAW", header: "".}: cint var IPPROTO_TCP* {.importc: "IPPROTO_TCP", header: "".}: cint var IPPROTO_UDP* {.importc: "IPPROTO_UDP", header: "".}: cint diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index f98f9a4446..96c377187e 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -71,6 +71,7 @@ type IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. + IPPROTO_ICMPV6 ## Control message protocol for IPv6. Unsupported on Windows. Servent* = object ## information about a service name*: string @@ -154,6 +155,7 @@ when not useWinVersion: of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 of IPPROTO_RAW: result = posix.IPPROTO_RAW of IPPROTO_ICMP: result = posix.IPPROTO_ICMP + of IPPROTO_ICMPV6: result = posix.IPPROTO_ICMPV6 else: proc toInt(domain: Domain): cint = @@ -179,7 +181,7 @@ proc toSockType*(protocol: Protocol): SockType = SOCK_STREAM of IPPROTO_UDP: SOCK_DGRAM - of IPPROTO_IP, IPPROTO_IPV6, IPPROTO_RAW, IPPROTO_ICMP: + of IPPROTO_IP, IPPROTO_IPV6, IPPROTO_RAW, IPPROTO_ICMP, IPPROTO_ICMPV6: SOCK_RAW proc createNativeSocket*(domain: Domain = AF_INET, @@ -255,7 +257,8 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and not defined(android) and not defined(haiku): if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED - var gaiResult = getaddrinfo(address, $port, addr(hints), result) + let socket_port = if sockType == SOCK_RAW: "" else: $port + var gaiResult = getaddrinfo(address, socket_port, addr(hints), result) if gaiResult != 0'i32: when useWinVersion: raiseOSError(osLastError()) diff --git a/tests/stdlib/tgetaddrinfo.nim b/tests/stdlib/tgetaddrinfo.nim new file mode 100644 index 0000000000..39102e131e --- /dev/null +++ b/tests/stdlib/tgetaddrinfo.nim @@ -0,0 +1,36 @@ +discard """ + exitcode: 0 + output: "" +""" + +# bug: https://github.com/nim-lang/Nim/issues/10198 + +import nativesockets + +block DGRAM_UDP: + let aiList = getAddrInfo("127.0.0.1", 999.Port, AF_INET, SOCK_DGRAM, IPPROTO_UDP) + doAssert aiList != nil + doAssert aiList.ai_addr != nil + doAssert aiList.ai_addrlen == 16 + doAssert aiList.ai_next == nil + freeAddrInfo aiList + +when defined(posix): + + block RAW_ICMP: + # the port will be ignored + let aiList = getAddrInfo("127.0.0.1", 999.Port, AF_INET, SOCK_RAW, IPPROTO_ICMP) + doAssert aiList != nil + doAssert aiList.ai_addr != nil + doAssert aiList.ai_addrlen == 16 + doAssert aiList.ai_next == nil + freeAddrInfo aiList + + block RAW_ICMPV6: + # the port will be ignored + let aiList = getAddrInfo("::1", 999.Port, AF_INET6, SOCK_RAW, IPPROTO_ICMPV6) + doAssert aiList != nil + doAssert aiList.ai_addr != nil + doAssert aiList.ai_addrlen == 28 + doAssert aiList.ai_next == nil + freeAddrInfo aiList From ea3e32c2249ee010eab986850ce52c50fdf4c46e Mon Sep 17 00:00:00 2001 From: c-blake Date: Sun, 6 Jan 2019 18:42:49 -0500 Subject: [PATCH 029/350] Remove totally unneeded import of hashes. (#10221) --- lib/pure/parsecfg.nim | 2 +- lib/pure/parsesql.nim | 2 +- lib/pure/parsexml.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index b991dd57f1..106d59017e 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -111,7 +111,7 @@ ## dict.writeConfig("config.ini") import - hashes, strutils, lexbase, streams, tables + strutils, lexbase, streams, tables include "system/inclrtl" diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 20f02e8156..f0961829bc 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -11,7 +11,7 @@ ## parser. It parses PostgreSQL syntax and the SQL ANSI standard. import - hashes, strutils, lexbase + strutils, lexbase # ------------------- scanner ------------------------------------------------- diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 0967f79830..953c5cdde2 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -147,7 +147,7 @@ an HTML document contains. ]## import - hashes, strutils, lexbase, streams, unicode + strutils, lexbase, streams, unicode # the parser treats ``
`` as ``

`` From b7be67349b2677a251aba8fda999a49b7a85af70 Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Mon, 7 Jan 2019 00:48:55 +0100 Subject: [PATCH 030/350] Fix ODBC SQL Error string decoding (#10207) [backport] ODBC Errors were presented to the users as a sequence of characters. I.e.: >test_oracle.exe Error: ['H', 'Y', '0', '0', '0', '\x00', '\x00', ...] test_oracle.nim(15) test_oracle test_oracle.nim(8) test_oracle db_odbc.nim(534) open db_odbc.nim(168) dbError Error: unhandled exception: ODBC Error [DbError] This patch fix the string decoding, creating a real string: >test_oracle.exe Error: HY000 [Oracle][ODBC][Ora]ORA-12541: TNS:no listener test_oracle.nim(15) test_oracle test_oracle.nim(8) test_oracle db_odbc.nim(534) open db_odbc.nim(168) dbError Error: unhandled exception: ODBC Error [DbError] --- lib/impure/db_odbc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index 72d7aa8001..b7af5128a9 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -131,7 +131,7 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. 511.TSqlSmallInt, retSz.addr.PSQLSMALLINT) except: discard - return (res.int, $sqlState, $nativeErr, $errMsg) + return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg)) proc dbError*(db: var DbConn) {. tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = From 15773c455a404c65fe7ff4e06b09bed5942d2d57 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Mon, 7 Jan 2019 01:50:21 +0200 Subject: [PATCH 031/350] Add changelog for the isNil JavaScript change (#10184) Forgot to add an item, breaking change --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index 47642cab53..b3ccc1b2e4 100644 --- a/changelog.md +++ b/changelog.md @@ -52,6 +52,8 @@ - splitFile now correctly handles edge cases, see #10047 +- `isNil` is no longer false for undefined in the JavaScript backend: now it's true for both nil and undefined. Use `isNull` or `isUndefined` if you need exact equallity: `isNil` is consistent with `===`, `isNull` and `isUndefined` with `==`. + #### Breaking changes in the compiler - The compiler now implements the "generic symbol prepass" for `when` statements From e77dd683eb9b6b2b9c2638f4145f8857fece12cb Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Mon, 7 Jan 2019 05:21:17 +0530 Subject: [PATCH 032/350] Fix defer not not-working at top level (#10191) --- compiler/sem.nim | 2 -- compiler/semexprs.nim | 2 ++ tests/async/tasyncfilewrite.nim | 3 +-- tests/exception/tdefer1.nim | 10 +--------- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index 8332af346e..3763c9b84c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -556,8 +556,6 @@ proc isEmptyTree(n: PNode): bool = else: result = false proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = - if n.kind == nkDefer: - localError(c.config, n.info, "defer statement not supported at top level") if c.topStmts == 0 and not isImportSystemStmt(c.graph, n): if sfSystemModule notin c.module.flags and not isEmptyTree(n): c.importTable.addSym c.graph.systemModule # import the "System" identifier diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 19d008557f..7a124c7698 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2624,6 +2624,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkStaticStmt: result = semStaticStmt(c, n) of nkDefer: + if c.currentScope == c.topLevelScope: + localError(c.config, n.info, "defer statement not supported at top level") n.sons[0] = semExpr(c, n.sons[0]) if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]): localError(c.config, n.info, "'defer' takes a 'void' expression") diff --git a/tests/async/tasyncfilewrite.nim b/tests/async/tasyncfilewrite.nim index 373b93301b..3baf2bbc63 100644 --- a/tests/async/tasyncfilewrite.nim +++ b/tests/async/tasyncfilewrite.nim @@ -9,7 +9,6 @@ import os, asyncfile, asyncdispatch const F = "test_async.txt" removeFile(F) -defer: removeFile(F) let f = openAsync(F, fmWrite) var futs = newSeq[Future[void]]() for i in 1..3: @@ -17,4 +16,4 @@ for i in 1..3: waitFor(all(futs)) f.close() echo readFile(F) - +removeFile(F) diff --git a/tests/exception/tdefer1.nim b/tests/exception/tdefer1.nim index b84ba76810..db46bad274 100644 --- a/tests/exception/tdefer1.nim +++ b/tests/exception/tdefer1.nim @@ -1,6 +1,5 @@ discard """ output: '''hi -hi 1 hi 2 @@ -10,13 +9,6 @@ A''' # bug #1742 -template test(): untyped = - let a = 0 - defer: echo "hi" - a - -let i = test() - import strutils let x = try: parseInt("133a") except: -1 @@ -31,7 +23,7 @@ template atFuncEnd = template testB(): untyped = let a = 0 - defer: echo "hi" # Delete this line to make it work + defer: echo "hi" a proc main = From be9d1280ae39b64e6c76179a788034ad6c50a52d Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Mon, 7 Jan 2019 00:52:19 +0100 Subject: [PATCH 033/350] Added examples to strtabs module (#10160) --- lib/pure/strtabs.nim | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index d8a23286a5..7bafe16756 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -12,6 +12,34 @@ ## style-insensitive mode. An efficient string substitution operator ``%`` ## for the string table is also provided. +runnableExamples: + + var t = newStringTable() + t["name"] = "John" + t["city"] = "Monaco" + doAssert t.len == 2 + doAssert t.hasKey "name" + doAssert "name" in t + +## String tables can be created from a table constructor: + +runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + + +## When using the style insensitive mode ``modeStyleInsensitive``, +## all letters are compared case insensitively within the ASCII range +## and underscores are ignored. + +runnableExamples: + + var x = newStringTable(modeStyleInsensitive) + x["first_name"] = "John" + x["LastName"] = "Doe" + + doAssert x["firstName"] == "John" + doAssert x["last_name"] == "Doe" + import hashes, strutils @@ -214,6 +242,9 @@ proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. rtlFunc, extern: "nstFormat".} = ## The `%` operator for string tables. + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert "${name} lives in ${city}" % t == "John lives in Monaco" const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} result = "" From 5345c5b1302e7beca5eb88ed510570e8e4431413 Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 7 Jan 2019 10:37:49 +0100 Subject: [PATCH 034/350] remove deprecated modules (#10215) * removed from `compiler`: * lists (deprecated 2 years ago) * removed from `lib` (all deprecated 3 years ago): * ssl * matchers * httpserver * removed from `lib/deprecated`: * unsigned * actors (and three accompanying tests) * parseurl * moved to `lib/deprecated`: * securehash (the reason for not directly removing - it was deprecated (only) one year ago) --- compiler/lists.nim | 28 - doc/lib.rst | 14 +- lib/deprecated/core/unsigned.nim | 18 - lib/deprecated/pure/actors.nim | 239 -------- lib/deprecated/pure/actors.nim.cfg | 3 - lib/deprecated/pure/parseurl.nim | 112 ---- lib/deprecated/pure/rawsockets.nim | 14 - lib/{ => deprecated}/pure/securehash.nim | 0 lib/impure/ssl.nim | 99 ---- lib/pure/httpserver.nim | 535 ------------------ lib/pure/matchers.nim | 68 --- .../keineschweine/enet_server/enet_server.nim | 2 +- .../keineschweine/server/old_dirserver.nim | 2 +- .../keineschweine/server/old_sg_server.nim | 2 +- tests/threads/tactors.nim | 13 - tests/threads/tactors2.nim | 25 - tests/threads/trecursive_actor.nim | 20 - tests/tuples/tuple_with_nil.nim | 1 - tools/kochdocs.nim | 4 - 19 files changed, 5 insertions(+), 1194 deletions(-) delete mode 100644 compiler/lists.nim delete mode 100644 lib/deprecated/core/unsigned.nim delete mode 100644 lib/deprecated/pure/actors.nim delete mode 100644 lib/deprecated/pure/actors.nim.cfg delete mode 100644 lib/deprecated/pure/parseurl.nim delete mode 100644 lib/deprecated/pure/rawsockets.nim rename lib/{ => deprecated}/pure/securehash.nim (100%) delete mode 100644 lib/impure/ssl.nim delete mode 100644 lib/pure/httpserver.nim delete mode 100644 lib/pure/matchers.nim delete mode 100644 tests/threads/tactors.nim delete mode 100644 tests/threads/tactors2.nim delete mode 100644 tests/threads/trecursive_actor.nim diff --git a/compiler/lists.nim b/compiler/lists.nim deleted file mode 100644 index bfd0522047..0000000000 --- a/compiler/lists.nim +++ /dev/null @@ -1,28 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module is deprecated, don't use it. -# TODO Remove this - -import os - -static: - echo "WARNING: imported deprecated module compiler/lists.nim, use seq ore lists from the standard library" - -proc appendStr*(list: var seq[string]; data: string) {.deprecated.} = - # just use system.add - list.add(data) - -proc includeStr(list: var seq[string]; data: string): bool {.deprecated.} = - if list.contains(data): - result = true - else: - result = false - list.add data - diff --git a/doc/lib.rst b/doc/lib.rst index 89e3cca40d..a16bf2677b 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -141,9 +141,6 @@ String handling Ropes can represent very long strings efficiently; especially concatenation is done in O(1) instead of O(n). -* `matchers `_ - This module contains various string matchers for email addresses, etc. - * `subexes `_ This module implements advanced string substitution operations. @@ -275,8 +272,8 @@ Internet Protocols and Support module. * `net `_ - This module implements a high-level sockets API. It will replace the - ``sockets`` module in the future. + This module implements a high-level sockets API. It replaces the + ``sockets`` module. * `nativesockets `_ This module implements a low-level sockets API. @@ -456,13 +453,6 @@ Database support for other databases too. -Other ------ - -* `ssl `_ - This module provides an easy to use sockets-style - Nim interface to the OpenSSL library. - Wrappers ======== diff --git a/lib/deprecated/core/unsigned.nim b/lib/deprecated/core/unsigned.nim deleted file mode 100644 index 93a29e1c94..0000000000 --- a/lib/deprecated/core/unsigned.nim +++ /dev/null @@ -1,18 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** Since version 0.11.4 this module is deprecated. -## -## This module implemented basic arithmetic operators for unsigned integers. -## These operators are now available in the ``system`` module directly. - -{.deprecated.} - -export `shr`, `shl`, `and`, `or`, `xor`, `==`, `+`, `-`, `*`, `div`, `mod`, - `<=`, `<` diff --git a/lib/deprecated/pure/actors.nim b/lib/deprecated/pure/actors.nim deleted file mode 100644 index 77c67a3e48..0000000000 --- a/lib/deprecated/pure/actors.nim +++ /dev/null @@ -1,239 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## `Actor`:idx: support for Nim. An actor is implemented as a thread with -## a channel as its inbox. This module requires the ``--threads:on`` -## command line switch. -## -## Example: -## -## .. code-block:: nim -## -## var -## a: ActorPool[int, void] -## createActorPool(a) -## for i in 0 ..< 300: -## a.spawn(i, proc (x: int) {.thread.} = echo x) -## a.join() -## -## **Note**: This whole module is deprecated. Use `threadpool` and ``spawn`` -## instead. - -{.deprecated.} - -from os import sleep - -type - Task*[In, Out] = object{.pure, final.} ## a task - when Out isnot void: - receiver*: ptr Channel[Out] ## the receiver channel of the response - action*: proc (x: In): Out {.thread.} ## action to execute; - ## sometimes useful - shutDown*: bool ## set to tell an actor to shut-down - data*: In ## the data to process - - Actor[In, Out] = object{.pure, final.} - i: Channel[Task[In, Out]] - t: Thread[ptr Actor[In, Out]] - - PActor*[In, Out] = ptr Actor[In, Out] ## an actor - -proc spawn*[In, Out](action: proc( - self: PActor[In, Out]){.thread.}): PActor[In, Out] = - ## creates an actor; that is a thread with an inbox. The caller MUST call - ## ``join`` because that also frees the actor's associated resources. - result = cast[PActor[In, Out]](allocShared0(sizeof(result[]))) - open(result.i) - createThread(result.t, action, result) - -proc inbox*[In, Out](self: PActor[In, Out]): ptr Channel[In] = - ## gets a pointer to the associated inbox of the actor `self`. - result = addr(self.i) - -proc running*[In, Out](a: PActor[In, Out]): bool = - ## returns true if the actor `a` is running. - result = running(a.t) - -proc ready*[In, Out](a: PActor[In, Out]): bool = - ## returns true if the actor `a` is ready to process new messages. - result = ready(a.i) - -proc join*[In, Out](a: PActor[In, Out]) = - ## joins an actor. - joinThread(a.t) - close(a.i) - deallocShared(a) - -proc recv*[In, Out](a: PActor[In, Out]): Task[In, Out] = - ## receives a task from `a`'s inbox. - result = recv(a.i) - -proc send*[In, Out, X, Y](receiver: PActor[In, Out], msg: In, - sender: PActor[X, Y]) = - ## sends a message to `a`'s inbox. - var t: Task[In, Out] - t.receiver = addr(sender.i) - shallowCopy(t.data, msg) - send(receiver.i, t) - -proc send*[In, Out](receiver: PActor[In, Out], msg: In, - sender: ptr Channel[Out] = nil) = - ## sends a message to `receiver`'s inbox. - var t: Task[In, Out] - t.receiver = sender - shallowCopy(t.data, msg) - send(receiver.i, t) - -proc sendShutdown*[In, Out](receiver: PActor[In, Out]) = - ## send a shutdown message to `receiver`. - var t: Task[In, Out] - t.shutdown = true - send(receiver.i, t) - -proc reply*[In, Out](t: Task[In, Out], m: Out) = - ## sends a message to io's output message box. - when Out is void: - {.error: "you cannot reply to a void outbox".} - assert t.receiver != nil - send(t.receiver[], m) - - -# ----------------- actor pools ---------------------------------------------- - -type - ActorPool*[In, Out] = object{.pure, final.} ## an actor pool - actors: seq[PActor[In, Out]] - when Out isnot void: - outputs: Channel[Out] - -proc `^`*[T](f: ptr Channel[T]): T = - ## alias for 'recv'. - result = recv(f[]) - -proc poolWorker[In, Out](self: PActor[In, Out]) {.thread.} = - while true: - var m = self.recv - if m.shutDown: break - when Out is void: - m.action(m.data) - else: - send(m.receiver[], m.action(m.data)) - #self.reply() - -proc createActorPool*[In, Out](a: var ActorPool[In, Out], poolSize = 4) = - ## creates an actor pool. - newSeq(a.actors, poolSize) - when Out isnot void: - open(a.outputs) - for i in 0 ..< a.actors.len: - a.actors[i] = spawn(poolWorker[In, Out]) - -proc sync*[In, Out](a: var ActorPool[In, Out], polling=50) = - ## waits for every actor of `a` to finish with its work. Currently this is - ## implemented as polling every `polling` ms and has a slight chance - ## of failing since we check for every actor to be in `ready` state and not - ## for messages still in ether. This will change in a later - ## version, however. - var allReadyCount = 0 - while true: - var wait = false - for i in 0..high(a.actors): - if not a.actors[i].i.ready: - wait = true - allReadyCount = 0 - break - if not wait: - # it's possible that some actor sent a message to some other actor but - # both appeared to be non-working as the message takes some time to - # arrive. We assume that this won't take longer than `polling` and - # simply attempt a second time and declare victory then. ;-) - inc allReadyCount - if allReadyCount > 1: break - sleep(polling) - -proc terminate*[In, Out](a: var ActorPool[In, Out]) = - ## terminates each actor in the actor pool `a` and frees the - ## resources attached to `a`. - var t: Task[In, Out] - t.shutdown = true - for i in 0.. send message to the thread which has the least - # messages pending: - var minIdx = -1 - var minVal = high(int) - for i in 0..high(p.actors): - var curr = p.actors[i].i.peek - if curr == 0: - # ok, is ready now: - p.actors[i].i.send(t) - return - if curr < minVal and curr >= 0: - minVal = curr - minIdx = i - if minIdx >= 0: - p.actors[minIdx].i.send(t) - else: - raise newException(DeadThreadError, "cannot send message; thread died") - -proc spawn*[In, Out](p: var ActorPool[In, Out], input: In, - action: proc (input: In): Out {.thread.} - ): ptr Channel[Out] = - ## uses the actor pool to run ``action(input)`` concurrently. - ## `spawn` is guaranteed to not block. - var t: Task[In, Out] - setupTask() - result = addr(p.outputs) - t.receiver = result - schedule() - -proc spawn*[In](p: var ActorPool[In, void], input: In, - action: proc (input: In) {.thread.}) = - ## uses the actor pool to run ``action(input)`` concurrently. - ## `spawn` is guaranteed to not block. - var t: Task[In, void] - setupTask() - schedule() - -when not defined(testing) and isMainModule: - var - a: ActorPool[int, void] - createActorPool(a) - for i in 0 ..< 300: - a.spawn(i, proc (x: int) {.thread.} = echo x) - - when false: - proc treeDepth(n: PNode): int {.thread.} = - var x = a.spawn(treeDepth, n.le) - var y = a.spawn(treeDepth, n.ri) - result = max(^x, ^y) + 1 - - a.join() - - diff --git a/lib/deprecated/pure/actors.nim.cfg b/lib/deprecated/pure/actors.nim.cfg deleted file mode 100644 index c6bb9c5451..0000000000 --- a/lib/deprecated/pure/actors.nim.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# to shut up the tester: ---threads:on - diff --git a/lib/deprecated/pure/parseurl.nim b/lib/deprecated/pure/parseurl.nim deleted file mode 100644 index b19f6dc852..0000000000 --- a/lib/deprecated/pure/parseurl.nim +++ /dev/null @@ -1,112 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warnings:** This module is deprecated since version 0.10.2. -## Use the `uri `_ module instead. -## -## Parses & constructs URLs. - -{.deprecated.} - -import strutils - -type - Url* = tuple[ ## represents a *Uniform Resource Locator* (URL) - ## any optional component is "" if it does not exist - scheme, username, password, - hostname, port, path, query, anchor: string] - -proc parseUrl*(url: string): Url {.deprecated.} = - var i = 0 - - var scheme, username, password: string = "" - var hostname, port, path, query, anchor: string = "" - - var temp = "" - - if url[i] != '/': # url isn't a relative path - while true: - # Scheme - if url[i] == ':': - if url[i+1] == '/' and url[i+2] == '/': - scheme = temp - temp.setLen(0) - inc(i, 3) # Skip the // - # Authority(username, password) - if url[i] == '@': - username = temp - let colon = username.find(':') - if colon >= 0: - password = username.substr(colon+1) - username = username.substr(0, colon-1) - temp.setLen(0) - inc(i) #Skip the @ - # hostname(subdomain, domain, port) - if url[i] == '/' or url[i] == '\0': - hostname = temp - let colon = hostname.find(':') - if colon >= 0: - port = hostname.substr(colon+1) - hostname = hostname.substr(0, colon-1) - - temp.setLen(0) - break - - temp.add(url[i]) - inc(i) - - if url[i] == '/': inc(i) # Skip the '/' - # Path - while true: - if url[i] == '?': - path = temp - temp.setLen(0) - if url[i] == '#': - if temp[0] == '?': - query = temp - else: - path = temp - temp.setLen(0) - - if url[i] == '\0': - if temp[0] == '?': - query = temp - elif temp[0] == '#': - anchor = temp - else: - path = temp - break - - temp.add(url[i]) - inc(i) - - return (scheme, username, password, hostname, port, path, query, anchor) - -proc `$`*(u: Url): string {.deprecated.} = - ## turns the URL `u` into its string representation. - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - result.add("/") - result.add(u.path) - result.add(u.query) - result.add(u.anchor) - diff --git a/lib/deprecated/pure/rawsockets.nim b/lib/deprecated/pure/rawsockets.nim deleted file mode 100644 index 876334f9e8..0000000000 --- a/lib/deprecated/pure/rawsockets.nim +++ /dev/null @@ -1,14 +0,0 @@ -import nativesockets -export nativesockets - -{.warning: "rawsockets module is deprecated, use nativesockets instead".} - -template newRawSocket*(domain, sockType, protocol: cint): untyped = - {.warning: "newRawSocket is deprecated, use newNativeSocket instead".} - newNativeSocket(domain, sockType, protocol) - -template newRawSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): untyped = - {.warning: "newRawSocket is deprecated, use newNativeSocket instead".} - newNativeSocket(domain, sockType, protocol) diff --git a/lib/pure/securehash.nim b/lib/deprecated/pure/securehash.nim similarity index 100% rename from lib/pure/securehash.nim rename to lib/deprecated/pure/securehash.nim diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim deleted file mode 100644 index 0bcb3f217f..0000000000 --- a/lib/impure/ssl.nim +++ /dev/null @@ -1,99 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module provides an easy to use sockets-style -## nim interface to the OpenSSL library. -## -## **Warning:** This module is deprecated, use the SSL procedures defined in -## the ``net`` module instead. - -{.deprecated.} - -import openssl, strutils, os - -type - SecureSocket* = object - ssl: SslPtr - bio: BIO - -proc connect*(sock: var SecureSocket, address: string, - port: int): int = - ## Connects to the specified `address` on the specified `port`. - ## Returns the result of the certificate validation. - SslLoadErrorStrings() - ERR_load_BIO_strings() - - if SSL_library_init() != 1: - raiseOSError(osLastError()) - - var ctx = SSL_CTX_new(SSLv23_client_method()) - if ctx == nil: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - - #if SSL_CTX_load_verify_locations(ctx, - # "/tmp/openssl-0.9.8e/certs/vsign1.pem", NIL) == 0: - # echo("Failed load verify locations") - # ERR_print_errors_fp(stderr) - - sock.bio = BIO_new_ssl_connect(ctx) - if BIO_get_ssl(sock.bio, addr(sock.ssl)) == 0: - raiseOSError(osLastError()) - - if BIO_set_conn_hostname(sock.bio, address & ":" & $port) != 1: - raiseOSError(osLastError()) - - if BIO_do_connect(sock.bio) <= 0: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - - result = SSL_get_verify_result(sock.ssl) - -proc recvLine*(sock: SecureSocket, line: var TaintedString): bool = - ## Acts in a similar fashion to the `recvLine` in the sockets module. - ## Returns false when no data is available to be read. - ## `Line` must be initialized and not nil! - setLen(line.string, 0) - while true: - var c: array[0..0, char] - var n = BIO_read(sock.bio, addr c, c.len.cint) - if n <= 0: return false - if c[0] == '\r': - n = BIO_read(sock.bio, addr c, c.len.cint) - if n > 0 and c[0] == '\L': - return true - elif n <= 0: - return false - elif c[0] == '\L': return true - add(line.string, c[0]) - - -proc send*(sock: SecureSocket, data: string) = - ## Writes `data` to the socket. - if BIO_write(sock.bio, data, data.len.cint) <= 0: - raiseOSError(osLastError()) - -proc close*(sock: SecureSocket) = - ## Closes the socket - if BIO_free(sock.bio) <= 0: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - -when not defined(testing) and isMainModule: - var s: SecureSocket - echo connect(s, "smtp.gmail.com", 465) - - #var buffer: array[0..255, char] - #echo BIO_read(bio, buffer, buffer.len) - var buffer: string = "" - - echo s.recvLine(buffer) - echo buffer - echo buffer.len - diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim deleted file mode 100644 index a81e8c0a8d..0000000000 --- a/lib/pure/httpserver.nim +++ /dev/null @@ -1,535 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a simple HTTP-Server. -## -## **Warning**: This module will soon be deprecated in favour of -## the ``asyncdispatch`` module, you should use it instead. -## -## Example: -## -## .. code-block:: nim -## import strutils, sockets, httpserver -## -## var counter = 0 -## proc handleRequest(client: Socket, path, query: string): bool {.procvar.} = -## inc(counter) -## client.send("Hello for the $#th time." % $counter & wwwNL) -## return false # do not stop processing -## -## run(handleRequest, Port(80)) -## - -import parseutils, strutils, os, osproc, strtabs, streams, sockets, asyncio - -const - wwwNL* = "\r\L" - ServerSig = "Server: httpserver.nim/1.0.0" & wwwNL - -# --------------- output messages -------------------------------------------- - -proc sendTextContentType(client: Socket) = - send(client, "Content-type: text/html" & wwwNL) - send(client, wwwNL) - -proc sendStatus(client: Socket, status: string) = - send(client, "HTTP/1.1 " & status & wwwNL) - -proc badRequest(client: Socket) = - # Inform the client that a request it has made has a problem. - send(client, "HTTP/1.1 400 Bad Request" & wwwNL) - sendTextContentType(client) - send(client, "

Your browser sent a bad request, " & - "such as a POST without a Content-Length.

" & wwwNL) - -when false: - proc cannotExec(client: Socket) = - send(client, "HTTP/1.1 500 Internal Server Error" & wwwNL) - sendTextContentType(client) - send(client, "

Error prohibited CGI execution." & wwwNL) - -proc headers(client: Socket, filename: string) = - # XXX could use filename to determine file type - send(client, "HTTP/1.1 200 OK" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - -proc notFound(client: Socket) = - send(client, "HTTP/1.1 404 NOT FOUND" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - send(client, "Not Found" & wwwNL) - send(client, "

The server could not fulfill" & wwwNL) - send(client, "your request because the resource specified" & wwwNL) - send(client, "is unavailable or nonexistent.

" & wwwNL) - send(client, "" & wwwNL) - -proc unimplemented(client: Socket) = - send(client, "HTTP/1.1 501 Method Not Implemented" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - send(client, "Method Not Implemented" & - "" & - "

HTTP request method not supported.

" & - "" & wwwNL) - -# ----------------- file serving --------------------------------------------- - -when false: - proc discardHeaders(client: Socket) = skip(client) - -proc serveFile*(client: Socket, filename: string) = - ## serves a file to the client. - var f: File - if open(f, filename): - headers(client, filename) - const bufSize = 8000 # != 8K might be good for memory manager - var buf = alloc(bufsize) - while true: - var bytesread = readBuffer(f, buf, bufsize) - if bytesread > 0: - var byteswritten = send(client, buf, bytesread) - if bytesread != bytesWritten: - dealloc(buf) - close(f) - raiseOSError(osLastError()) - if bytesread != bufSize: break - dealloc(buf) - close(f) - else: - notFound(client) - -# ------------------ CGI execution ------------------------------------------- -when false: - # TODO: Fix this, or get rid of it. - type - RequestMethod = enum reqGet, reqPost - - proc executeCgi(client: Socket, path, query: string, meth: RequestMethod) = - var env = newStringTable(modeCaseInsensitive) - var contentLength = -1 - case meth - of reqGet: - discardHeaders(client) - - env["REQUEST_METHOD"] = "GET" - env["QUERY_STRING"] = query - of reqPost: - var buf = TaintedString"" - var dataAvail = false - while dataAvail: - dataAvail = recvLine(client, buf) # TODO: This is incorrect. - var L = toLowerAscii(buf.string) - if L.startsWith("content-length:"): - var i = len("content-length:") - while L[i] in Whitespace: inc(i) - contentLength = parseInt(substr(L, i)) - - if contentLength < 0: - badRequest(client) - return - - env["REQUEST_METHOD"] = "POST" - env["CONTENT_LENGTH"] = $contentLength - - send(client, "HTTP/1.0 200 OK" & wwwNL) - - var process = startProcess(command=path, env=env) - if meth == reqPost: - # get from client and post to CGI program: - var buf = alloc(contentLength) - if recv(client, buf, contentLength) != contentLength: - dealloc(buf) - raiseOSError() - var inp = process.inputStream - inp.writeData(buf, contentLength) - dealloc(buf) - - var outp = process.outputStream - var line = newStringOfCap(120).TaintedString - while true: - if outp.readLine(line): - send(client, line.string) - send(client, wwwNL) - elif not running(process): break - - # --------------- Server Setup ----------------------------------------------- - - proc acceptRequest(client: Socket) = - var cgi = false - var query = "" - var buf = TaintedString"" - discard recvLine(client, buf) - var path = "" - var data = buf.string.split() - var meth = reqGet - - var q = find(data[1], '?') - - # extract path - if q >= 0: - # strip "?..." from path, this may be found in both POST and GET - path = "." & data[1].substr(0, q-1) - else: - path = "." & data[1] - # path starts with "/", by adding "." in front of it we serve files from cwd - - if cmpIgnoreCase(data[0], "GET") == 0: - if q >= 0: - cgi = true - query = data[1].substr(q+1) - elif cmpIgnoreCase(data[0], "POST") == 0: - cgi = true - meth = reqPost - else: - unimplemented(client) - - if path[path.len-1] == '/' or existsDir(path): - path = path / "index.html" - - if not existsFile(path): - discardHeaders(client) - notFound(client) - else: - when defined(Windows): - var ext = splitFile(path).ext.toLowerAscii - if ext == ".exe" or ext == ".cgi": - # XXX: extract interpreter information here? - cgi = true - else: - if {fpUserExec, fpGroupExec, fpOthersExec} * path.getFilePermissions != {}: - cgi = true - if not cgi: - serveFile(client, path) - else: - executeCgi(client, path, query, meth) - -type - Server* = object of RootObj ## contains the current server state - socket: Socket - port: Port - client*: Socket ## the socket to write the file data to - reqMethod*: string ## Request method. GET or POST. - path*, query*: string ## path and query the client requested - headers*: StringTableRef ## headers with which the client made the request - body*: string ## only set with POST requests - ip*: string ## ip address of the requesting client - - PAsyncHTTPServer* = ref AsyncHTTPServer - AsyncHTTPServer = object of Server - asyncSocket: AsyncSocket - -proc open*(s: var Server, port = Port(80), reuseAddr = false) = - ## creates a new server at port `port`. If ``port == 0`` a free port is - ## acquired that can be accessed later by the ``port`` proc. - s.socket = socket(AF_INET) - if s.socket == invalidSocket: raiseOSError(osLastError()) - if reuseAddr: - s.socket.setSockOpt(OptReuseAddr, true) - bindAddr(s.socket, port) - listen(s.socket) - - if port == Port(0): - s.port = getSockName(s.socket) - else: - s.port = port - s.client = invalidSocket - s.reqMethod = "" - s.body = "" - s.path = "" - s.query = "" - s.headers = {:}.newStringTable() - -proc port*(s: var Server): Port = - ## get the port number the server has acquired. - result = s.port - -proc next*(s: var Server) = - ## proceed to the first/next request. - var client: Socket - new(client) - var ip: string - acceptAddr(s.socket, client, ip) - s.client = client - s.ip = ip - s.headers = newStringTable(modeCaseInsensitive) - #headers(s.client, "") - var data = "" - s.client.readLine(data) - if data == "": - # Socket disconnected - s.client.close() - next(s) - return - var header = "" - while true: - s.client.readLine(header) - if header == "\c\L": break - if header != "": - var i = 0 - var key = "" - var value = "" - i = header.parseUntil(key, ':') - inc(i) # skip : - i += header.skipWhiteSpace(i) - i += header.parseUntil(value, {'\c', '\L'}, i) - s.headers[key] = value - else: - s.client.close() - next(s) - return - - var i = skipWhitespace(data) - if skipIgnoreCase(data, "GET") > 0: - s.reqMethod = "GET" - inc(i, 3) - elif skipIgnoreCase(data, "POST") > 0: - s.reqMethod = "POST" - inc(i, 4) - else: - unimplemented(s.client) - s.client.close() - next(s) - return - - if s.reqMethod == "POST": - # Check for Expect header - if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLowerAscii == "100-continue": - s.client.sendStatus("100 Continue") - else: - s.client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if s.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(s.headers["Content-Length"], contentLength) == 0: - badRequest(s.client) - s.client.close() - next(s) - return - else: - var totalRead = 0 - var totalBody = "" - while totalRead < contentLength: - var chunkSize = 8000 - if (contentLength - totalRead) < 8000: - chunkSize = (contentLength - totalRead) - var bodyData = newString(chunkSize) - var octetsRead = s.client.recv(cstring(bodyData), chunkSize) - if octetsRead <= 0: - s.client.close() - next(s) - return - totalRead += octetsRead - totalBody.add(bodyData) - if totalBody.len != contentLength: - s.client.close() - next(s) - return - - s.body = totalBody - else: - badRequest(s.client) - s.client.close() - next(s) - return - - var L = skipWhitespace(data, i) - inc(i, L) - # XXX we ignore "HTTP/1.1" etc. for now here - var query = 0 - var last = i - while last < data.len and data[last] notin Whitespace: - if data[last] == '?' and query == 0: query = last - inc(last) - if query > 0: - s.query = data.substr(query+1, last-1) - s.path = data.substr(i, query-1) - else: - s.query = "" - s.path = data.substr(i, last-1) - -proc close*(s: Server) = - ## closes the server (and the socket the server uses). - close(s.socket) - -proc run*(handleRequest: proc (client: Socket, - path, query: string): bool {.closure.}, - port = Port(80)) = - ## encapsulates the server object and main loop - var s: Server - open(s, port, reuseAddr = true) - #echo("httpserver running on port ", s.port) - while true: - next(s) - if handleRequest(s.client, s.path, s.query): break - close(s.client) - close(s) - -# -- AsyncIO begin - -proc nextAsync(s: PAsyncHTTPServer) = - ## proceed to the first/next request. - var client: Socket - new(client) - var ip: string - acceptAddr(getSocket(s.asyncSocket), client, ip) - s.client = client - s.ip = ip - s.headers = newStringTable(modeCaseInsensitive) - #headers(s.client, "") - var data = "" - s.client.readLine(data) - if data == "": - # Socket disconnected - s.client.close() - return - var header = "" - while true: - s.client.readLine(header) # TODO: Very inefficient here. Prone to DOS. - if header == "\c\L": break - if header != "": - var i = 0 - var key = "" - var value = "" - i = header.parseUntil(key, ':') - inc(i) # skip : - if i < header.len: - i += header.skipWhiteSpace(i) - i += header.parseUntil(value, {'\c', '\L'}, i) - s.headers[key] = value - else: - s.client.close() - return - - var i = skipWhitespace(data) - if skipIgnoreCase(data, "GET") > 0: - s.reqMethod = "GET" - inc(i, 3) - elif skipIgnoreCase(data, "POST") > 0: - s.reqMethod = "POST" - inc(i, 4) - else: - unimplemented(s.client) - s.client.close() - return - - if s.reqMethod == "POST": - # Check for Expect header - if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLowerAscii == "100-continue": - s.client.sendStatus("100 Continue") - else: - s.client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if s.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(s.headers["Content-Length"], contentLength) == 0: - badRequest(s.client) - s.client.close() - return - else: - var totalRead = 0 - var totalBody = "" - while totalRead < contentLength: - var chunkSize = 8000 - if (contentLength - totalRead) < 8000: - chunkSize = (contentLength - totalRead) - var bodyData = newString(chunkSize) - var octetsRead = s.client.recv(cstring(bodyData), chunkSize) - if octetsRead <= 0: - s.client.close() - return - totalRead += octetsRead - totalBody.add(bodyData) - if totalBody.len != contentLength: - s.client.close() - return - - s.body = totalBody - else: - badRequest(s.client) - s.client.close() - return - - var L = skipWhitespace(data, i) - inc(i, L) - # XXX we ignore "HTTP/1.1" etc. for now here - var query = 0 - var last = i - while last < data.len and data[last] notin Whitespace: - if data[last] == '?' and query == 0: query = last - inc(last) - if query > 0: - s.query = data.substr(query+1, last-1) - s.path = data.substr(i, query-1) - else: - s.query = "" - s.path = data.substr(i, last-1) - -proc asyncHTTPServer*(handleRequest: proc (server: PAsyncHTTPServer, client: Socket, - path, query: string): bool {.closure, gcsafe.}, - port = Port(80), address = "", - reuseAddr = false): PAsyncHTTPServer = - ## Creates an Asynchronous HTTP server at ``port``. - var capturedRet: PAsyncHTTPServer - new(capturedRet) - capturedRet.asyncSocket = asyncSocket() - capturedRet.asyncSocket.handleAccept = - proc (s: AsyncSocket) = - nextAsync(capturedRet) - let quit = handleRequest(capturedRet, capturedRet.client, capturedRet.path, - capturedRet.query) - if quit: capturedRet.asyncSocket.close() - if reuseAddr: - capturedRet.asyncSocket.setSockOpt(OptReuseAddr, true) - - capturedRet.asyncSocket.bindAddr(port, address) - capturedRet.asyncSocket.listen() - if port == Port(0): - capturedRet.port = getSockName(capturedRet.asyncSocket) - else: - capturedRet.port = port - - capturedRet.client = invalidSocket - capturedRet.reqMethod = "" - capturedRet.body = "" - capturedRet.path = "" - capturedRet.query = "" - capturedRet.headers = {:}.newStringTable() - result = capturedRet - -proc register*(d: Dispatcher, s: PAsyncHTTPServer) = - ## Registers a ``PAsyncHTTPServer`` with a ``Dispatcher``. - d.register(s.asyncSocket) - -proc close*(h: PAsyncHTTPServer) = - ## Closes the ``PAsyncHTTPServer``. - h.asyncSocket.close() - -when not defined(testing) and isMainModule: - var counter = 0 - - var s: Server - open(s, Port(0)) - echo("httpserver running on port ", s.port) - while true: - next(s) - - inc(counter) - s.client.send("Hello, Andreas, for the $#th time. $# ? $#" % [ - $counter, s.path, s.query] & wwwNL) - - close(s.client) - close(s) - diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim deleted file mode 100644 index 97223ed019..0000000000 --- a/lib/pure/matchers.nim +++ /dev/null @@ -1,68 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module contains various string matchers for email addresses, etc. -## -## **Warning:** This module is deprecated since version 0.14.0. -{.deprecated.} - -{.deadCodeElim: on.} # dce option deprecated - -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! - -include "system/inclrtl" - -import parseutils, strutils - -proc validEmailAddress*(s: string): bool {.noSideEffect, - rtl, extern: "nsuValidEmailAddress".} = - ## returns true if `s` seems to be a valid e-mail address. - ## The checking also uses a domain list. - const - chars = Letters + Digits + {'!','#','$','%','&', - '\'','*','+','/','=','?','^','_','`','{','}','|','~','-','.'} - var i = 0 - if i >= s.len or s[i] notin chars or s[i] == '.': return false - while i < s.len and s[i] in chars: - if i+1 < s.len and s[i] == '.' and s[i+1] == '.': return false - inc(i) - if i >= s.len or s[i] != '@': return false - var j = len(s)-1 - if j >= 0 and s[j] notin Letters: return false - while j >= i and s[j] in Letters: dec(j) - inc(i) # skip '@' - while i < s.len and s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) - if i != s.len: return false - - var x = substr(s, j+1) - if len(x) == 2 and x[0] in Letters and x[1] in Letters: return true - case toLowerAscii(x) - of "com", "org", "net", "gov", "mil", "biz", "info", "mobi", "name", - "aero", "jobs", "museum": return true - else: return false - -proc parseInt*(s: string, value: var int, validRange: HSlice[int, int]) {. - noSideEffect, rtl, extern: "nmatchParseInt".} = - ## parses `s` into an integer in the range `validRange`. If successful, - ## `value` is modified to contain the result. Otherwise no exception is - ## raised and `value` is not touched; this way a reasonable default value - ## won't be overwritten. - var x = value - try: - discard parseutils.parseInt(s, x, 0) - except OverflowError: - discard - if x in validRange: value = x - -when isMainModule: - doAssert "wuseldusel@codehome.com".validEmailAddress - -{.pop.} - diff --git a/tests/manyloc/keineschweine/enet_server/enet_server.nim b/tests/manyloc/keineschweine/enet_server/enet_server.nim index 3bb2593865..336e577557 100644 --- a/tests/manyloc/keineschweine/enet_server/enet_server.nim +++ b/tests/manyloc/keineschweine/enet_server/enet_server.nim @@ -103,7 +103,7 @@ handlers[HZoneJoinReq] = proc(client: PClient; buffer: PBuffer) = when true: - import parseopt, matchers, os, json + import parseopt, os, json if enetInit() != 0: diff --git a/tests/manyloc/keineschweine/server/old_dirserver.nim b/tests/manyloc/keineschweine/server/old_dirserver.nim index cfb0b03776..390b738aa6 100644 --- a/tests/manyloc/keineschweine/server/old_dirserver.nim +++ b/tests/manyloc/keineschweine/server/old_dirserver.nim @@ -157,7 +157,7 @@ proc poll*(timeout: int = 250) = c.outputBuf.flush() when true: - import parseopt, matchers, strutils + import parseopt, strutils var cfgFile = "dirserver_settings.json" for kind, key, val in getOpt(): case kind diff --git a/tests/manyloc/keineschweine/server/old_sg_server.nim b/tests/manyloc/keineschweine/server/old_sg_server.nim index d046df9dd8..473880e2be 100644 --- a/tests/manyloc/keineschweine/server/old_sg_server.nim +++ b/tests/manyloc/keineschweine/server/old_sg_server.nim @@ -142,7 +142,7 @@ proc poll*(timeout: int = 250) = c.outputBuf.flush() when true: - import parseopt, matchers, strutils + import parseopt, strutils var zoneCfgFile = "./server_settings.json" for kind, key, val in getOpt(): case kind diff --git a/tests/threads/tactors.nim b/tests/threads/tactors.nim deleted file mode 100644 index ea052b9bd5..0000000000 --- a/tests/threads/tactors.nim +++ /dev/null @@ -1,13 +0,0 @@ -discard """ - outputsub: "150" -""" - -import actors - -var - pool: ActorPool[int, void] -createActorPool(pool) -for i in 0 ..< 300: - pool.spawn(i, proc (x: int) {.thread.} = echo x) -pool.join() - diff --git a/tests/threads/tactors2.nim b/tests/threads/tactors2.nim deleted file mode 100644 index e8afe203c4..0000000000 --- a/tests/threads/tactors2.nim +++ /dev/null @@ -1,25 +0,0 @@ -discard """ - output: "1" -""" - -import actors - -type - some_type {.pure, final.} = object - bla: int - -proc thread_proc(input: some_type): some_type {.thread.} = - result.bla = 1 - -proc main() = - var actorPool: ActorPool[some_type, some_type] - createActorPool(actorPool, 1) - - var some_data: some_type - - var inchannel = spawn(actorPool, some_data, thread_proc) - var recv_data = ^inchannel - close(inchannel[]) - echo recv_data.bla - -main() diff --git a/tests/threads/trecursive_actor.nim b/tests/threads/trecursive_actor.nim deleted file mode 100644 index d7072aa531..0000000000 --- a/tests/threads/trecursive_actor.nim +++ /dev/null @@ -1,20 +0,0 @@ -discard """ - disabled: yes - outputsub: "0" -""" - -import actors - -var - a: TActorPool[int, void] -createActorPool(a) - -proc task(i: int) {.thread.} = - echo i - if i != 0: a.spawn (i-1, task) - -# count from 9 till 0 and check 0 is somewhere in the output -a.spawn(9, task) -a.join() - - diff --git a/tests/tuples/tuple_with_nil.nim b/tests/tuples/tuple_with_nil.nim index e09c53407d..1b210b2bfe 100644 --- a/tests/tuples/tuple_with_nil.nim +++ b/tests/tuples/tuple_with_nil.nim @@ -4,7 +4,6 @@ import parseutils import unicode import math import fenv -#import unsigned import pegs import streams diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 658b9ead71..6741fcf5d8 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -126,7 +126,6 @@ lib/js/asyncjs.nim lib/pure/os.nim lib/pure/strutils.nim lib/pure/math.nim -lib/pure/matchers.nim lib/std/editdistance.nim lib/std/wordwrap.nim lib/experimental/diff.nim @@ -161,10 +160,8 @@ lib/impure/db_mysql.nim lib/impure/db_sqlite.nim lib/impure/db_odbc.nim lib/pure/db_common.nim -lib/pure/httpserver.nim lib/pure/httpclient.nim lib/pure/smtp.nim -lib/impure/ssl.nim lib/pure/ropes.nim lib/pure/unidecode/unidecode.nim lib/pure/xmlparser.nim @@ -218,7 +215,6 @@ lib/pure/selectors.nim lib/pure/sugar.nim lib/pure/collections/chains.nim lib/pure/asyncfile.nim -lib/deprecated/pure/ftpclient.nim lib/pure/asyncftpclient.nim lib/pure/lenientops.nim lib/pure/md5.nim From 87f8ec5b92d5647ab4b1875262e845d51dd82763 Mon Sep 17 00:00:00 2001 From: zah Date: Mon, 7 Jan 2019 13:10:54 +0200 Subject: [PATCH 035/350] Fix #10073 (#10218) --- compiler/semexprs.nim | 9 ++++++++- tests/statictypes/tstatictypes.nim | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7a124c7698..f03af3db97 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1043,7 +1043,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = of skConst: markUsed(c.config, n.info, s, c.graph.usageSym) onUse(n.info, s) - case skipTypes(s.typ, abstractInst-{tyTypeDesc}).kind + let typ = skipTypes(s.typ, abstractInst-{tyTypeDesc}) + case typ.kind of tyNil, tyChar, tyInt..tyInt64, tyFloat..tyFloat128, tyTuple, tySet, tyUInt..tyUInt64: if s.magic == mNone: result = inlineConst(c, n, s) @@ -1061,6 +1062,12 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = # deal with two different ``[]``. if s.ast.len == 0: result = inlineConst(c, n, s) else: result = newSymNode(s, n.info) + of tyStatic: + if typ.n != nil: + result = typ.n + result.typ = typ.base + else: + result = newSymNode(s, n.info) else: result = newSymNode(s, n.info) of skMacro: diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index 2a3b7332df..9888165cce 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -116,3 +116,19 @@ block: Tensor[B: static[Backend]; T] = object BackProp[B: static[Backend],T] = proc (gradient: Tensor[B,T]): Tensor[B,T] + +# https://github.com/nim-lang/Nim/issues/10073 +block: + proc foo[N: static int](x: var int, + y: int, + z: static int, + arr: array[N, int]): auto = + var t1 = (a: x, b: y, c: z, d: N) + var t2 = (x, y, z, N) + doAssert t1 == t2 + result = t1 + + var y = 20 + var x = foo(y, 10, 15, [1, 2, 3]) + doAssert x == (20, 10, 15, 3) + From 139fa396e8fa0e8603d4f53ac90841421e50aa3f Mon Sep 17 00:00:00 2001 From: alaviss Date: Mon, 7 Jan 2019 18:27:07 +0700 Subject: [PATCH 036/350] os.execShellCmd: use WEXITSTATUS to retrieve exit code (#10222) According to POSIX, system() shall returns the termination status in the format specified by waitpid(), which means WEXITSTATUS should be used to retrieve the exit code portably. This fixes execShellCmd on Haiku. --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 96833056a0..a218121ed1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1298,7 +1298,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## shell involved, use the `execProcess` proc of the `osproc` ## module. when defined(posix): - result = c_system(command) shr 8 + result = WEXITSTATUS(c_system(command)) else: result = c_system(command) From 044cef152f6006927a905d69dc527cada8206b0f Mon Sep 17 00:00:00 2001 From: jcosborn Date: Mon, 7 Jan 2019 05:36:06 -0600 Subject: [PATCH 037/350] add custom pragma support for var and let symbols (#9582) * add custom pragma support for var and let symbols * updated changelog for custom pragmas on var and let symbols * add oldast switch for backwards compatibility --- changelog.md | 10 +++++---- compiler/ast.nim | 7 +++++++ compiler/commands.nim | 2 ++ compiler/guards.nim | 8 ++++---- compiler/options.nim | 3 ++- compiler/semstmts.nim | 23 +++++++++++++++++---- doc/advopt.txt | 1 + lib/core/macros.nim | 8 +++++++- tests/pragmas/tcustom_pragma.nim | 35 ++++++++++++++++++++++++++------ 9 files changed, 77 insertions(+), 20 deletions(-) diff --git a/changelog.md b/changelog.md index b3ccc1b2e4..05f65516d4 100644 --- a/changelog.md +++ b/changelog.md @@ -23,7 +23,8 @@ - The undocumented ``#? strongSpaces`` parsing mode has been removed. - The `not` operator is now always a unary operator, this means that code like ``assert not isFalse(3)`` compiles. - +- `getImpl` on a `var` or `let` symbol will now return the full `IdentDefs` + tree from the symbol declaration instead of just the initializer portion. #### Breaking changes in the standard library @@ -133,8 +134,9 @@ proc enumToString*(enums: openArray[enum]): string = the `gcsafe` pragma block. - added os.getCurrentProcessId() - User defined pragmas are now allowed in the pragma blocks -- Pragma blocks are now longer eliminated from the typed AST tree to preserve +- Pragma blocks are no longer eliminated from the typed AST tree to preserve pragmas for further analysis by macros +- Custom pragmas are now supported for `var` and `let` symbols. ### Language changes @@ -143,10 +145,10 @@ proc enumToString*(enums: openArray[enum]): string = it's more recognizable and allows tools like github to recognize it as Nim, see [#9647](https://github.com/nim-lang/Nim/issues/9647). The previous extension will continue to work. -- Pragma syntax is now consistent. Previous syntax where type pragmas did not +- Pragma syntax is now consistent. Previous syntax where type pragmas did not follow the type name is now deprecated. Also pragma before generic parameter list is deprecated to be consistent with how pragmas are used with a proc. See - [#8514](https://github.com/nim-lang/Nim/issues/8514) and + [#8514](https://github.com/nim-lang/Nim/issues/8514) and [#1872](https://github.com/nim-lang/Nim/issues/1872) for further details. diff --git a/compiler/ast.nim b/compiler/ast.nim index 5f5f296cbc..24891d6d3e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1087,6 +1087,13 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym, when debugIds: registerId(result) +proc astdef*(s: PSym): PNode = + # get only the definition (initializer) portion of the ast + if s.ast != nil and s.ast.kind == nkIdentDefs: + s.ast[2] + else: + s.ast + proc isMetaType*(t: PType): bool = return t.kind in tyMetaTypes or (t.kind == tyStatic and t.n == nil) or diff --git a/compiler/commands.nim b/compiler/commands.nim index 8b73884e8f..47687046f6 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -291,6 +291,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool of "patterns": result = contains(conf.options, optPatterns) of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace) of "nilseqs": result = contains(conf.options, optNilSeqs) + of "oldast": result = contains(conf.options, optOldAst) else: invalidCmdLineOption(conf, passCmd1, switch, info) proc processPath(conf: ConfigRef; path: string, info: TLineInfo, @@ -508,6 +509,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; localError(conf, info, errOnOrOffExpectedButXFound % arg) of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info) of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info) + of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info) of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info) of "floatchecks": processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info) diff --git a/compiler/guards.nim b/compiler/guards.nim index a01c023e48..46e18d3bf9 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -257,9 +257,9 @@ proc canon*(n: PNode; o: Operators): PNode = for i in 0 ..< n.len: result.sons[i] = canon(n.sons[i], o) elif n.kind == nkSym and n.sym.kind == skLet and - n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin + + n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin + someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv): - result = n.sym.ast.copyTree + result = n.sym.astdef.copyTree else: result = n case result.getMagic @@ -395,8 +395,8 @@ proc usefulFact(n: PNode; o: Operators): PNode = # if a: # ... # We make can easily replace 'a' by '2 < x' here: - if n.sym.ast != nil: - result = usefulFact(n.sym.ast, o) + if n.sym.astdef != nil: + result = usefulFact(n.sym.astdef, o) elif n.kind == nkStmtListExpr: result = usefulFact(n.lastSon, o) diff --git a/compiler/options.nim b/compiler/options.nim index 3730d16f81..ea159ee1d8 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -40,7 +40,8 @@ type # please make sure we have under 32 options optMemTracker, optHotCodeReloading, optLaxStrings, - optNilSeqs + optNilSeqs, + optOldAst TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 39f200ba86..12283e0422 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -330,9 +330,9 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = proc checkNilable(c: PContext; v: PSym) = if {sfGlobal, sfImportC} * v.flags == {sfGlobal} and {tfNotNil, tfNeedsInit} * v.typ.flags != {}: - if v.ast.isNil: + if v.astdef.isNil: message(c.config, v.info, warnProveInit, v.name.s) - elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags: + elif tfNotNil in v.typ.flags and tfNotNil notin v.astdef.typ.flags: message(c.config, v.info, warnProveInit, v.name.s) include semasgn @@ -518,8 +518,6 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = message(c.config, a.info, warnShadowIdent, v.name.s) if a.kind != nkVarTuple: if def.kind != nkEmpty: - # this is needed for the evaluation pass and for the guard checking: - v.ast = def if sfThread in v.flags: localError(c.config, def.info, errThreadvarCannotInit) setVarType(c, v, typ) b = newNodeI(nkIdentDefs, a.info) @@ -531,6 +529,23 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = addSon(b, a.sons[length-2]) addSon(b, copyTree(def)) addToVarSection(c, result, n, b) + if optOldAst in c.config.options: + if def.kind != nkEmpty: + v.ast = def + else: + # this is needed for the evaluation pass, guard checking + # and custom pragmas: + var ast = newNodeI(nkIdentDefs, a.info) + if a[j].kind == nkPragmaExpr: + var p = newNodeI(nkPragmaExpr, a.info) + p.add newSymNode(v) + p.add a[j][1].copyTree + ast.add p + else: + ast.add newSymNode(v) + ast.add a.sons[length-2].copyTree + ast.add def + v.ast = ast else: if def.kind in {nkPar, nkTupleConstr}: v.ast = def[j] # bug #7663, for 'nim check' this can be a non-tuple: diff --git a/doc/advopt.txt b/doc/advopt.txt index 5ddd064ef6..2e91deed9a 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -76,6 +76,7 @@ Advanced options: strings is allowed; only for backwards compatibility --nilseqs:on|off allow 'nil' for strings/seqs for backwards compatibility + --oldast:on|off use old AST for backwards compatibility --skipCfg do not read the general configuration file --skipUserCfg do not read the user's configuration file --skipParentCfg do not read the parent dirs' configuration files diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 291858bede..64334161e6 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1402,8 +1402,14 @@ proc customPragmaNode(n: NimNode): NimNode = let impl = n.getImpl() if impl.kind in RoutineNodes: return impl.pragma + elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr: + return impl[0][1] else: - return typ.getImpl()[0][1] + let timpl = typ.getImpl() + if timpl.len>0 and timpl[0].len>1: + return timpl[0][1] + else: + return timpl if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index 0bc4d2f18b..fefcc0b5fa 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -175,24 +175,47 @@ var foo: Something foo.cardinal = north doAssert foo.b.hasCustomPragma(thingy) == true - -proc myproc(s: string): int = +proc myproc(s: string): int = {.thingy.}: s.len doAssert myproc("123") == 3 let xx = compiles: - proc myproc_bad(s: string): int = + proc myproc_bad(s: string): int = {.not_exist.}: s.len doAssert: xx == false - -macro checkSym(s: typed{nkSym}): untyped = +macro checkSym(s: typed{nkSym}): untyped = let body = s.getImpl.body doAssert body[1].kind == nnkPragmaBlock doAssert body[1][0].kind == nnkPragma doAssert body[1][0][0] == bindSym"thingy" -checkSym(myproc) \ No newline at end of file +checkSym(myproc) + +# var and let pragmas +block: + template myAttr() {.pragma.} + template myAttr2(x: int) {.pragma.} + template myAttr3(x: string) {.pragma.} + + let a {.myAttr,myAttr2(2),myAttr3:"test".}: int = 0 + let b {.myAttr,myAttr2(2),myAttr3:"test".} = 0 + var x {.myAttr,myAttr2(2),myAttr3:"test".}: int = 0 + var y {.myAttr,myAttr2(2),myAttr3:"test".}: int + var z {.myAttr,myAttr2(2),myAttr3:"test".} = 0 + + template check(s: untyped) = + doAssert s.hasCustomPragma(myAttr) + doAssert s.hasCustomPragma(myAttr2) + doAssert s.getCustomPragmaVal(myAttr2) == 2 + doAssert s.hasCustomPragma(myAttr3) + doAssert s.getCustomPragmaVal(myAttr3) == "test" + + check(a) + check(b) + check(x) + check(y) + check(z) From 387831d657bcbd130b7e739e4d58f7ccc29adae9 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Mon, 7 Jan 2019 20:32:47 +0530 Subject: [PATCH 038/350] Show error when trying to export individual enum field (#10109) --- compiler/semexprs.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f03af3db97..8852cb6535 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2335,7 +2335,6 @@ proc semExportExcept(c: PContext, n: PNode): PNode = proc semExport(c: PContext, n: PNode): PNode = result = newNodeI(nkExportStmt, n.info) - for i in 0.. Date: Mon, 7 Jan 2019 18:09:57 +0100 Subject: [PATCH 039/350] Fix for sizeof bitsize combination (#10227) * fix #10082 * added test --- compiler/sizealignoffsetimpl.nim | 11 ++++++++--- tests/misc/tsizeof.nim | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index ea4730f57a..fb256b8951 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -154,12 +154,17 @@ proc computePackedObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, initialOf if result == szIllegalRecursion: break of nkSym: + var size = szUnknownSize if n.sym.bitsize == 0: computeSizeAlign(conf, n.sym.typ) - n.sym.offset = initialOffset.int - result = n.sym.offset + n.sym.typ.size - else: + size = n.sym.typ.size.int + + if initialOffset == szUnknownSize or size == szUnknownSize: + n.sym.offset = szUnknownSize result = szUnknownSize + else: + n.sym.offset = int(initialOffset) + result = initialOffset + n.sym.typ.size else: result = szUnknownSize diff --git a/tests/misc/tsizeof.nim b/tests/misc/tsizeof.nim index 4422e900e4..25c566171c 100644 --- a/tests/misc/tsizeof.nim +++ b/tests/misc/tsizeof.nim @@ -402,6 +402,18 @@ type assert sizeof(C) == 3 + +type + MixedBitsize = object {.packed.} + a: uint32 + b {.bitsize: 8.}: uint8 + c {.bitsize: 1.}: uint8 + d {.bitsize: 7.}: uint8 + e {.bitsize: 16.}: uint16 + f: uint32 + +doAssert sizeof(MixedBitsize) == 12 + if failed: quit("FAIL") else: From 20402579a2d22f506bf6079976c3480a7bd06531 Mon Sep 17 00:00:00 2001 From: alaviss Date: Tue, 8 Jan 2019 14:42:10 +0700 Subject: [PATCH 040/350] system/strmantle: avoid leaking docs to system.nim [ci skip] (#10233) thanks @timotheecour for spotting this --- lib/system/strmantle.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index ceaecb4f9c..3c681fc535 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## Compilerprocs for strings that do not depend on the string implementation. +# Compilerprocs for strings that do not depend on the string implementation. proc cmpStrings(a, b: string): int {.inline, compilerProc.} = let alen = a.len From 3ed833198b95e74de29d2b0905247ead52478aa3 Mon Sep 17 00:00:00 2001 From: treeform Date: Tue, 8 Jan 2019 00:50:25 -0800 Subject: [PATCH 041/350] Better LibSSL search order. (#10230) --- lib/wrappers/openssl.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 2f072d5c7c..68f0d8eb26 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -38,7 +38,7 @@ when useWinVersion: from winlean import SocketHandle else: - const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + const versions = "(.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.1.1|.46|.45|.44|.43|.41|.39|.38|.10|)" when defined(macosx): const From 427435a814248b8b5778aa0d78f128a3f60978cd Mon Sep 17 00:00:00 2001 From: Clyybber Date: Tue, 8 Jan 2019 10:13:39 +0100 Subject: [PATCH 042/350] Don't use deprecated pragma syntax (#10187) * Don't use deprecated pragma syntax * Remove pure pragma, since it's a noop now --- compiler/types.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/types.nim b/compiler/types.nim index b163ca4e99..63a87f14eb 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -756,7 +756,7 @@ type TTypeCmpFlags* = set[TTypeCmpFlag] - TSameTypeClosure = object {.pure.} + TSameTypeClosure = object cmp: TDistinctCompare recCheck: int flags: TTypeCmpFlags From 821920aa391c1361ee17befedf4452edba47c87c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 8 Jan 2019 01:38:34 -0800 Subject: [PATCH 043/350] [nimpretty] fix #10211; fix #10199 (#10212) * [nimpretty] fix #10211; fix #10199 * address comments * un-document --backup and set its default to false --- changelog.md | 2 ++ nimpretty/nimpretty.nim | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index 05f65516d4..35d2adbd1c 100644 --- a/changelog.md +++ b/changelog.md @@ -156,6 +156,8 @@ proc enumToString*(enums: openArray[enum]): string = - `jsondoc` now include a `moduleDescription` field with the module description. `jsondoc0` shows comments as it's own objects as shown in the documentation. +- `nimpretty`: --backup now defaults to `off` instead of `on` and the flag was + un-documented; use `git` instead of relying on backup files. ### Compiler changes - The deprecated `fmod` proc is now unavailable on the VM'. diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index 628bc163eb..c6c558f92f 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -25,11 +25,10 @@ const Usage: nimpretty [options] file.nim Options: - --backup:on|off create a backup file before overwritting (default: ON) - --output:file set the output file (default: overwrite the .nim file) - --indent:N set the number of spaces that is used for indentation - --version show the version - --help show this help + --output:file set the output file (default: overwrite the input file) + --indent:N[=2] set the number of spaces that is used for indentation + --version show the version + --help show this help """ proc writeHelp() = @@ -62,7 +61,11 @@ proc prettyPrint(infile, outfile: string, opt: PrettyOptions) = proc main = var infile, outfile: string - var backup = true + var backup = false + # when `on`, create a backup file of input in case + # `prettyPrint` could over-write it (note that the backup may happen even + # if input is not actually over-written, when nimpretty is a noop). + # --backup was un-documented (rely on git instead). var opt: PrettyOptions for kind, key, val in getopt(): case kind @@ -79,9 +82,14 @@ proc main = of cmdEnd: assert(false) # cannot happen if infile.len == 0: quit "[Error] no input file." + if outfile.len == 0: + outfile = infile + if not existsFile(outfile) or not sameFile(infile, outfile): + backup = false # no backup needed since won't be over-written if backup: - os.copyFile(source=infile, dest=changeFileExt(infile, ".nim.backup")) - if outfile.len == 0: outfile = infile + let infileBackup = infile & ".backup" # works with .nim or .nims + echo "writing backup " & infile & " > " & infileBackup + os.copyFile(source = infile, dest = infileBackup) prettyPrint(infile, outfile, opt) main() From 6737634d88a70a3d87774c9f51f2ac6d2bf4da4f Mon Sep 17 00:00:00 2001 From: alaviss Date: Tue, 8 Jan 2019 18:41:15 +0700 Subject: [PATCH 044/350] os.execShellCmd: fixes #10231 (#10232) Darwin has long deprecated the wait union, but their macros still assume it unless you define _POSIX_C_SOURCE. This trips up C++ compilers. This commit duplicates the behavior of WEXITSTATUS when _POSIX_C_SOURCE is defined. --- lib/pure/os.nim | 4 +++- tests/stdlib/t10231.nim | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/stdlib/t10231.nim diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a218121ed1..3a959d4e2c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1297,7 +1297,9 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - when defined(posix): + when defined(macosx): + result = c_system(command) shr 8 + elif defined(posix): result = WEXITSTATUS(c_system(command)) else: result = c_system(command) diff --git a/tests/stdlib/t10231.nim b/tests/stdlib/t10231.nim new file mode 100644 index 0000000000..5d1101aa42 --- /dev/null +++ b/tests/stdlib/t10231.nim @@ -0,0 +1,13 @@ +discard """ + target: cpp + action: run + exitcode: 0 +""" + +import os + +if paramCount() == 0: + # main process + doAssert execShellCmd(getAppFilename().quoteShell & " test") == 1 +else: + quit 1 From c60916a2aff121fc0220f9452803a7bcfb6300d0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 8 Jan 2019 04:08:19 -0800 Subject: [PATCH 045/350] [CI] fixes #10041 move bulk of `travis` and `appveyor` logic to koch.nim (#10183) --- .travis.yml | 21 +++------------------ appveyor.yml | 18 +----------------- koch.nim | 46 +++++++++++++++++++++++++++++++++++++++++++--- tools/kochdocs.nim | 6 ++++++ 4 files changed, 53 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index abd115701f..32a40bcaf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,26 +40,11 @@ before_script: - sh build.sh - cd .. - export PATH=$(pwd)/bin${PATH:+:$PATH} + - echo PATH:${PATH} + script: - nim c koch - - env NIM_COMPILE_TO_CPP=false ./koch boot - - ./koch boot -d:release - - ./koch nimble - - nim e tests/test_nimscript.nims - #- nimble install zip -y - #- nimble install opengl - #- nimble install sdl1 - #- nimble install jester@#head -y - #- nimble install niminst - - nim c -d:nimCoroutines testament/tester - - testament/tester --pedantic all -d:nimCoroutines - - nim c -o:bin/nimpretty nimpretty/nimpretty.nim - - nim c -r nimpretty/tester.nim - - ./koch docs --git.commit:devel - - ./koch csource - - ./koch nimsuggest - - nim c -r nimsuggest/tester - - nim c -r nimdoc/tester + - ./koch runCI before_deploy: # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html diff --git a/appveyor.yml b/appveyor.yml index 9c8525d724..8c4b2a07e7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,22 +44,6 @@ install: build_script: - bin\nim c koch - - koch boot -d:release - - koch nimble - - nim e tests/test_nimscript.nims - - nim c -o:bin/nimpretty.exe nimpretty/nimpretty.nim - - nim c -r nimpretty/tester.nim -# - nimble install zip -y -# - nimble install opengl -y -# - nimble install sdl1 -y -# - nimble install jester@#head -y - - nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester - - nim c -d:nimCoroutines testament/tester - -test_script: - - testament\tester --pedantic all -d:nimCoroutines - - nim c -r nimdoc\tester -# - koch csource -# - koch zip + - koch runCI deploy: off diff --git a/koch.nim b/koch.nim index 3082dba8f2..afdce30837 100644 --- a/koch.nim +++ b/koch.nim @@ -26,6 +26,7 @@ import import tools / kochdocs const VersionAsString = system.NimVersion +const env_NIM_COMPILE_TO_CPP = "NIM_COMPILE_TO_CPP" const HelpText = """ @@ -56,6 +57,7 @@ Boot options: for bootstrapping Commands for core developers: + runCI runs continuous integration (CI), eg from travis docs [options] generates the full documentation csource -d:release builds the C sources for installation pdf builds the PDF documentation @@ -272,8 +274,9 @@ proc boot(args: string) = var output = "compiler" / "nim".exe var finalDest = "bin" / "nim".exe # default to use the 'c' command: - let defaultCommand = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": "cpp" else: "c" + let defaultCommand = if getEnv(env_NIM_COMPILE_TO_CPP, "false") == "true": "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" + echo "boot: defaultCommand: ", defaultCommand, " bootOptions: ", bootOptions let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & hostOs & "_" & hostCpu @@ -352,8 +355,8 @@ proc winReleaseArch(arch: string) = # Rebuilding koch is necessary because it uses its pointer size to # determine which mingw link to put in the NSIS installer. nimexec "c --cpu:$# koch" % cpu - exec "koch boot -d:release --cpu:$#" % cpu - exec "koch --latest zip -d:release" + kochExec "boot -d:release --cpu:$#" % cpu + kochExec "--latest zip -d:release" overwriteFile r"build\nim-$#.zip" % VersionAsString, r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch] @@ -428,6 +431,42 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) +proc runCI(cmd: string) = + doAssert cmd.len == 0, cmd # avoid silently ignoring + echo "runCI:", cmd + # note(@araq): Do not replace these commands with direct calls (eg boot()) + # as that would weaken our testing efforts. + when defined(posix): # appveyor (on windows) didn't run this + # todo: implement `execWithEnv` + exec("env NIM_COMPILE_TO_CPP=false $1 boot" % kochExe.quoteShell) + kochExec "boot -d:release" + kochExec "nimble" + exec "nim e tests/test_nimscript.nims" + + when false: + for pkg in "zip opengl sdl1 jester@#head niminst".split: + exec "nimble install -y" & pkg + + when defined(windows): + # note: will be over-written below + exec "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester" + when false: + kochExec "csource" + kochExec "zip" + + # main bottleneck: runs all main tests + exec "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines" + exec "nim c -r nimdoc/tester" + + nimCompile "nimpretty/nimpretty.nim" + exec "nim c -r nimpretty/tester.nim" + + when defined(posix): + kochExec "docs --git.commit:devel" + kochExec "csource" + kochExec "nimsuggest" + exec "nim c -r nimsuggest/tester" + proc pushCsources() = if not dirExists("../csources/.git"): quit "[Error] no csources git repository found" @@ -536,6 +575,7 @@ when isMainModule: of "distrohelper": geninstall() of "install": install(op.cmdLineRest) of "testinstall": testUnixInstall(op.cmdLineRest) + of "runci": runCI(op.cmdLineRest) of "test", "tests": tests(op.cmdLineRest) of "temp": temp(op.cmdLineRest) of "xtemp": xtemp(op.cmdLineRest) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 6741fcf5d8..19b0b3d78a 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -51,6 +51,12 @@ proc execCleanPath*(cmd: string, if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) +let kochExe* = os.getAppFilename() + # note: assumes `kochdocs` is only used by koch.nim + +proc kochExec*(cmd: string) = + exec kochExe.quoteShell & " " & cmd + proc nimexec*(cmd: string) = # Consider using `nimCompile` instead exec findNim() & " " & cmd From d0366c519487610ff26d28175e98a9c3ef178670 Mon Sep 17 00:00:00 2001 From: narimiran Date: Sat, 5 Jan 2019 10:12:13 +0100 Subject: [PATCH 046/350] update issue templates [ci skip] --- .github/ISSUE_TEMPLATE/bug_report.md | 24 +++++++++++------------ .github/ISSUE_TEMPLATE/feature_request.md | 17 ++++++++++++---- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 383ec23f8d..5fe7971a35 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,37 +1,32 @@ --- name: Bug report -about: You found an unexpected behaviour? Use this template. - +about: Have you found an unexpected behavior? Use this template. --- - - + +Function `echo` outputs the wrong string. ### Example - ```nim echo "Hello World!" ``` ---> ### Current Output - ### Expected Output - ``` Hello World! ``` ---> ### Possible Solution @@ -40,6 +35,11 @@ Hello World! ### Additional Information +``` +$ nim -v +Nim Compiler Version 0.1.2 +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 04da773bf3..52aa53a67d 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,9 +1,18 @@ --- name: Feature request -about: You want to suggest a new feature? Use this template. +about: Do you want to suggest a new feature? Use this template. --- + + + ### Summary @@ -18,6 +27,6 @@ about: You want to suggest a new feature? Use this template. ### Additional Information +* A link to a project where the issue is relevant. +* A link to a related issue or discussion. +--> From 21078798ead6f6ba107102913cd26970ef957e9d Mon Sep 17 00:00:00 2001 From: Andre von Houck Date: Tue, 8 Jan 2019 10:29:10 -0800 Subject: [PATCH 047/350] Fix libssl order. Newest one is 1.1 --- lib/wrappers/openssl.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 68f0d8eb26..447592a686 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -38,7 +38,7 @@ when useWinVersion: from winlean import SocketHandle else: - const versions = "(.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.1.1|.46|.45|.44|.43|.41|.39|.38|.10|)" + const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.46|.45|.44|.43|.41|.39|.38|.10|)" when defined(macosx): const From fb26b95f815b5426e0a8aad98ca0ff018ef1f4db Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Wed, 9 Jan 2019 00:14:47 +0530 Subject: [PATCH 048/350] {.deprecated: msg.} now works for vars and lets (#10234) --- compiler/pragmas.nim | 2 +- compiler/suggest.nim | 8 ++++++-- tests/deprecated/tannot.nim | 9 +++++++++ tests/deprecated/tnoannot.nim | 7 ------- 4 files changed, 16 insertions(+), 10 deletions(-) create mode 100644 tests/deprecated/tannot.nim delete mode 100644 tests/deprecated/tnoannot.nim diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 39b58d0b18..bb5707cd52 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -881,7 +881,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wExplain: sym.flags.incl sfExplain of wDeprecated: - if sym != nil and sym.kind in routineKinds + {skType}: + if sym != nil and sym.kind in routineKinds + {skType, skVar, skLet}: if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it) incl(sym.flags, sfDeprecated) elif sym != nil and sym.kind != skModule: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 144b86224f..f3f960136f 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -456,13 +456,17 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; proc extractPragma(s: PSym): PNode = if s.kind in routineKinds: result = s.ast[pragmasPos] - elif s.kind in {skType}: + elif s.kind in {skType, skVar, skLet}: # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma] result = s.ast[0][1] doAssert result == nil or result.kind == nkPragma proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) = - let pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s) + var pragmaNode: PNode + if optOldAst in conf.options and s.kind in {skVar, skLet}: + pragmaNode = nil + else: + pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s) let name = if s.kind == skEnumField and sfDeprecated notin s.flags: "enum '" & s.owner.name.s & "' which contains field '" & s.name.s & "'" else: s.name.s diff --git a/tests/deprecated/tannot.nim b/tests/deprecated/tannot.nim new file mode 100644 index 0000000000..d14f6cc235 --- /dev/null +++ b/tests/deprecated/tannot.nim @@ -0,0 +1,9 @@ +discard """ + nimout: '''tannot.nim(9, 1) Warning: efgh; foo1 is deprecated [Deprecated] +tannot.nim(9, 8) Warning: abcd; foo is deprecated [Deprecated] +''' +""" + +let foo* {.deprecated: "abcd".} = 42 +var foo1* {.deprecated: "efgh".} = 42 +foo1 = foo diff --git a/tests/deprecated/tnoannot.nim b/tests/deprecated/tnoannot.nim deleted file mode 100644 index ac168952e9..0000000000 --- a/tests/deprecated/tnoannot.nim +++ /dev/null @@ -1,7 +0,0 @@ -discard """ - errormsg: "annotation to deprecated not supported here" - line: 7 -""" - -var foo* {.deprecated.} = 42 -var foo1* {.deprecated: "no".} = 42 From bf3a308e86e7c5999855546962aed564218a8121 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 8 Jan 2019 15:58:47 -0800 Subject: [PATCH 049/350] [error messages, stacktraces] fix #8794 #9270 #9767 #9768 (#9766) * fixes #8794 : `Error: undeclared field: 'foo'` should show type (+ where type is defined) (hard to guess in generic code) * fixes #9270: `--listFullPaths` not honored by `declared in foo.nim` messages * fixes #9767: VM stacktrace doesn't honor --excessiveStackTrace:on * fixes #9768: VM stacktrace misses column info, can lead to ambiguous or harder to read stacktraces * refactors some col+1 code to col + ColOffset (self documents code) * make getProcHeader show declared info location also for types and all routine kinds (including macros,templates) instead of just (rather arbitrarily) for iterator,proc,func,method * --listFullPaths now is honored in more places * fix typo system/except.nim => lib/system/excpt.nim * remove substr(foo, 0) hack in compiler/vm.nim which seems old and not applicable anymore --- compiler/lookups.nim | 2 +- compiler/msgs.nim | 24 +++++++++++++++--------- compiler/options.nim | 2 +- compiler/semcall.nim | 15 ++++++++++++++- compiler/types.nim | 31 +++++++++++++++++-------------- compiler/vm.nim | 15 ++++++++++----- tests/errmsgs/m8794.nim | 2 ++ tests/errmsgs/t8794.nim | 39 +++++++++++++++++++++++++++++++++++++++ tests/errmsgs/t9768.nim | 30 ++++++++++++++++++++++++++++++ 9 files changed, 129 insertions(+), 31 deletions(-) create mode 100644 tests/errmsgs/m8794.nim create mode 100644 tests/errmsgs/t8794.nim create mode 100644 tests/errmsgs/t9768.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index d4959db12d..269447486d 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -150,7 +150,7 @@ type proc getSymRepr*(conf: ConfigRef; s: PSym): string = case s.kind - of skProc, skFunc, skMethod, skConverter, skIterator: + of routineKinds, skType: result = getProcHeader(conf, s) else: result = s.name.s diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 18c196085a..0dd5820b4a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -135,6 +135,10 @@ const WarningColor = fgYellow HintTitle = "Hint: " HintColor = fgGreen + # NOTE: currently line info line numbers start with 1, + # but column numbers start with 0, however most editors expect + # first column to be 1, so we need to +1 here + ColOffset* = 1 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L) @@ -155,7 +159,10 @@ template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string = if fileIdx.int32 < 0 or conf == nil: "???" else: - conf.m.fileInfos[fileIdx.int32].projPath.string + if optListFullPaths in conf.globalOptions: + conf.m.fileInfos[fileIdx.int32].fullPath.string + else: + conf.m.fileInfos[fileIdx.int32].projPath.string proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string = if fileIdx.int32 < 0 or conf == nil: result = "???" @@ -192,10 +199,10 @@ proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string = result = "???" return let absPath = conf.m.fileInfos[info.fileIndex.int32].fullPath.string - let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string if optListFullPaths in conf.globalOptions: result = absPath else: + let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string result = if absPath.len < relPath.len: absPath else: relPath proc toLinenumber*(info: TLineInfo): int {.inline.} = @@ -208,7 +215,9 @@ proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} = result = toFilename(conf, info) & ":" & $info.line proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} = - result = toFilename(conf, info) & "(" & $info.line & ", " & $(info.col+1) & ")" + # consider calling `helpers.lineInfoToString` instead + result = toFilename(conf, info) & "(" & $info.line & ", " & + $(info.col + ColOffset) & ")" proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info) @@ -359,7 +368,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) = styledMsgWriteln(styleBright, PosFormat % [toMsgFilename(conf, context.info), coordToStr(context.info.line.int), - coordToStr(context.info.col+1)], + coordToStr(context.info.col+ColOffset)], resetStyle, message) info = context.info @@ -446,7 +455,7 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s of hintMin..hintMax: HintTitle else: ErrorTitle result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int), - coordToStr(info.col+1)] & + coordToStr(info.col+ColOffset)] & title & getMessageStr(msg, arg) @@ -483,11 +492,8 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, color = HintColor if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)] inc(conf.hintCounter) - # NOTE: currently line info line numbers start with 1, - # but column numbers start with 0, however most editors expect - # first column to be 1, so we need to +1 here let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int), - coordToStr(info.col+1)] + coordToStr(info.col+ColOffset)] let s = getMessageStr(msg, arg) if not ignoreMsg: diff --git a/compiler/options.nim b/compiler/options.nim index ea159ee1d8..d39b0a2682 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -78,7 +78,7 @@ type # please make sure we have under 32 options optShowAllMismatches # show all overloading resolution candidates optWholeProject # for 'doc2': output any dependency optMixedMode # true if some module triggered C++ codegen - optListFullPaths + optListFullPaths # use full paths in toMsgFilename, toFilename optNoNimblePath optDynlibOverrideAll diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 7e0ea54904..7991640eac 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -287,7 +287,20 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = let ident = considerQuotedIdent(c, f, n).s if nfDotField in n.flags and nfExplicitCall notin n.flags: - result = errUndeclaredField % ident & result + let sym = n.sons[1].typ.sym + var typeHint = "" + if sym == nil: + #[ + Perhaps we're in a `compiles(foo.bar)` expression, or + in a concept, eg: + ExplainedConcept {.explain.} = concept o + o.foo is int + We coudl use: `(c.config $ n.sons[1].info)` to get more context. + ]# + discard + else: + typeHint = " for type " & getProcHeader(c.config, sym) + result = errUndeclaredField % ident & typeHint & " " & result else: if result.len == 0: result = errUndeclaredRoutine % ident else: result = errBadRoutine % [ident, result] diff --git a/compiler/types.nim b/compiler/types.nim index 63a87f14eb..5b8b9e6027 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -107,20 +107,23 @@ proc isFloatLit*(t: PType): bool {.inline.} = result = t.kind == tyFloat and t.n != nil and t.n.kind == nkFloatLit proc getProcHeader*(conf: ConfigRef; sym: PSym; prefer: TPreferedDesc = preferName): string = - result = sym.owner.name.s & '.' & sym.name.s & '(' - var n = sym.typ.n - for i in countup(1, sonsLen(n) - 1): - let p = n.sons[i] - if p.kind == nkSym: - add(result, p.sym.name.s) - add(result, ": ") - add(result, typeToString(p.sym.typ, prefer)) - if i != sonsLen(n)-1: add(result, ", ") - else: - result.add renderTree(p) - add(result, ')') - if n.sons[0].typ != nil: - result.add(": " & typeToString(n.sons[0].typ, prefer)) + assert sym != nil + result = sym.owner.name.s & '.' & sym.name.s + if sym.kind in routineKinds: + result.add '(' + var n = sym.typ.n + for i in countup(1, sonsLen(n) - 1): + let p = n.sons[i] + if p.kind == nkSym: + add(result, p.sym.name.s) + add(result, ": ") + add(result, typeToString(p.sym.typ, prefer)) + if i != sonsLen(n)-1: add(result, ", ") + else: + result.add renderTree(p) + add(result, ')') + if n.sons[0].typ != nil: + result.add(": " & typeToString(n.sons[0].typ, prefer)) result.add "[declared in " result.add(conf$sym.info) result.add "]" diff --git a/compiler/vm.nim b/compiler/vm.nim index c8784c3e7d..e2586e615b 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -65,15 +65,20 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = return stackTraceAux(c, x.next, x.comesFrom, recursionLimit-1) var info = c.debug[pc] - # we now use the same format as in system/except.nim - var s = substr(toFilename(c.config, info), 0) - # this 'substr' prevents a strange corruption. XXX This needs to be - # investigated eventually but first attempts to fix it broke everything - # see the araq-wip-fixed-writebarrier branch. + # we now use a format similar to the one in lib/system/excpt.nim + var s = "" + # todo: factor with quotedFilename + if optExcessiveStackTrace in c.config.globalOptions: + s = toFullPath(c.config, info) + else: + s = toFilename(c.config, info) var line = toLinenumber(info) + var col = toColumn(info) if line > 0: add(s, '(') add(s, $line) + add(s, ", ") + add(s, $(col + ColOffset)) add(s, ')') if x.prc != nil: for k in 1..max(1, 25-s.len): add(s, ' ') diff --git a/tests/errmsgs/m8794.nim b/tests/errmsgs/m8794.nim new file mode 100644 index 0000000000..12e61cf540 --- /dev/null +++ b/tests/errmsgs/m8794.nim @@ -0,0 +1,2 @@ +type Foo3* = object + a1: int diff --git a/tests/errmsgs/t8794.nim b/tests/errmsgs/t8794.nim new file mode 100644 index 0000000000..7f16a42fe1 --- /dev/null +++ b/tests/errmsgs/t8794.nim @@ -0,0 +1,39 @@ +discard """ + cmd: "nim check $options $file" + errormsg: "" + nimout: ''' +t8794.nim(39, 27) Error: undeclared field: 'a3' for type m8794.Foo3[declared in m8794.nim(1, 6)] +''' +""" + + + + + + + + + + + + +## line 20 + +## issue #8794 + +import m8794 + +when false: # pending https://github.com/nim-lang/Nim/pull/10091 add this + type Foo = object + a1: int + + discard Foo().a2 + +type Foo3b = Foo3 +var x2: Foo3b + +proc getFun[T](): T = + var a: T + a + +discard getFun[type(x2)]().a3 diff --git a/tests/errmsgs/t9768.nim b/tests/errmsgs/t9768.nim new file mode 100644 index 0000000000..18588c87c4 --- /dev/null +++ b/tests/errmsgs/t9768.nim @@ -0,0 +1,30 @@ +discard """ + errmsg: "unhandled exception:" + file: "system.nim" + nimout: ''' +stack trace: (most recent call last) +t9768.nim(28, 33) main +t9768.nim(23, 11) foo1 +''' +""" + + + + + + + + + + +## line 20 + +proc foo1(a: int): auto = + doAssert a < 4 + result = a * 2 + +proc main()= + static: + if foo1(1) > 0: discard foo1(foo1(2)) + +main() From fc7fcca9df305c02489bad4b969621cf9d8cb4f5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 8 Jan 2019 17:07:46 -0800 Subject: [PATCH 050/350] fix leftover comment from #9766 --- compiler/semcall.nim | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 7991640eac..3723d3fc1c 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -290,13 +290,11 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = let sym = n.sons[1].typ.sym var typeHint = "" if sym == nil: - #[ - Perhaps we're in a `compiles(foo.bar)` expression, or - in a concept, eg: - ExplainedConcept {.explain.} = concept o - o.foo is int - We coudl use: `(c.config $ n.sons[1].info)` to get more context. - ]# + # Perhaps we're in a `compiles(foo.bar)` expression, or + # in a concept, eg: + # ExplainedConcept {.explain.} = concept x + # x.foo is int + # We coudl use: `(c.config $ n.sons[1].info)` to get more context. discard else: typeHint = " for type " & getProcHeader(c.config, sym) From 6ce3949c8aa489d268f272829d03edab175bfc7a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 8 Jan 2019 18:37:25 -0800 Subject: [PATCH 051/350] add `isNamedTuple`; make $(1, 2) be (1, 2) instead of (Field0: 1, Field1: 2) which leaked implementation detail (#10070) * add `isNamedTuple`; make $(1, 2) be (1, 2) instead of leaking implementation detail (Field0: 1, Field1: 2) fixes this: #8670 (comment) /cc @alehander42 @Vindaar @mratsim * Note: isNamedTuple is useful in other places, eg #10010 (comment) * move isNamedTuple to helpers.nim to avoid exposing new symbol to system.nim * remove workaround in tests/vm/tissues.nim failing test now that #10218 was makes it work --- lib/pure/typetraits.nim | 3 +++ lib/system.nim | 21 +++++++++++---- lib/system/helpers.nim | 19 ++++++++++++- lib/system/helpers2.nim | 2 ++ tests/array/tarray.nim | 2 +- tests/ccgbugs/t5701.nim | 6 ++--- tests/collections/tcollections_to_string.nim | 6 ++--- .../errmsgs/tnested_generic_instantiation.nim | 6 +++++ .../tnested_generic_instantiation2.nim | 27 +++++++++++++++++++ tests/generics/treentranttypes.nim | 2 +- tests/metatype/tmetatype_issues.nim | 2 +- tests/metatype/ttypetraits2.nim | 15 +++++++++++ tests/system/tsystem_misc.nim | 11 ++++++++ tests/vm/tissues.nim | 14 +++++----- 14 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 tests/errmsgs/tnested_generic_instantiation2.nim diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 5f5bfdbd7a..a373a93704 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -10,7 +10,10 @@ ## This module defines compile-time reflection procs for ## working with types +include "system/helpers" # for `isNamedTuple` + export system.`$` +export isNamedTuple proc name*(t: typedesc): string {.magic: "TypeTrait".} ## Alias for system.`$`(t) since Nim v0.20.0. diff --git a/lib/system.nim b/lib/system.nim index 1d24019401..c68eba2e08 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2669,19 +2669,28 @@ proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime. ## echo "'+' for integers is available" discard +include "system/helpers" # for `lineInfoToString`, `isNamedTuple` + proc `$`*[T: tuple|object](x: T): string = ## generic ``$`` operator for tuples that is lifted from the components ## of `x`. Example: ## ## .. code-block:: nim - ## $(23, 45) == "(Field0: 23, Field1: 45)" + ## $(23, 45) == "(23, 45)" + ## $(a: 23, b: 45) == "(a: 23, b: 45)" ## $() == "()" result = "(" var firstElement = true + const isNamed = T is object or isNamedTuple(T) + when not isNamed: + var count = 0 for name, value in fieldPairs(x): if not firstElement: result.add(", ") - result.add(name) - result.add(": ") + when isNamed: + result.add(name) + result.add(": ") + else: + count.inc when compiles($value): when value isnot string and value isnot seq and compiles(value.isNil): if value.isNil: result.add "nil" @@ -2691,6 +2700,10 @@ proc `$`*[T: tuple|object](x: T): string = firstElement = false else: result.add("...") + when not isNamed: + if count == 1: + result.add(",") # $(1,) should print as the semantically legal (1,) + result.add(")") proc collectionToString[T](x: T, prefix, separator, suffix: string): string = @@ -3958,8 +3971,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = tags: [].} Hide(raiseAssert)(msg) -include "system/helpers" # for `lineInfoToString` - template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = const loc = $instantiationInfo(-1, true) bind instantiationInfo diff --git a/lib/system/helpers.nim b/lib/system/helpers.nim index 7b2b326790..a7e47915e3 100644 --- a/lib/system/helpers.nim +++ b/lib/system/helpers.nim @@ -6,6 +6,23 @@ proc lineInfoToString(file: string, line, column: int): string = file & "(" & $line & ", " & $column & ")" -proc `$`(info: type(instantiationInfo(0))): string = +type InstantiationInfo = tuple[filename: string, line: int, column: int] + +proc `$`(info: InstantiationInfo): string = # The +1 is needed here + # instead of overriding `$` (and changing its meaning), consider explicit name. lineInfoToString(info.fileName, info.line, info.column+1) + +proc isNamedTuple(T: type): bool = + ## return true for named tuples, false for any other type. + when T isnot tuple: result = false + else: + var t: T + for name, _ in t.fieldPairs: + when name == "Field0": + return compiles(t.Field0) + else: + return true + # empty tuple should be un-named, + # see https://github.com/nim-lang/Nim/issues/8861#issue-356631191 + return false diff --git a/lib/system/helpers2.nim b/lib/system/helpers2.nim index 1c9e7c068d..c67a2c2785 100644 --- a/lib/system/helpers2.nim +++ b/lib/system/helpers2.nim @@ -1,3 +1,5 @@ +# imported by other modules, unlike helpers.nim which is included + template formatErrorIndexBound*[T](i, a, b: T): string = "index out of bounds: (a:" & $a & ") <= (i:" & $i & ") <= (b:" & $b & ") " diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index 2a371b7889..f7c1dbf7f0 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -18,7 +18,7 @@ paper @[2, 3, 4]321 9.0 4.0 3 -@[(Field0: 1, Field1: 2), (Field0: 3, Field1: 5)] +@[(1, 2), (3, 5)] 2 @["a", "new one", "c"] @[1, 2, 3] diff --git a/tests/ccgbugs/t5701.nim b/tests/ccgbugs/t5701.nim index e69acbf311..ee6e484982 100644 --- a/tests/ccgbugs/t5701.nim +++ b/tests/ccgbugs/t5701.nim @@ -1,7 +1,7 @@ discard """ - output: '''(Field0: 1, Field1: 1) -(Field0: 2, Field1: 2) -(Field0: 3, Field1: 3) + output: '''(1, 1) +(2, 2) +(3, 3) ''' """ diff --git a/tests/collections/tcollections_to_string.nim b/tests/collections/tcollections_to_string.nim index 0c4f1e91c7..686b9916b6 100644 --- a/tests/collections/tcollections_to_string.nim +++ b/tests/collections/tcollections_to_string.nim @@ -9,9 +9,9 @@ import lists import critbits # Tests for tuples -doAssert $(1, 2, 3) == "(Field0: 1, Field1: 2, Field2: 3)" -doAssert $("1", "2", "3") == """(Field0: "1", Field1: "2", Field2: "3")""" -doAssert $('1', '2', '3') == """(Field0: '1', Field1: '2', Field2: '3')""" +doAssert $(1, 2, 3) == "(1, 2, 3)" +doAssert $("1", "2", "3") == """("1", "2", "3")""" +doAssert $('1', '2', '3') == """('1', '2', '3')""" # Tests for seqs doAssert $(@[1, 2, 3]) == "@[1, 2, 3]" diff --git a/tests/errmsgs/tnested_generic_instantiation.nim b/tests/errmsgs/tnested_generic_instantiation.nim index 6aea7cbcc3..77353605c6 100644 --- a/tests/errmsgs/tnested_generic_instantiation.nim +++ b/tests/errmsgs/tnested_generic_instantiation.nim @@ -17,3 +17,9 @@ converter toWrapped[T](value: T): Wrapped[T] = let result = Plain() discard $result + +proc foo[T2](a: Wrapped[T2]) = + # Error: generic instantiation too nested + discard $a + +foo(result) diff --git a/tests/errmsgs/tnested_generic_instantiation2.nim b/tests/errmsgs/tnested_generic_instantiation2.nim new file mode 100644 index 0000000000..d9bba15b08 --- /dev/null +++ b/tests/errmsgs/tnested_generic_instantiation2.nim @@ -0,0 +1,27 @@ +discard """ +errormsg: "generic instantiation too nested" +""" + +#[ +bug #4766 +see also: tnested_generic_instantiation.nim +]# + +proc toString*[T](x: T) = + for name, value in fieldPairs(x): + when compiles(toString(value)): + discard + toString(value) + +type + Plain = ref object + discard + + Wrapped[T] = object + value: T + +converter toWrapped[T](value: T): Wrapped[T] = + Wrapped[T](value: value) + +let result = Plain() +toString(result) diff --git a/tests/generics/treentranttypes.nim b/tests/generics/treentranttypes.nim index 2ef049ce2f..31fa252930 100644 --- a/tests/generics/treentranttypes.nim +++ b/tests/generics/treentranttypes.nim @@ -1,6 +1,6 @@ discard """ output: ''' -(Field0: 10, Field1: (Field0: "test", Field1: 1.2)) +(10, ("test", 1.2)) 3x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0], [2.0, 0.0, 5.0]] 2x3 Matrix [[0.0, 2.0, 3.0], [2.0, 0.0, 5.0]] diff --git a/tests/metatype/tmetatype_issues.nim b/tests/metatype/tmetatype_issues.nim index c5040f9bab..08d486de2e 100644 --- a/tests/metatype/tmetatype_issues.nim +++ b/tests/metatype/tmetatype_issues.nim @@ -1,7 +1,7 @@ discard """ output:''' void -(Field0: "string", Field1: "string") +("string", "string") 1 mod 7 @[2, 2, 2, 2, 2] impl 2 called diff --git a/tests/metatype/ttypetraits2.nim b/tests/metatype/ttypetraits2.nim index de80e10b17..a436da7ecc 100644 --- a/tests/metatype/ttypetraits2.nim +++ b/tests/metatype/ttypetraits2.nim @@ -1,3 +1,18 @@ # todo: merge with $nimc_D/tests/metatype/ttypetraits.nim (currently disabled) from typetraits import `$` # checks fix for https://github.com/c-blake/cligen/issues/84 + +import typetraits + +block: # isNamedTuple + type Foo1 = (a:1,).type + type Foo2 = (Field0:1,).type + type Foo3 = ().type + type Foo4 = object + + doAssert (a:1,).type.isNamedTuple + doAssert Foo1.isNamedTuple + doAssert Foo2.isNamedTuple + doAssert not Foo3.isNamedTuple + doAssert not Foo4.isNamedTuple + doAssert not (1,).type.isNamedTuple diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index f53a86e9a1..a20e6b3bf3 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -148,3 +148,14 @@ let a = @[1, 2, 3] # a.boundedOpenArray(1, 2).foo() # Works echo a.boundedOpenArray(1, 2).len # Internal compiler error + +block: # `$`*[T: tuple|object](x: T) + doAssert $(foo1:0, bar1:"a") == """(foo1: 0, bar1: "a")""" + doAssert $(foo1:0, ) == """(foo1: 0)""" + doAssert $(0, "a") == """(0, "a")""" + doAssert $(0, ) == "(0,)" + type Foo = object + x:int + x2:float + doAssert $Foo(x:2) == "(x: 2, x2: 0.0)" + doAssert $() == "()" diff --git a/tests/vm/tissues.nim b/tests/vm/tissues.nim index 021b902adf..063559d2e5 100644 --- a/tests/vm/tissues.nim +++ b/tests/vm/tissues.nim @@ -1,16 +1,14 @@ -discard """ - nimout: "(Field0: 2, Field1: 2, Field2: 2, Field3: 2)" -""" - import macros -block t9043: - proc foo[N: static[int]](dims: array[N, int])= +block t9043: # issue #9043 + proc foo[N: static[int]](dims: array[N, int]): string = const N1 = N const N2 = dims.len - static: echo (N, dims.len, N1, N2) + const ret = $(N, dims.len, N1, N2) + static: doAssert ret == $(N, dims.len, N1, N2) + ret - foo([1, 2]) + doAssert foo([1, 2]) == "(2, 2, 2, 2)" block t4952: proc doCheck(tree: NimNode) = From 258952832f312a25d0ab9771237a547297b336b8 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 9 Jan 2019 07:01:32 +0100 Subject: [PATCH 052/350] remove float128 from the manual, fixes #10213 [ci skip] --- doc/manual.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index f79811002d..09b9b4d784 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -480,7 +480,6 @@ The type suffixes are: ``'d`` float64 ``'f32`` float32 ``'f64`` float64 - ``'f128`` float128 ================= ========================= Floating point literals may also be in binary, octal or hexadecimal From 23c1ee982e2d3795001879a4527581f33875cd33 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 9 Jan 2019 00:46:44 -0800 Subject: [PATCH 053/350] add `alignTable`, `parseTableCells` to align/format a tab(etc) delimited table (#10182) * add compiler/unittest_light.nim for easy diffing: assertEquals and mismatch * fixup * add alignTable, parseTableCells --- compiler/asciitables.nim | 83 ++++++++++++++++++++++ compiler/unittest_light.nim | 37 ++++++++++ tests/compiler/nim.cfg | 7 ++ tests/compiler/tasciitables.nim | 109 +++++++++++++++++++++++++++++ tests/compiler/tunittest_light.nim | 55 +++++++++++++++ 5 files changed, 291 insertions(+) create mode 100644 compiler/asciitables.nim create mode 100644 compiler/unittest_light.nim create mode 100644 tests/compiler/nim.cfg create mode 100644 tests/compiler/tasciitables.nim create mode 100644 tests/compiler/tunittest_light.nim diff --git a/compiler/asciitables.nim b/compiler/asciitables.nim new file mode 100644 index 0000000000..c25d54bdea --- /dev/null +++ b/compiler/asciitables.nim @@ -0,0 +1,83 @@ +#[ +move to std/asciitables.nim once stable, or to a nimble paackage +once compiler can depend on nimble +]# + +type Cell* = object + text*: string + width*, row*, col*, ncols*, nrows*: int + +iterator parseTableCells*(s: string, delim = '\t'): Cell = + ## iterates over all cells in a `delim`-delimited `s`, after a 1st + ## pass that computes number of rows, columns, and width of each column. + var widths: seq[int] + var cell: Cell + template update() = + if widths.len<=cell.col: + widths.setLen cell.col+1 + widths[cell.col] = cell.width + else: + widths[cell.col] = max(widths[cell.col], cell.width) + cell.width = 0 + + for a in s: + case a + of '\n': + update() + cell.col = 0 + cell.row.inc + elif a == delim: + update() + cell.col.inc + else: + # todo: consider multi-width chars when porting to non-ascii implementation + cell.width.inc + if s.len > 0 and s[^1] != '\n': + update() + + cell.ncols = widths.len + cell.nrows = cell.row + 1 + cell.row = 0 + cell.col = 0 + cell.width = 0 + + template update2() = + cell.width = widths[cell.col] + yield cell + cell.text = "" + cell.width = 0 + cell.col.inc + + template finishRow() = + for col in cell.col.. 0 and s[^1] != '\n': + finishRow() + +proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string = + ## formats a `delim`-delimited `s` representing a table; each cell is aligned + ## to a width that's computed for each column; consecutive columns are + ## delimted by `sep`, and alignment space is filled using `fill`. + ## More customized formatting can be done by calling `parseTableCells` directly. + for cell in parseTableCells(s, delim): + result.add cell.text + for i in cell.text.len.. Date: Wed, 9 Jan 2019 10:46:23 +0100 Subject: [PATCH 054/350] epoll selector starts with reasonable fd set size (1024) and increases in powers of two when needed. This prevents the selector to allocate large amounts of memory at startup on systems with a high RLIMIT_NOFILE setting (#10194) --- lib/pure/ioselects/ioselectors_epoll.nim | 22 +++++++++++++++++++--- lib/pure/selectors.nim | 3 +++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 16d901ff07..ffd60120e5 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -53,6 +53,7 @@ when hasThreadSupport: SelectorImpl[T] = object epollFD: cint maxFD: int + numFD: int fds: ptr SharedArray[SelectorKey[T]] count: int Selector*[T] = ptr SelectorImpl[T] @@ -61,6 +62,7 @@ else: SelectorImpl[T] = object epollFD: cint maxFD: int + numFD: int fds: seq[SelectorKey[T]] count: int Selector*[T] = ref SelectorImpl[T] @@ -76,6 +78,8 @@ proc newSelector*[T](): Selector[T] = raiseOsError(osLastError()) var maxFD = int(a.rlim_max) doAssert(maxFD > 0) + # Start with a reasonable size, checkFd() will grow this on demand + const numFD = 1024 var epollFD = epoll_create(MAX_EPOLL_EVENTS) if epollFD < 0: @@ -85,14 +89,16 @@ proc newSelector*[T](): Selector[T] = result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) result.epollFD = epollFD result.maxFD = maxFD - result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.numFD = numFD + result.fds = allocSharedArray[SelectorKey[T]](numFD) else: result = Selector[T]() result.epollFD = epollFD result.maxFD = maxFD - result.fds = newSeq[SelectorKey[T]](maxFD) + result.numFD = numFD + result.fds = newSeq[SelectorKey[T]](numFD) - for i in 0 ..< maxFD: + for i in 0 ..< numFD: result.fds[i].ident = InvalidIdent proc close*[T](s: Selector[T]) = @@ -127,6 +133,16 @@ template checkFd(s, f) = # FD if there is too many. -- DP if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + if f >= s.numFD: + var numFD = s.numFD + while numFD <= f: numFD *= 2 + when hasThreadSupport: + s.fds = reallocSharedArray(s.fds, numFD) + else: + s.fds.setLen(numFD) + for i in s.numFD ..< numFD: + s.fds[i].ident = InvalidIdent + s.numFD = numFD proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index e4c2b21244..b9c834127b 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -239,6 +239,9 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) type From 11050d110461edc3646a66b8afefb7907187f78c Mon Sep 17 00:00:00 2001 From: Notkea Date: Wed, 9 Jan 2019 15:06:40 +0100 Subject: [PATCH 055/350] make Stream.{read,peek} procs public (#9806) Those are useful in generic code, and `proc write*[T](s: Stream, x: T)` was already public. --- lib/pure/streams.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index b0ac62525b..b5254b4f70 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -146,12 +146,12 @@ proc writeLine*(s: Stream, args: varargs[string, `$`]) = for str in args: write(s, str) write(s, "\n") -proc read[T](s: Stream, result: var T) = +proc read*[T](s: Stream, result: var T) = ## generic read procedure. Reads `result` from the stream `s`. if readData(s, addr(result), sizeof(T)) != sizeof(T): raise newEIO("cannot read from stream") -proc peek[T](s: Stream, result: var T) = +proc peek*[T](s: Stream, result: var T) = ## generic peek procedure. Peeks `result` from the stream `s`. if peekData(s, addr(result), sizeof(T)) != sizeof(T): raise newEIO("cannot read from stream") From 87232a38e4fc40a3acb154a78bf32f8ea5734d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Thu, 10 Jan 2019 08:49:57 +0100 Subject: [PATCH 056/350] ValueError now inherits from CatchableError (#10246) --- changelog.md | 4 +++- lib/system.nim | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 35d2adbd1c..87885aec96 100644 --- a/changelog.md +++ b/changelog.md @@ -30,7 +30,9 @@ - `osproc.execProcess` now also takes a `workingDir` parameter. -- `options.UnpackError` is no longer a ref type and inherits from `System.Defect` instead of `System.ValueError`. +- `options.UnpackError` is no longer a ref type and inherits from `system.Defect` instead of `system.ValueError`. + +- `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`. - nre's `RegexMatch.{captureBounds,captures}[]` no longer return `Option` or `nil`/`""`, respectivly. Use the newly added `n in p.captures` method to diff --git a/lib/system.nim b/lib/system.nim index c68eba2e08..8a35714a51 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -606,7 +606,7 @@ type ## Raised when assertion is proved wrong. ## ## Usually the result of using the `assert() template <#assert>`_. - ValueError* = object of Defect ## \ + ValueError* = object of CatchableError ## \ ## Raised for string and object conversion errors. KeyError* = object of ValueError ## \ ## Raised if a key cannot be found in a table. From da57c0ab99eadaee1aaecfab1688a9382e96edb9 Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 8 Jan 2019 17:04:07 +0100 Subject: [PATCH 057/350] remove `subexes` --- doc/lib.rst | 3 - lib/pure/subexes.nim | 406 -------------------------------------- tests/test_nimscript.nims | 1 - tools/kochdocs.nim | 1 - 4 files changed, 411 deletions(-) delete mode 100644 lib/pure/subexes.nim diff --git a/doc/lib.rst b/doc/lib.rst index a16bf2677b..569e3f845e 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -141,9 +141,6 @@ String handling Ropes can represent very long strings efficiently; especially concatenation is done in O(1) instead of O(n). -* `subexes `_ - This module implements advanced string substitution operations. - * `std/editdistance `_ This module contains an algorithm to compute the edit distance between two Unicode strings. diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim deleted file mode 100644 index 638e71f04d..0000000000 --- a/lib/pure/subexes.nim +++ /dev/null @@ -1,406 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nim support for `substitution expressions`:idx: (`subex`:idx:). -## -## .. include:: ../../doc/subexes.txt -## - -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! - -from strutils import parseInt, cmpIgnoreStyle, Digits -include "system/inclrtl" -import system/helpers2 - -proc findNormalized(x: string, inArray: openarray[string]): int = - var i = 0 - while i < high(inArray): - if cmpIgnoreStyle(x, inArray[i]) == 0: return i - inc(i, 2) # incrementing by 1 would probably lead to a - # security hole... - return -1 - -type - SubexError* = object of ValueError ## exception that is raised for - ## an invalid subex - -proc raiseInvalidFormat(msg: string) {.noinline.} = - raise newException(SubexError, "invalid format string: " & msg) - -type - FormatParser = object {.pure, final.} - when defined(js): - f: string # we rely on the '\0' terminator - # which JS's native string doesn't have - else: - f: cstring - num, i, lineLen: int - -template call(x: untyped): untyped = - p.i = i - x - i = p.i - -template callNoLineLenTracking(x: untyped): untyped = - let oldLineLen = p.lineLen - p.i = i - x - i = p.i - p.lineLen = oldLineLen - -proc getFormatArg(p: var FormatParser, a: openArray[string]): int = - const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} - var i = p.i - var f = p.f - case f[i] - of '#': - result = p.num - inc i - inc p.num - of '1'..'9', '-': - var j = 0 - var negative = f[i] == '-' - if negative: inc i - while f[i] in Digits: - j = j * 10 + ord(f[i]) - ord('0') - inc i - result = if not negative: j-1 else: a.len-j - of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': - var name = "" - while f[i] in PatternChars: - name.add(f[i]) - inc(i) - result = findNormalized(name, a)+1 - of '$': - inc(i) - call: - result = getFormatArg(p, a) - result = parseInt(a[result])-1 - else: - raiseInvalidFormat("'#', '$', number or identifier expected") - if result >=% a.len: raiseInvalidFormat(formatErrorIndexBound(result, a.len)) - p.i = i - -proc scanDollar(p: var FormatParser, a: openarray[string], s: var string) {. - noSideEffect.} - -proc emitChar(p: var FormatParser, x: var string, ch: char) {.inline.} = - x.add(ch) - if ch == '\L': p.lineLen = 0 - else: inc p.lineLen - -proc emitStrLinear(p: var FormatParser, x: var string, y: string) {.inline.} = - for ch in items(y): emitChar(p, x, ch) - -proc emitStr(p: var FormatParser, x: var string, y: string) {.inline.} = - x.add(y) - inc p.lineLen, y.len - -proc scanQuote(p: var FormatParser, x: var string, toAdd: bool) = - var i = p.i+1 - var f = p.f - while true: - if f[i] == '\'': - inc i - if f[i] != '\'': break - inc i - if toAdd: emitChar(p, x, '\'') - elif f[i] == '\0': raiseInvalidFormat("closing \"'\" expected") - else: - if toAdd: emitChar(p, x, f[i]) - inc i - p.i = i - -proc scanBranch(p: var FormatParser, a: openArray[string], - x: var string, choice: int) = - var i = p.i - var f = p.f - var c = 0 - var elsePart = i - var toAdd = choice == 0 - while true: - case f[i] - of ']': break - of '|': - inc i - elsePart = i - inc c - if toAdd: break - toAdd = choice == c - of '\'': - call: scanQuote(p, x, toAdd) - of '\0': raiseInvalidFormat("closing ']' expected") - else: - if toAdd: - if f[i] == '$': - inc i - call: scanDollar(p, a, x) - else: - emitChar(p, x, f[i]) - inc i - else: - inc i - if not toAdd and choice >= 0: - # evaluate 'else' part: - var last = i - i = elsePart - while true: - case f[i] - of '|', ']': break - of '\'': - call: scanQuote(p, x, true) - of '$': - inc i - call: scanDollar(p, a, x) - else: - emitChar(p, x, f[i]) - inc i - i = last - p.i = i+1 - -proc scanSlice(p: var FormatParser, a: openarray[string]): tuple[x, y: int] = - var slice = false - var i = p.i - var f = p.f - - if f[i] == '{': inc i - else: raiseInvalidFormat("'{' expected") - if f[i] == '.' and f[i+1] == '.': - inc i, 2 - slice = true - else: - call: result.x = getFormatArg(p, a) - if f[i] == '.' and f[i+1] == '.': - inc i, 2 - slice = true - if slice: - if f[i] != '}': - call: result.y = getFormatArg(p, a) - else: - result.y = high(a) - else: - result.y = result.x - if f[i] != '}': raiseInvalidFormat("'}' expected") - inc i - p.i = i - -proc scanDollar(p: var FormatParser, a: openarray[string], s: var string) = - var i = p.i - var f = p.f - case f[i] - of '$': - emitChar p, s, '$' - inc i - of '*': - for j in 0..a.high: emitStr p, s, a[j] - inc i - of '{': - call: - let (x, y) = scanSlice(p, a) - for j in x..y: emitStr p, s, a[j] - of '[': - inc i - var start = i - call: scanBranch(p, a, s, -1) - var x: int - if f[i] == '{': - inc i - call: x = getFormatArg(p, a) - if f[i] != '}': raiseInvalidFormat("'}' expected") - inc i - else: - call: x = getFormatArg(p, a) - var last = i - let choice = parseInt(a[x]) - i = start - call: scanBranch(p, a, s, choice) - i = last - of '\'': - var sep = "" - callNoLineLenTracking: scanQuote(p, sep, true) - if f[i] == '~': - # $' '~{1..3} - # insert space followed by 1..3 if not empty - inc i - call: - let (x, y) = scanSlice(p, a) - var L = 0 - for j in x..y: inc L, a[j].len - if L > 0: - emitStrLinear p, s, sep - for j in x..y: emitStr p, s, a[j] - else: - block StringJoin: - block OptionalLineLengthSpecifier: - var maxLen = 0 - case f[i] - of '0'..'9': - while f[i] in Digits: - maxLen = maxLen * 10 + ord(f[i]) - ord('0') - inc i - of '$': - # do not skip the '$' here for `getFormatArg`! - call: - maxLen = getFormatArg(p, a) - else: break OptionalLineLengthSpecifier - var indent = "" - case f[i] - of 'i': - inc i - callNoLineLenTracking: scanQuote(p, indent, true) - - call: - let (x, y) = scanSlice(p, a) - if maxLen < 1: emitStrLinear(p, s, indent) - var items = 1 - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - if items >= maxLen: - emitStrLinear p, s, indent - items = 0 - emitStr p, s, a[j] - inc items - of 'c': - inc i - callNoLineLenTracking: scanQuote(p, indent, true) - - call: - let (x, y) = scanSlice(p, a) - if p.lineLen + a[x].len > maxLen: emitStrLinear(p, s, indent) - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - if p.lineLen + a[j].len > maxLen: emitStrLinear(p, s, indent) - emitStr p, s, a[j] - - else: raiseInvalidFormat("unit 'c' (chars) or 'i' (items) expected") - break StringJoin - - call: - let (x, y) = scanSlice(p, a) - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - emitStr p, s, a[j] - else: - call: - var x = getFormatArg(p, a) - emitStr p, s, a[x] - p.i = i - - -type - Subex* = distinct string ## string that contains a substitution expression - -proc subex*(s: string): Subex = - ## constructs a *substitution expression* from `s`. Currently this performs - ## no syntax checking but this may change in later versions. - result = Subex(s) - -proc addf*(s: var string, formatstr: Subex, a: varargs[string, `$`]) {. - noSideEffect, rtl, extern: "nfrmtAddf".} = - ## The same as ``add(s, formatstr % a)``, but more efficient. - var p: FormatParser - p.f = formatstr.string - var i = 0 - while i < len(formatstr.string): - if p.f[i] == '$': - inc i - call: scanDollar(p, a, s) - else: - emitChar(p, s, p.f[i]) - inc(i) - -proc `%` *(formatstr: Subex, a: openarray[string]): string {.noSideEffect, - rtl, extern: "nfrmtFormatOpenArray".} = - ## The `substitution`:idx: operator performs string substitutions in - ## `formatstr` and returns a modified `formatstr`. This is often called - ## `string interpolation`:idx:. - ## - result = newStringOfCap(formatstr.string.len + a.len shl 4) - addf(result, formatstr, a) - -proc `%` *(formatstr: Subex, a: string): string {.noSideEffect, - rtl, extern: "nfrmtFormatSingleElem".} = - ## This is the same as ``formatstr % [a]``. - result = newStringOfCap(formatstr.string.len + a.len) - addf(result, formatstr, [a]) - -proc format*(formatstr: Subex, a: varargs[string, `$`]): string {.noSideEffect, - rtl, extern: "nfrmtFormatVarargs".} = - ## The `substitution`:idx: operator performs string substitutions in - ## `formatstr` and returns a modified `formatstr`. This is often called - ## `string interpolation`:idx:. - ## - result = newStringOfCap(formatstr.string.len + a.len shl 4) - addf(result, formatstr, a) - -{.pop.} - -when isMainModule: - from strutils import replace - - proc `%`(formatstr: string, a: openarray[string]): string = - result = newStringOfCap(formatstr.len + a.len shl 4) - addf(result, formatstr.Subex, a) - - proc `%`(formatstr: string, a: string): string = - result = newStringOfCap(formatstr.len + a.len) - addf(result, formatstr.Subex, [a]) - - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - - doAssert "$[abc|def]# $3 $# $#" % ["17", "b", "c"] == "def c b c" - doAssert "$[abc|def]# $3 $# $#" % ["1", "b", "c"] == "def c b c" - doAssert "$[abc|def]# $3 $# $#" % ["0", "b", "c"] == "abc c b c" - doAssert "$[abc|def|]# $3 $# $#" % ["17", "b", "c"] == " c b c" - - doAssert "$[abc|def|]# $3 $# $#" % ["-9", "b", "c"] == " c b c" - doAssert "$1($', '{2..})" % ["f", "a", "b"] == "f(a, b)" - - doAssert "$[$1($', '{2..})|''''|fg'$3']1" % ["7", "a", "b"] == "fg$3" - - doAssert "$[$#($', '{#..})|''''|$3]1" % ["0", "a", "b"] == "0(a, b)" - doAssert "$' '~{..}" % "" == "" - doAssert "$' '~{..}" % "P0" == " P0" - doAssert "${$1}" % "1" == "1" - doAssert "${$$-1} $$1" % "1" == "1 $1" - - doAssert(("$#($', '10c'\n '{#..})" % ["doAssert", "longishA", "longish"]).replace(" \n", "\n") == - """doAssert( - longishA, - longish)""") - - doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", - "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent(""" - type MyEnum* = enum - fieldA, fieldB, - FiledClkad, fieldD, - fieldE, longishFieldName""", 6)) - - doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" - - doAssert subex"$1 $[files|file|files]{1} copied" % ["1"] == "1 file copied" - - doAssert subex"$['''|'|''''|']']#" % "0" == "'|" - - doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ - "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent(""" - type - Enum = enum - fieldNameA, fieldNameB, fieldNameC, - fieldNameD""", 6)) diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index b9a6097c29..3efbb0a4c1 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -17,7 +17,6 @@ import parseutils import deques import sequtils import strutils -import subexes import tables import unicode import uri diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 19b0b3d78a..b455a517ab 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -195,7 +195,6 @@ lib/pure/collections/sequtils.nim lib/pure/collections/rtarrays.nim lib/pure/cookies.nim lib/pure/memfiles.nim -lib/pure/subexes.nim lib/pure/collections/critbits.nim lib/core/locks.nim lib/core/rlocks.nim From 8d9b093440939a890d9f7df1ffe037e90f6d7a6c Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 8 Jan 2019 17:19:34 +0100 Subject: [PATCH 058/350] remove `scgi` --- doc/lib.rst | 3 - lib/pure/scgi.nim | 295 --------------------------- tests/bind/tnicerrorforsymchoice.nim | 9 +- tools/kochdocs.nim | 1 - 4 files changed, 7 insertions(+), 301 deletions(-) delete mode 100644 lib/pure/scgi.nim diff --git a/doc/lib.rst b/doc/lib.rst index 569e3f845e..6b1af43e16 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -230,9 +230,6 @@ Internet Protocols and Support * `cgi `_ This module implements helpers for CGI applications. -* `scgi `_ - This module implements helpers for SCGI applications. - * `browsers `_ This module implements procs for opening URLs with the user's default browser. diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim deleted file mode 100644 index e36803823a..0000000000 --- a/lib/pure/scgi.nim +++ /dev/null @@ -1,295 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements helper procs for SCGI applications. Example: -## -## .. code-block:: Nim -## -## import strtabs, sockets, scgi -## -## var counter = 0 -## proc handleRequest(client: Socket, input: string, -## headers: StringTableRef): bool {.procvar.} = -## inc(counter) -## client.writeStatusOkTextContent() -## client.send("Hello for the $#th time." % $counter & "\c\L") -## return false # do not stop processing -## -## run(handleRequest) -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. -## -## **Warning:** This module only supports the old asynchronous interface. -## You may wish to use the `asynchttpserver `_ -## instead for web applications. - -include "system/inclrtl" - -import sockets, strutils, os, strtabs, asyncio - -type - ScgiError* = object of IOError ## the exception that is raised, if a SCGI error occurs - -proc raiseScgiError*(msg: string) {.noreturn.} = - ## raises an ScgiError exception with message `msg`. - var e: ref ScgiError - new(e) - e.msg = msg - raise e - -proc parseWord(inp: string, outp: var string, start: int): int = - result = start - while inp[result] != '\0': inc(result) - outp = substr(inp, start, result-1) - -proc parseHeaders(s: string, L: int): StringTableRef = - result = newStringTable() - var i = 0 - while i < L: - var key, val: string - i = parseWord(s, key, i)+1 - i = parseWord(s, val, i)+1 - result[key] = val - if s[i] == ',': inc(i) - else: raiseScgiError("',' after netstring expected") - -proc recvChar(s: Socket): char = - var c: char - if recv(s, addr(c), sizeof(c)) == sizeof(c): - result = c - -type - ScgiState* = object of RootObj ## SCGI state object - server: Socket - bufLen: int - client*: Socket ## the client socket to send data to - headers*: StringTableRef ## the parsed headers - input*: string ## the input buffer - - - # Async - - ClientMode = enum - ClientReadChar, ClientReadHeaders, ClientReadContent - - AsyncClient = ref object - c: AsyncSocket - mode: ClientMode - dataLen: int - headers: StringTableRef ## the parsed headers - input: string ## the input buffer - - AsyncScgiStateObj = object - handleRequest: proc (client: AsyncSocket, - input: string, - headers: StringTableRef) {.closure, gcsafe.} - asyncServer: AsyncSocket - disp: Dispatcher - AsyncScgiState* = ref AsyncScgiStateObj - -proc recvBuffer(s: var ScgiState, L: int) = - if L > s.bufLen: - s.bufLen = L - s.input = newString(L) - if L > 0 and recv(s.client, cstring(s.input), L) != L: - raiseScgiError("could not read all data") - setLen(s.input, L) - -proc open*(s: var ScgiState, port = Port(4000), address = "127.0.0.1", - reuseAddr = false) = - ## opens a connection. - s.bufLen = 4000 - s.input = newString(s.bufLen) # will be reused - - s.server = socket() - if s.server == invalidSocket: raiseOSError(osLastError()) - new(s.client) # Initialise s.client for `next` - if s.server == invalidSocket: raiseScgiError("could not open socket") - #s.server.connect(connectionName, port) - if reuseAddr: - s.server.setSockOpt(OptReuseAddr, true) - bindAddr(s.server, port, address) - listen(s.server) - -proc close*(s: var ScgiState) = - ## closes the connection. - s.server.close() - -proc next*(s: var ScgiState, timeout: int = -1): bool = - ## proceed to the first/next request. Waits ``timeout`` milliseconds for a - ## request, if ``timeout`` is `-1` then this function will never time out. - ## Returns `true` if a new request has been processed. - var rsocks = @[s.server] - if select(rsocks, timeout) == 1 and rsocks.len == 1: - new(s.client) - accept(s.server, s.client) - var L = 0 - while true: - var d = s.client.recvChar() - if d == '\0': - s.client.close() - return false - if d notin strutils.Digits: - if d != ':': raiseScgiError("':' after length expected") - break - L = L * 10 + ord(d) - ord('0') - recvBuffer(s, L+1) - s.headers = parseHeaders(s.input, L) - if s.headers.getOrDefault("SCGI") != "1": raiseScgiError("SCGI Version 1 expected") - L = parseInt(s.headers.getOrDefault("CONTENT_LENGTH")) - recvBuffer(s, L) - return true - -proc writeStatusOkTextContent*(c: Socket, contentType = "text/html") = - ## sends the following string to the socket `c`:: - ## - ## Status: 200 OK\r\LContent-Type: text/html\r\L\r\L - ## - ## You should send this before sending your HTML page, for example. - c.send("Status: 200 OK\r\L" & - "Content-Type: $1\r\L\r\L" % contentType) - -proc run*(handleRequest: proc (client: Socket, input: string, - headers: StringTableRef): bool {.nimcall,gcsafe.}, - port = Port(4000)) = - ## encapsulates the SCGI object and main loop. - var s: ScgiState - s.open(port) - var stop = false - while not stop: - if next(s): - stop = handleRequest(s.client, s.input, s.headers) - s.client.close() - s.close() - -# -- AsyncIO start - -proc recvBufferAsync(client: AsyncClient, L: int): ReadLineResult = - result = ReadPartialLine - var data = "" - if L < 1: - raiseScgiError("Cannot read negative or zero length: " & $L) - let ret = recvAsync(client.c, data, L) - if ret == 0 and data == "": - client.c.close() - return ReadDisconnected - if ret == -1: - return ReadNone # No more data available - client.input.add(data) - if ret == L: - return ReadFullLine - -proc checkCloseSocket(client: AsyncClient) = - if not client.c.isClosed: - if client.c.isSendDataBuffered: - client.c.setHandleWrite do (s: AsyncSocket): - if not s.isClosed and not s.isSendDataBuffered: - s.close() - s.delHandleWrite() - else: client.c.close() - -proc handleClientRead(client: AsyncClient, s: AsyncScgiState) = - case client.mode - of ClientReadChar: - while true: - var d = "" - let ret = client.c.recvAsync(d, 1) - if d == "" and ret == 0: - # Disconnected - client.c.close() - return - if ret == -1: - return # No more data available - if d[0] notin strutils.Digits: - if d[0] != ':': raiseScgiError("':' after length expected") - break - client.dataLen = client.dataLen * 10 + ord(d[0]) - ord('0') - client.mode = ClientReadHeaders - handleClientRead(client, s) # Allow progression - of ClientReadHeaders: - let ret = recvBufferAsync(client, (client.dataLen+1)-client.input.len) - case ret - of ReadFullLine: - client.headers = parseHeaders(client.input, client.input.len-1) - if client.headers.getOrDefault("SCGI") != "1": raiseScgiError("SCGI Version 1 expected") - client.input = "" # For next part - - let contentLen = parseInt(client.headers.getOrDefault("CONTENT_LENGTH")) - if contentLen > 0: - client.mode = ClientReadContent - else: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - of ReadPartialLine, ReadDisconnected, ReadNone: return - of ClientReadContent: - let L = parseInt(client.headers.getOrDefault("CONTENT_LENGTH")) - - client.input.len - if L > 0: - let ret = recvBufferAsync(client, L) - case ret - of ReadFullLine: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - of ReadPartialLine, ReadDisconnected, ReadNone: return - else: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - -proc handleAccept(sock: AsyncSocket, s: AsyncScgiState) = - var client: AsyncSocket - new(client) - accept(s.asyncServer, client) - var asyncClient = AsyncClient(c: client, mode: ClientReadChar, dataLen: 0, - headers: newStringTable(), input: "") - client.handleRead = - proc (sock: AsyncSocket) = - handleClientRead(asyncClient, s) - s.disp.register(client) - -proc open*(handleRequest: proc (client: AsyncSocket, - input: string, headers: StringTableRef) {. - closure, gcsafe.}, - port = Port(4000), address = "127.0.0.1", - reuseAddr = false): AsyncScgiState = - ## Creates an ``AsyncScgiState`` object which serves as a SCGI server. - ## - ## After the execution of ``handleRequest`` the client socket will be closed - ## automatically unless it has already been closed. - var cres: AsyncScgiState - new(cres) - cres.asyncServer = asyncSocket() - cres.asyncServer.handleAccept = proc (s: AsyncSocket) = handleAccept(s, cres) - if reuseAddr: - cres.asyncServer.setSockOpt(OptReuseAddr, true) - bindAddr(cres.asyncServer, port, address) - listen(cres.asyncServer) - cres.handleRequest = handleRequest - result = cres - -proc register*(d: Dispatcher, s: AsyncScgiState): Delegate {.discardable.} = - ## Registers ``s`` with dispatcher ``d``. - result = d.register(s.asyncServer) - s.disp = d - -proc close*(s: AsyncScgiState) = - ## Closes the ``AsyncScgiState``. - s.asyncServer.close() - -when false: - var counter = 0 - proc handleRequest(client: Socket, input: string, - headers: StringTableRef): bool {.procvar.} = - inc(counter) - client.writeStatusOkTextContent() - client.send("Hello for the $#th time." % $counter & "\c\L") - return false # do not stop processing - - run(handleRequest) - diff --git a/tests/bind/tnicerrorforsymchoice.nim b/tests/bind/tnicerrorforsymchoice.nim index f16b323de3..c069268056 100644 --- a/tests/bind/tnicerrorforsymchoice.nim +++ b/tests/bind/tnicerrorforsymchoice.nim @@ -1,10 +1,15 @@ discard """ errormsg: "type mismatch: got " - line: 18 + line: 23 """ +# Fake ScgiState objects, from now-deprecated scgi module +type + ScgiState* = object of RootObj ## SCGI state object + AsyncScgiState* = object of RootObj ## SCGI state object + #bug #442 -import scgi, sockets, asyncio, strtabs +import sockets, asyncio, strtabs proc handleSCGIRequest[TScgi: ScgiState | AsyncScgiState](s: TScgi) = discard proc handleSCGIRequest(client: AsyncSocket, headers: StringTableRef, diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index b455a517ab..9a502a2569 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -177,7 +177,6 @@ lib/pure/colors.nim lib/pure/mimetypes.nim lib/pure/json.nim lib/pure/base64.nim -lib/pure/scgi.nim lib/impure/nre.nim lib/impure/nre/private/util.nim lib/deprecated/pure/sockets.nim From 1a3763fe133690a4faf4300d2e2c48e7a9c06f2a Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 8 Jan 2019 17:26:58 +0100 Subject: [PATCH 059/350] remove `smtp` --- doc/lib.rst | 3 - lib/pure/smtp.nim | 253 ------------------------------------------ lib/pure/smtp.nim.cfg | 1 - tools/kochdocs.nim | 1 - 4 files changed, 258 deletions(-) delete mode 100644 lib/pure/smtp.nim delete mode 100644 lib/pure/smtp.nim.cfg diff --git a/doc/lib.rst b/doc/lib.rst index 6b1af43e16..fdc188ef21 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -238,9 +238,6 @@ Internet Protocols and Support This module implements a simple HTTP client which supports both synchronous and asynchronous retrieval of web pages. -* `smtp `_ - This module implement a simple SMTP client. - * `cookies `_ This module contains helper procs for parsing and generating cookies. diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index 5f4b09f801..0000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,253 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(useSsl = true, debug=true) -## smtpConn.connect("smtp.gmail.com", Port 465) -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with ``-d:ssl``. - -import net, strutils, strtabs, base64, os -import asyncnet, asyncdispatch - -export Port - -type - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - SmtpBase[SocketType] = ref object - sock: SocketType - debug: bool - - Smtp* = SmtpBase[Socket] - AsyncSmtp* = SmtpBase[AsyncSocket] - -proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = - if smtp.debug: - echo("C:" & cmd) - await smtp.sock.send(cmd) - -proc debugRecv(smtp: Smtp | AsyncSmtp): Future[TaintedString] {.multisync.} = - result = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & result.string) - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - type PSSLContext = ref object - let defaultSSLContext: PSSLContext = nil -else: - var defaultSSLContext {.threadvar.}: SSLContext - - proc getSSLContext(): SSLContext = - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) - result = defaultSSLContext - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openarray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for ``Message``. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newSmtp*(useSsl = false, debug=false, - sslContext: SSLContext = nil): Smtp = - ## Creates a new ``Smtp`` instance. - new result - result.debug = debug - result.sock = newSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc newAsyncSmtp*(useSsl = false, debug=false, - sslContext: SSLContext = nil): AsyncSmtp = - ## Creates a new ``AsyncSmtp`` instance. - new result - result.debug = debug - - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.debugSend("QUIT") - sendFut.callback = - proc () = - # TODO: Fix this in async procs. - raise newException(ReplyError, msg) - return retFuture - -proc checkReply(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = - var line = await smtp.debugRecv() - if not line.startswith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) - -proc connect*(smtp: Smtp | AsyncSmtp, - address: string, port: Port) {.multisync.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - await smtp.sock.connect(address, port) - - await smtp.checkReply("220") - await smtp.debugSend("HELO " & address & "\c\L") - await smtp.checkReply("250") - -proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.debugSend("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.debugSend(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.debugSend(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentification was successful. - -proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.multisync.} = - ## Sends ``msg`` from ``fromAddr`` to the addresses specified in ``toAddrs``. - ## Messages may be formed using ``createMessage`` by converting the - ## Message into a string. - - await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.debugSend("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.debugSend("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.debugSend(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - # To test with a real SMTP service, create a smtp.ini file, e.g.: - # username = "" - # password = "" - # smtphost = "smtp.gmail.com" - # port = 465 - # use_tls = true - # sender = "" - # recipient = "" - - import parsecfg - - proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) - - let - conf = loadConfig("smtp.ini") - msg = createMessage("Hello from Nim's SMTP!", - "Hello!\n Is this awesome or what?", @[conf["recipient"]]) - - assert conf["smtphost"] != "" - - proc async_test() {.async.} = - let client = newAsyncSmtp( - conf["use_tls"].parseBool, - debug=true - ) - await client.connect(conf["smtphost"], conf["port"].parseInt.Port) - await client.auth(conf["username"], conf["password"]) - await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) - await client.close() - echo "async email sent" - - proc sync_test() = - var smtpConn = newSmtp( - conf["use_tls"].parseBool, - debug=true - ) - smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) - smtpConn.auth(conf["username"], conf["password"]) - smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) - smtpConn.close() - echo "sync email sent" - - waitFor async_test() - sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de41..0000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 9a502a2569..32784e7521 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -167,7 +167,6 @@ lib/impure/db_sqlite.nim lib/impure/db_odbc.nim lib/pure/db_common.nim lib/pure/httpclient.nim -lib/pure/smtp.nim lib/pure/ropes.nim lib/pure/unidecode/unidecode.nim lib/pure/xmlparser.nim From be958d60c92bf052cc85ac80dd49b589edb95e64 Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 8 Jan 2019 17:29:54 +0100 Subject: [PATCH 060/350] remove `oids` --- doc/lib.rst | 6 --- lib/pure/oids.nim | 93 ---------------------------------------------- tools/kochdocs.nim | 1 - 3 files changed, 100 deletions(-) delete mode 100644 lib/pure/oids.nim diff --git a/doc/lib.rst b/doc/lib.rst index fdc188ef21..d6c296e8d4 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -372,12 +372,6 @@ Multimedia support Miscellaneous ------------- -* `oids `_ - An OID is a global ID that consists of a timestamp, - a unique counter and a random value. This combination should suffice to - produce a globally distributed unique ID. This implementation was extracted - from the Mongodb interface and it thus binary compatible with a Mongo OID. - * `endians `_ This module contains helpers that deal with different byte orders. diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim deleted file mode 100644 index d6369b5f9f..0000000000 --- a/lib/pure/oids.nim +++ /dev/null @@ -1,93 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nim OID support. An OID is a global ID that consists of a timestamp, -## a unique counter and a random value. This combination should suffice to -## produce a globally distributed unique ID. This implementation was extracted -## from the Mongodb interface and it thus binary compatible with a Mongo OID. -## -## This implementation calls ``math.randomize()`` for the first call of -## ``genOid``. - -import times, endians - -type - Oid* = object ## an OID - time: int32 ## - fuzz: int32 ## - count: int32 ## - -proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality - return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) - -proc hexbyte*(hex: char): int = - case hex - of '0'..'9': result = (ord(hex) - ord('0')) - of 'a'..'f': result = (ord(hex) - ord('a') + 10) - of 'A'..'F': result = (ord(hex) - ord('A') + 10) - else: discard - -proc parseOid*(str: cstring): Oid = - ## parses an OID. - var bytes = cast[cstring](addr(result.time)) - var i = 0 - while i < 12: - bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) - inc(i) - -proc oidToString*(oid: Oid, str: cstring) = - const hex = "0123456789abcdef" - # work around a compiler bug: - var str = str - var o = oid - var bytes = cast[cstring](addr(o)) - var i = 0 - while i < 12: - let b = bytes[i].ord - str[2 * i] = hex[(b and 0xF0) shr 4] - str[2 * i + 1] = hex[b and 0xF] - inc(i) - str[24] = '\0' - -proc `$`*(oid: Oid): string = - result = newString(24) - oidToString(oid, result) - -var - incr: int - fuzz: int32 - -proc genOid*(): Oid = - ## generates a new OID. - proc rand(): cint {.importc: "rand", header: "", nodecl.} - proc srand(seed: cint) {.importc: "srand", header: "", nodecl.} - - var t = getTime().toUnix.int32 - - var i = int32(atomicInc(incr)) - - if fuzz == 0: - # racy, but fine semantically: - srand(t) - fuzz = rand() - bigEndian32(addr result.time, addr(t)) - result.fuzz = fuzz - bigEndian32(addr result.count, addr(i)) - -proc generatedTime*(oid: Oid): Time = - ## returns the generated timestamp of the OID. - var tmp: int32 - var dummy = oid.time - bigEndian32(addr(tmp), addr(dummy)) - result = fromUnix(tmp) - -when not defined(testing) and isMainModule: - let xo = genOid() - echo xo.generatedTime diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 32784e7521..f2b09d380a 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -196,7 +196,6 @@ lib/pure/memfiles.nim lib/pure/collections/critbits.nim lib/core/locks.nim lib/core/rlocks.nim -lib/pure/oids.nim lib/pure/endians.nim lib/pure/uri.nim lib/pure/nimprof.nim From 44b4e289d656475df5f0123de86ccc89e366edcb Mon Sep 17 00:00:00 2001 From: cooldome Date: Thu, 10 Jan 2019 08:25:35 +0000 Subject: [PATCH 061/350] destructors: lift type bound operations for case and distinct objects (#10238) --- compiler/semasgn.nim | 46 +++++++++++++++++--- tests/destructor/tdestructor.nim | 73 ++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 27 deletions(-) diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 5d676dc76d..c1ecaf8a27 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -49,6 +49,14 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) = liftBodyAux(c, f.typ, body, x.dotField(f), y.dotField(f)) of nkNilLit: discard of nkRecCase: + if c.kind in {attachedSink, attachedAsgn, attachedDeepCopy}: + ## the value needs to be destroyed before we assign the selector + ## or the value is lost + let prevKind = c.kind + c.kind = attachedDestructor + liftBodyObj(c, n, body, x, y) + c.kind = prevKind + # copy the selector: liftBodyObj(c, n[0], body, x, y) # we need to generate a case statement: @@ -66,7 +74,6 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) = liftBodyObj(c, n[i].lastSon, branch.sons[L-1], x, y) caseStmt.add(branch) body.add(caseStmt) - localError(c.c.config, c.info, "cannot lift assignment operator to 'case' object") of nkRecList: for t in items(n): liftBodyObj(c, t, body, x, y) else: @@ -235,11 +242,12 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = discard considerOverloadedOp(c, t, body, x, y) else: defaultOp(c, t, body, x, y) - of tyObject, tyDistinct: + of tyObject: if not considerOverloadedOp(c, t, body, x, y): - if t.sons[0] != nil: - liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y) - if t.kind == tyObject: liftBodyObj(c, t.n, body, x, y) + liftBodyObj(c, t.n, body, x, y) + of tyDistinct: + if not considerOverloadedOp(c, t, body, x, y): + liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y) of tyTuple: liftBodyTup(c, t, body, x, y) of tyProc: @@ -279,8 +287,36 @@ proc addParam(procType: PType; param: PSym) = addSon(procType.n, newSymNode(param)) rawAddSon(procType, param.typ) +proc liftBodyDistinctType(c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = + assert typ.kind == tyDistinct + let baseType = typ[0] + case kind + of attachedAsgn: + if baseType.assignment == nil: + discard liftBody(c, baseType, kind, info) + typ.assignment = baseType.assignment + result = typ.assignment + of attachedSink: + if baseType.sink == nil: + discard liftBody(c, baseType, kind, info) + typ.sink = baseType.sink + result = typ.sink + of attachedDeepCopy: + if baseType.deepCopy == nil: + discard liftBody(c, baseType, kind, info) + typ.deepCopy = baseType.deepCopy + result = typ.deepCopy + of attachedDestructor: + if baseType.destructor == nil: + discard liftBody(c, baseType, kind, info) + typ.destructor = baseType.destructor + result = typ.destructor + proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = + if typ.kind == tyDistinct: + return liftBodyDistinctType(c, typ, kind, info) + var a: TLiftCtx a.info = info a.c = c diff --git a/tests/destructor/tdestructor.nim b/tests/destructor/tdestructor.nim index c9f1caf2da..09dce19ab7 100644 --- a/tests/destructor/tdestructor.nim +++ b/tests/destructor/tdestructor.nim @@ -7,21 +7,28 @@ mygeneric1 constructed mygeneric1 destroyed ---- mygeneric2 constructed -mygeneric2 destroyed myobj destroyed +mygeneric2 destroyed ---- mygeneric3 constructed mygeneric1 destroyed ---- -mygeneric1 destroyed +mydistinctObj constructed +myobj destroyed +mygeneric2 destroyed +------------------ +---- ---- myobj destroyed ----- ----- +mygeneric1 destroyed +myobj destroyed +myobj destroyed +myobj destroyed +--- +myobj destroyed +myobj destroyed myobj destroyed ''' - cmd: '''nim c --newruntime $file''' - disabled: "true" """ type @@ -29,6 +36,11 @@ type x, y: int p: pointer +proc `=destroy`(o: var TMyObj) = + if o.p != nil: dealloc o.p + echo "myobj destroyed" + +type TMyGeneric1[T] = object x: T @@ -36,37 +48,40 @@ type x: A y: B +proc `=destroy`(o: var TMyGeneric1[int]) = + echo "mygeneric1 destroyed" + +proc `=destroy`[A, B](o: var TMyGeneric2[A, B]) = + echo "mygeneric2 destroyed" + +type TMyGeneric3[A, B, C] = object x: A y: B z: C - TObjKind = enum A, B, C, D + TDistinctObjX = distinct TMyGeneric3[TMyObj, TMyGeneric2[int, int], int] + TDistinctObj = TDistinctObjX + + TObjKind = enum Z, A, B, C, D TCaseObj = object + z: TMyGeneric3[TMyObj, float, int] case kind: TObjKind + of Z: discard of A: x: TMyGeneric1[int] of B, C: y: TMyObj else: case innerKind: TObjKind + of Z: discard of A, B, C: p: TMyGeneric3[int, float, string] of D: q: TMyGeneric3[TMyObj, int, int] r: string -proc `=destroy`(o: var TMyObj) = - if o.p != nil: dealloc o.p - echo "myobj destroyed" - -proc `=destroy`(o: var TMyGeneric1[int]) = - echo "mygeneric1 destroyed" - -proc `=destroy`[A, B](o: var TMyGeneric2[A, B]) = - echo "mygeneric2 destroyed" - proc open: TMyObj = # allow for superfluous () result = (TMyObj(x: 1, y: 2, p: alloc(3))) @@ -95,6 +110,12 @@ proc mygeneric3 = echo "mygeneric3 constructed" +proc mydistinctObj = + var x = TMyGeneric3[TMyObj, TMyGeneric2[int, int], int]( + x: open(), y: TMyGeneric2[int, int](x: 5, y: 15), z: 20) + + echo "mydistinctObj constructed" + echo "----" myobj() @@ -107,9 +128,11 @@ mygeneric2[int](10) echo "----" mygeneric3() +echo "----" +mydistinctObj() + proc caseobj = block: - echo "----" var o1 = TCaseObj(kind: A, x: TMyGeneric1[int](x: 10)) block: @@ -121,10 +144,16 @@ proc caseobj = var o3 = TCaseObj(kind: D, innerKind: B, r: "test", p: TMyGeneric3[int, float, string](x: 10, y: 1.0, z: "test")) - block: - echo "----" - var o4 = TCaseObj(kind: D, innerKind: D, r: "test", - q: TMyGeneric3[TMyObj, int, int](x: open(), y: 1, z: 0)) +echo "------------------" caseobj() +proc caseobj_test_sink: TCaseObj = + # check that lifted sink can destroy case val correctly + result = TCaseObj(kind: D, innerKind: D, r: "test", + q: TMyGeneric3[TMyObj, int, int](x: open(), y: 1, z: 0)) + result = TCaseObj(kind: B, y: open()) + + +echo "---" +discard caseobj_test_sink() \ No newline at end of file From f55c8d4d5430b9bf35a5a1ca8e8fe678c52a963b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Thu, 10 Jan 2019 09:27:01 +0100 Subject: [PATCH 062/350] fixes #10251 (#10255) --- compiler/lookups.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 269447486d..8bc2634854 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -190,7 +190,7 @@ proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) = wrongRedefinition(c, info, sym.name.s, conflict.info) proc addDecl*(c: PContext, sym: PSym) = - let conflict = c.currentScope.addUniqueSym(sym) + let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true) if conflict != nil: wrongRedefinition(c, sym.info, sym.name.s, conflict.info) From 3ed5f83704f575c56dfd44a43535966a9b65c025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Thu, 10 Jan 2019 09:28:09 +0100 Subject: [PATCH 063/350] Fix exception tracking for system.open (#10253) --- lib/system.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 8a35714a51..dff1954021 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3237,14 +3237,15 @@ when not defined(JS): #and not defined(nimscript): proc open*(f: var File, filename: string, mode: FileMode = fmRead, bufSize: int = -1): bool {.tags: [], - benign.} + raises: [], benign.} ## Opens a file named `filename` with given `mode`. ## ## Default mode is readonly. Returns true iff the file could be opened. ## This throws no exception if the file could not be opened. proc open*(f: var File, filehandle: FileHandle, - mode: FileMode = fmRead): bool {.tags: [], benign.} + mode: FileMode = fmRead): bool {.tags: [], raises: [], + benign.} ## Creates a ``File`` from a `filehandle` with given `mode`. ## ## Default mode is readonly. Returns true iff the file could be opened. @@ -3253,7 +3254,7 @@ when not defined(JS): #and not defined(nimscript): mode: FileMode = fmRead, bufSize: int = -1): File = ## Opens a file named `filename` with given `mode`. ## - ## Default mode is readonly. Raises an ``IO`` exception if the file + ## Default mode is readonly. Raises an ``IOError`` if the file ## could not be opened. if not open(result, filename, mode, bufSize): sysFatal(IOError, "cannot open: ", filename) From f7d2f9c5f0844af8831d780ad2b289305c482ada Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 10 Jan 2019 09:31:18 +0100 Subject: [PATCH 064/350] update changelog --- changelog.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 87885aec96..da06f3daa9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,4 +1,5 @@ -## v0.20.0 - XX/XX/2018 +## v0.20.0 - XX/XX/2019 + ### Changes affecting backwards compatibility @@ -26,6 +27,7 @@ - `getImpl` on a `var` or `let` symbol will now return the full `IdentDefs` tree from the symbol declaration instead of just the initializer portion. + #### Breaking changes in the standard library - `osproc.execProcess` now also takes a `workingDir` parameter. @@ -57,6 +59,13 @@ - `isNil` is no longer false for undefined in the JavaScript backend: now it's true for both nil and undefined. Use `isNull` or `isUndefined` if you need exact equallity: `isNil` is consistent with `===`, `isNull` and `isUndefined` with `==`. +- several deprecated modules were removed: `ssl`, `matchers`, `httpserver`, + `unsigned`, `actors`, `parseurl` + +- several poorly documented and not used modules were moved to graveyard + (they are available as Nimble packages): `subexes`, `scgi`, `smtp`, `oids` + + #### Breaking changes in the compiler - The compiler now implements the "generic symbol prepass" for `when` statements @@ -97,13 +106,16 @@ proc enumToString*(enums: openArray[enum]): string = is instantiation of generic proc symbol. - Added the parameter ``isSorted`` for the ``sequtils.deduplicate`` proc. + - There is a new stdlib module `std/diff` to compute the famous "diff" of two texts by line. - Added `os.relativePath`. + - Added `parseopt.remainingArgs`. + - Added `os.getCurrentCompilerExe` (implmented as `getAppFilename` at CT), - can be used to retrive the currently executing compiler. + can be used to retrieve the currently executing compiler. ### Library changes @@ -128,10 +140,12 @@ proc enumToString*(enums: openArray[enum]): string = - `os.joinPath` and `os.normalizePath` handle edge cases like ``"a/b/../../.."`` differently. +- `securehash` is moved to `lib/deprecated` + ### Language additions -- Vm suport for float32<->int32 and float64<->int64 casts was added. +- Vm support for float32<->int32 and float64<->int64 casts was added. - There is a new pragma block `noSideEffect` that works like the `gcsafe` pragma block. - added os.getCurrentProcessId() @@ -140,6 +154,7 @@ proc enumToString*(enums: openArray[enum]): string = pragmas for further analysis by macros - Custom pragmas are now supported for `var` and `let` symbols. + ### Language changes - The standard extension for SCF (source code filters) files was changed from @@ -161,7 +176,9 @@ proc enumToString*(enums: openArray[enum]): string = - `nimpretty`: --backup now defaults to `off` instead of `on` and the flag was un-documented; use `git` instead of relying on backup files. + ### Compiler changes - The deprecated `fmod` proc is now unavailable on the VM'. + ### Bugfixes From d2b95cb476f3b232862c2e4d4fe6fb67daa01988 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jan 2019 00:35:37 -0800 Subject: [PATCH 065/350] fixes #10039 : CI now runs buildTools (eg, nimfind wasn't being compiled before); refactoring (#10242) --- koch.nim | 56 ++++++++++++++++++++++++++++------------------ tools/kochdocs.nim | 6 ----- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/koch.nim b/koch.nim index afdce30837..845dffadac 100644 --- a/koch.nim +++ b/koch.nim @@ -26,7 +26,6 @@ import import tools / kochdocs const VersionAsString = system.NimVersion -const env_NIM_COMPILE_TO_CPP = "NIM_COMPILE_TO_CPP" const HelpText = """ @@ -47,6 +46,8 @@ Possible Commands: boot [options] bootstraps with given command line options distrohelper [bindir] helper for distro packagers tools builds Nim related tools + toolsNoNimble builds Nim related tools (except nimble) + doesn't require network connectivity nimble builds the Nimble tool Boot options: -d:release produce a release version of the compiler @@ -73,6 +74,11 @@ Web options: build the official docs, use UA-48159761-1 """ +let kochExe* = os.getAppFilename() + +proc kochExec*(cmd: string) = + exec kochExe.quoteShell & " " & cmd + template withDir(dir, body) = let old = getCurrentDir() try: @@ -123,9 +129,6 @@ proc bundleNimbleExe(latest: bool) = # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", options = "-d:release --nilseqs:on") -proc buildNimfind() = - nimCompile("tools/nimfind.nim", options = "-d:release") - proc buildNimble(latest: bool) = # old installations created nim/nimblepkg/*.nim files. We remove these # here so that it cannot cause problems (nimble bug #306): @@ -199,13 +202,12 @@ proc buildTool(toolname, args: string) = nimexec("cc $# $#" % [args, toolname]) copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe) -proc buildTools(latest: bool) = +proc buildTools() = bundleNimsuggest() nimCompile("tools/nimgrep.nim", options = "-d:release") when defined(windows): buildVccTool() nimCompile("nimpretty/nimpretty.nim", options = "-d:release") - buildNimble(latest) - buildNimfind() + nimCompile("tools/nimfind.nim", options = "-d:release") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest) @@ -274,7 +276,7 @@ proc boot(args: string) = var output = "compiler" / "nim".exe var finalDest = "bin" / "nim".exe # default to use the 'c' command: - let defaultCommand = if getEnv(env_NIM_COMPILE_TO_CPP, "false") == "true": "cpp" else: "c" + let defaultCommand = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "boot: defaultCommand: ", defaultCommand, " bootOptions: ", bootOptions let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & @@ -440,32 +442,38 @@ proc runCI(cmd: string) = # todo: implement `execWithEnv` exec("env NIM_COMPILE_TO_CPP=false $1 boot" % kochExe.quoteShell) kochExec "boot -d:release" + + ## build nimble early on to enable remainder to depend on it if needed kochExec "nimble" - exec "nim e tests/test_nimscript.nims" when false: for pkg in "zip opengl sdl1 jester@#head niminst".split: exec "nimble install -y" & pkg + buildTools() # altenatively, kochExec "tools --toolsNoNimble" + + ## run tests + exec "nim e tests/test_nimscript.nims" when defined(windows): # note: will be over-written below exec "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester" - when false: - kochExec "csource" - kochExec "zip" - # main bottleneck: runs all main tests + # main bottleneck here exec "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines" + exec "nim c -r nimdoc/tester" - - nimCompile "nimpretty/nimpretty.nim" exec "nim c -r nimpretty/tester.nim" + when defined(posix): + exec "nim c -r nimsuggest/tester" + ## remaining actions when defined(posix): kochExec "docs --git.commit:devel" kochExec "csource" - kochExec "nimsuggest" - exec "nim c -r nimsuggest/tester" + elif defined(windows): + when false: + kochExec "csource" + kochExec "zip" proc pushCsources() = if not dirExists("../csources/.git"): @@ -550,6 +558,10 @@ when isMainModule: var op = initOptParser() var latest = false var stable = false + template isLatest(): bool = + if stable: false + else: + existsDir(".git") or latest while true: op.next() case op.kind @@ -580,13 +592,13 @@ when isMainModule: of "temp": temp(op.cmdLineRest) of "xtemp": xtemp(op.cmdLineRest) of "wintools": bundleWinTools() - of "nimble": - if stable: buildNimble(false) - else: buildNimble(existsDir(".git") or latest) + of "nimble": buildNimble(isLatest()) of "nimsuggest": bundleNimsuggest() + of "toolsnonimble": + buildTools() of "tools": - if stable: buildTools(false) - else: buildTools(existsDir(".git") or latest) + buildTools() + buildNimble(isLatest()) of "pushcsource", "pushcsources": pushCsources() of "valgrind": valgrind(op.cmdLineRest) else: showHelp() diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 19b0b3d78a..6741fcf5d8 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -51,12 +51,6 @@ proc execCleanPath*(cmd: string, if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) -let kochExe* = os.getAppFilename() - # note: assumes `kochdocs` is only used by koch.nim - -proc kochExec*(cmd: string) = - exec kochExe.quoteShell & " " & cmd - proc nimexec*(cmd: string) = # Consider using `nimCompile` instead exec findNim() & " " & cmd From b3435d22dcb262371319f045aacb3a5d80d7f8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Thu, 10 Jan 2019 10:56:12 +0100 Subject: [PATCH 066/350] Times cosmetic changes (#10237) * Add more Date wrappers to jscore * Times cosmetic changes - Improved docs - Code wrapped at 80 chars - Formatting fixes using nimpretty - Remove some old deprecated procs --- lib/js/jscore.nim | 12 +- lib/pure/times.nim | 697 ++++++++++++++++++++++++--------------------- 2 files changed, 382 insertions(+), 327 deletions(-) diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index bf64b0794c..2e2bd2402a 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -73,7 +73,7 @@ proc parse*(d: DateLib, s: cstring): int {.importcpp.} proc newDate*(): DateTime {. importcpp: "new Date()".} -proc newDate*(date: int|string): DateTime {. +proc newDate*(date: int|int64|string): DateTime {. importcpp: "new Date(#)".} proc newDate*(year, month, day, hours, minutes, @@ -90,6 +90,16 @@ proc getSeconds*(d: DateTime): int {.importcpp.} proc getYear*(d: DateTime): int {.importcpp.} proc getTime*(d: DateTime): int {.importcpp.} proc toString*(d: DateTime): cstring {.importcpp.} +proc getUTCDate*(d: DateTime): int {.importcpp.} +proc getUTCFullYear*(d: DateTime): int {.importcpp.} +proc getUTCHours*(d: DateTime): int {.importcpp.} +proc getUTCMilliseconds*(d: DateTime): int {.importcpp.} +proc getUTCMinutes*(d: DateTime): int {.importcpp.} +proc getUTCMonth*(d: DateTime): int {.importcpp.} +proc getUTCSeconds*(d: DateTime): int {.importcpp.} +proc getUTCDay*(d: DateTime): int {.importcpp.} +proc getTimezoneOffset*(d: DateTime): int {.importcpp.} +proc setFullYear*(d: DateTime, year: int) {.importcpp.} #JSON library proc stringify*(l: JsonLib, s: JsRoot): cstring {.importcpp.} diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 0a06d5f9f0..8fe04a4b98 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1,35 +1,41 @@ # # # Nim's Runtime Library -# (c) Copyright 2017 Nim contributors +# (c) Copyright 2018 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ##[ - This module contains routines and types for dealing with time using a proleptic Gregorian calendar. - It's also available for the `JavaScript target `_. + The ``times`` module contains routines and types for dealing with time using + the `proleptic Gregorian calendar`_. + It's also available for the + `JavaScript target `_. - Although the types use nanosecond time resolution, the underlying resolution used by ``getTime()`` - depends on the platform and backend (JS is limited to millisecond precision). + Although the ``times`` module support nanosecond time resolution, the + resolution used by ``getTime()`` depends on the platform and backend + (JS is limited to millisecond precision). Examples: .. code-block:: nim - import times, os + # Simple benchmarking let time = cpuTime() - - sleep(100) # replace this with something to be timed + sleep(100) # Replace this with something to be timed echo "Time taken: ", cpuTime() - time - echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") - echo "Using predefined formats: ", getClockStr(), " ", getDateStr() + # Current date & time + let now1 = now() # Current timestamp as a DateTime in local time + let now2 = now().utc # Current timestamp as a DateTime in UTC + let now3 = getTime() # Current timestamp as a Time - echo "cpuTime() float value: ", cpuTime() - echo "An hour from now : ", now() + 1.hours - echo "An hour from (UTC) now: ", getTime().utc + initDuration(hours = 1) + # Arithmetic using Duration + echo "One hour from now : ", now() + initDuration(hours = 1) + # Arithmetic using TimeInterval + echo "One year from now : ", now() + 1.years + echo "One month from now : ", now() + 1.months Parsing and Formatting Dates ---------------------------- @@ -97,14 +103,14 @@ | ``24 AD -> 24`` | ``24 BC -> -23`` | ``12345 AD -> 12345`` - ``z`` Displays the timezone offset from UTC. | ``GMT+7 -> +7`` - | ``GMT-5 -> -5`` - ``zz`` Same as above but with leading 0. | ``GMT+7 -> +07`` - | ``GMT-5 -> -05`` - ``zzz`` Same as above but with ``:mm`` where *mm* represents minutes. | ``GMT+7 -> +07:00`` - | ``GMT-5 -> -05:00`` - ``zzzz`` Same as above but with ``:ss`` where *ss* represents seconds. | ``GMT+7 -> +07:00:00`` - | ``GMT-5 -> -05:00:00`` + ``z`` Displays the timezone offset from UTC. | ``UTC+7 -> +7`` + | ``UTC-5 -> -5`` + ``zz`` Same as above but with leading 0. | ``UTC+7 -> +07`` + | ``UTC-5 -> -05`` + ``zzz`` Same as above but with ``:mm`` where *mm* represents minutes. | ``UTC+7 -> +07:00`` + | ``UTC-5 -> -05:00`` + ``zzzz`` Same as above but with ``:ss`` where *ss* represents seconds. | ``UTC+7 -> +07:00:00`` + | ``UTC-5 -> -05:00:00`` ``g`` Era: AD or BC | ``300 AD -> AD`` | ``300 BC -> BC`` ``fff`` Milliseconds display | ``1000000 nanoseconds -> 1`` @@ -117,49 +123,109 @@ inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ``,``. A literal ``'`` can be specified with ``''``. - However you don't need to necessarily separate format patterns, a + However you don't need to necessarily separate format patterns, an unambiguous format string like ``yyyyMMddhhmmss`` is valid too (although only for years in the range 1..9999). + + Duration vs TimeInterval + ---------------------------- + The ``times`` module exports two similiar types that are both used to + represent some amount of time: ``Duration`` and ``TimeInterval``. + This section explains how they differ and when one should be prefered over the + other (short answer: use ``Duration`` unless support for months and years is + needed). + + Duration + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + A ``Duration`` represents a duration of time stored as seconds and + nanoseconds. A ``Duration`` is always fully normalized, so +``initDuration(hours = 1)`` and ``initDuration(minutes = 60)`` are equivilant. + + Arithmetics with a ``Duration`` is very fast, especially when used with the + ``Time`` type, since it only involves basic arithmetic. Because ``Duration`` + is more performant and easier to understand it should generally prefered. + + TimeInterval + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + A ``TimeInterval`` represents some amount of time expressed in calendar + units, for example "1 year and 2 days". Since some units cannot be + normalized (the length of a year is different for leap years for example), + the ``TimeInterval`` type uses seperate fields for every unit. The + ``TimeInterval``'s returned form the this module generally don't normalize + **anything**, so even units that could be normalized (like seconds, + milliseconds and so on) are left untouched. + + Arithmetics with a ``TimeInterval`` can be very slow, because it requires + timezone information. + + Since it's slower and more complex, the ``TimeInterval`` type should be + avoided unless the program explicitly needs the features it offers that + ``Duration`` doesn't have. + + How long is a day? + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + It should be especially noted that the handling of days differs between + ``TimeInterval`` and ``Duration``. The ``Duration`` type always treats a day + as exactly 86400 seconds. For ``TimeInterval``, it's more complex. + + As an example, consider the amount of time between these two timestamps, both + in the same timezone: + + - 2018-03-25T12:00+02:00 + - 2018-03-26T12:00+01:00 + + If only the date & time is considered, it appears that exatly one day has + passed. However, the UTC offsets are different, which means that the + UTC offset was changed somewhere between. This happens twice each year for + timezones that use daylight savings time. Because of this change, the amount + of time that has passed is actually 25 hours. + + The ``TimeInterval`` type uses calendar units, and will say that exactly one + day has passed. The ``Duration`` type on the other hand normalizes everything + to seconds, and will therefore say that 90000 seconds has passed, which is + the same as 25 hours. ]## -import - strutils, algorithm, math, options, strformat +import strutils, algorithm, math, options, strformat include "system/inclrtl" -# This is really bad, but overflow checks are broken badly for -# ints on the JS backend. See #6752. when defined(JS): + import jscore + + # This is really bad, but overflow checks are broken badly for + # ints on the JS backend. See #6752. {.push overflowChecks: off.} proc `*`(a, b: int64): int64 = - system.`* `(a, b) + system.`*`(a, b) proc `*`(a, b: int): int = - system.`* `(a, b) + system.`*`(a, b) proc `+`(a, b: int64): int64 = - system.`+ `(a, b) + system.`+`(a, b) proc `+`(a, b: int): int = - system.`+ `(a, b) + system.`+`(a, b) proc `-`(a, b: int64): int64 = - system.`- `(a, b) + system.`-`(a, b) proc `-`(a, b: int): int = - system.`- `(a, b) + system.`-`(a, b) proc inc(a: var int, b: int) = system.inc(a, b) proc inc(a: var int64, b: int) = system.inc(a, b) {.pop.} -when defined(posix): +elif defined(posix): import posix type CTime = posix.Time var realTimeClockId {.importc: "CLOCK_REALTIME", header: "".}: Clockid - cpuClockId {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "".}: Clockid + cpuClockId + {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "".}: Clockid - proc gettimeofday(tp: var Timeval, unused: pointer = nil) {. - importc: "gettimeofday", header: "".} + proc gettimeofday(tp: var Timeval, unused: pointer = nil) + {.importc: "gettimeofday", header: "".} when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "".}: int @@ -177,9 +243,24 @@ elif defined(windows): # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "".}: int + type + Tm {.importc: "struct tm", header: "", final, pure.} = object + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + + proc localtime(a1: var CTime): ptr Tm {.importc, header: "".} + type - Month* = enum ## Represents a month. Note that the enum starts at ``1``, so ``ord(month)`` will give - ## the month number in the range ``[1..12]``. + Month* = enum ## Represents a month. Note that the enum starts at ``1``, + ## so ``ord(month)`` will give the month number in the + ## range ``1..12``. mJan = (1, "January") mFeb = "February" mMar = "March" @@ -213,13 +294,19 @@ type seconds: int64 nanosecond: NanosecondRange - DateTime* = object of RootObj ## Represents a time in different parts. - ## Although this type can represent leap - ## seconds, they are generally not supported - ## in this module. They are not ignored, - ## but the ``DateTime``'s returned by - ## procedures in this module will never have - ## a leap second. + DateTime* = object of RootObj ## \ + ## Represents a time in different parts. Although this type can represent + ## leap seconds, they are generally not supported in this module. They are + ## not ignored, but the ``DateTime``'s returned by procedures in this + ## module will never have a leap second. + ## + ## **Warning**: even though the fields of ``DateTime`` are exported, + ## they should never be mutated directly. Doing so is unsafe and will + ## result in the ``DateTime`` ending up in an invalid state. + ## + ## Instead of mutating the fields directly, use the ``Duration`` + ## and ``TimeInterval`` types for arithmetic and use the ``initDateTime`` + ## procedure for changing a specific field. nanosecond*: NanosecondRange ## The number of nanoseconds after the second, ## in the range 0 to 999_999_999. second*: SecondRange ## The number of seconds after the minute, @@ -230,27 +317,48 @@ type hour*: HourRange ## The number of hours past midnight, ## in the range 0 to 23. monthday*: MonthdayRange ## The day of the month, in the range 1 to 31. - month*: Month ## The current month. - year*: int ## The current year, using astronomical year numbering - ## (meaning that before year 1 is year 0, then year -1 and so on). - weekday*: WeekDay ## The current day of the week. + month*: Month ## The month. + year*: int ## The year, using astronomical year numbering + ## (meaning that before year 1 is year 0, + ## then year -1 and so on). + weekday*: WeekDay ## The day of the week. yearday*: YeardayRange ## The number of days since January 1, ## in the range 0 to 365. isDst*: bool ## Determines whether DST is in effect. ## Always false for the JavaScript backend. - timezone*: Timezone ## The timezone represented as an implementation of ``Timezone``. - utcOffset*: int ## The offset in seconds west of UTC, including any offset due to DST. - ## Note that the sign of this number is the opposite - ## of the one in a formatted offset string like ``+01:00`` - ## (which would be parsed into the UTC offset ``-3600``). + timezone*: Timezone ## The timezone represented as an implementation + ## of ``Timezone``. + utcOffset*: int ## The offset in seconds west of UTC, including + ## any offset due to DST. Note that the sign of + ## this number is the opposite of the one in a + ## formatted offset string like ``+01:00`` (which + ## would be equivalent to the UTC offset + ## ``-3600``). - TimeInterval* = object ## Represents a non-fixed duration of time. Can be used to add and subtract - ## non-fixed time units from a ``DateTime`` or ``Time``. - ## ``TimeInterval`` doesn't represent a fixed duration of time, - ## since the duration of some units depend on the context (e.g a year - ## can be either 365 or 366 days long). The non-fixed time units are years, - ## months and days. + Duration* = object ## Represents a fixed duration of time, meaning a duration + ## that has constant length independent of the context. + seconds: int64 + nanosecond: NanosecondRange + TimeUnit* = enum ## Different units of time. + Nanoseconds, Microseconds, Milliseconds, Seconds, Minutes, Hours, Days, + Weeks, Months, Years + + FixedTimeUnit* = range[Nanoseconds..Weeks] ## \ + ## Subrange of ``TimeUnit`` that only includes units of fixed duration. + ## These are the units that can be represented by a ``Duration``. + + TimeInterval* = object ## \ + ## Represents a non-fixed duration of time. Can be used to add and + ## subtract non-fixed time units from a ``DateTime`` or ``Time``. + ## Note that ``TimeInterval`` doesn't represent a fixed duration of time, + ## since the duration of some units depend on the context (e.g a year + ## can be either 365 or 366 days long). The non-fixed time units are + ## years, months, days and week. + ## + ## Note that ``TimeInterval``'s returned from the ``times`` module are + ## never normalized. If you want to normalize a time unit, ``Duration`` + ## should be used instead. nanoseconds*: int ## The number of nanoseconds microseconds*: int ## The number of microseconds milliseconds*: int ## The number of milliseconds @@ -262,19 +370,6 @@ type months*: int ## The number of months years*: int ## The number of years - Duration* = object ## Represents a fixed duration of time. - ## Uses the same time resolution as ``Time``. - ## This type should be prefered over ``TimeInterval`` unless - ## non-static time units is needed. - seconds: int64 - nanosecond: NanosecondRange - - TimeUnit* = enum ## Different units of time. - Nanoseconds, Microseconds, Milliseconds, Seconds, Minutes, Hours, Days, Weeks, Months, Years - - FixedTimeUnit* = range[Nanoseconds..Weeks] ## Subrange of ``TimeUnit`` that only includes units of fixed duration. - ## These are the units that can be represented by a ``Duration``. - Timezone* = ref object ## \ ## Timezone interface for supporting ``DateTime``'s of arbritary ## timezones. The ``times`` module only supplies implementations for the @@ -317,8 +412,10 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] -proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T {.inline.} = +proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T + {.inline.} = ## Convert a quantity of some duration unit to another duration unit. + ## This proc only deals with integers, so the result might be truncated. runnableExamples: doAssert convert(Days, Hours, 2) == 48 doAssert convert(Days, Weeks, 13) == 1 # Truncated @@ -340,21 +437,41 @@ proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = result.nanosecond = nanosecond.int # Forward declarations -proc utcTzInfo(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZonedTimeFromTime(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc utcTzInfo(time: Time): ZonedTime + {.tags: [], raises: [], benign.} +proc localZonedTimeFromTime(time: Time): ZonedTime + {.tags: [], raises: [], benign.} +proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime + {.tags: [], raises: [], benign.} proc initTime*(unix: int64, nanosecond: NanosecondRange): Time - {.tags: [], raises: [], benign noSideEffect.} - -proc initDuration*(nanoseconds, microseconds, milliseconds, - seconds, minutes, hours, days, weeks: int64 = 0): Duration - {.tags: [], raises: [], benign noSideEffect.} + {.tags: [], raises: [], benign, noSideEffect.} proc nanosecond*(time: Time): NanosecondRange = ## Get the fractional part of a ``Time`` as the number ## of nanoseconds of the second. time.nanosecond +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration = + ## Create a new duration. + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.milliseconds == 1 + doAssert dur.seconds == 1 + + let seconds = convert(Weeks, Seconds, weeks) + + convert(Days, Seconds, days) + + convert(Minutes, Seconds, minutes) + + convert(Hours, Seconds, hours) + + convert(Seconds, Seconds, seconds) + + convert(Milliseconds, Seconds, milliseconds) + + convert(Microseconds, Seconds, microseconds) + + convert(Nanoseconds, Seconds, nanoseconds) + let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + + convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + + nanoseconds mod 1_000_000_000).int + # Nanoseconds might be negative so we must normalize. + result = normalize[Duration](seconds, nanoseconds) proc weeks*(dur: Duration): int64 {.inline.} = ## Number of whole weeks represented by the duration. @@ -407,9 +524,10 @@ proc fractional*(dur: Duration): Duration {.inline.} = doAssert dur.fractional == initDuration(nanoseconds = 5) initDuration(nanoseconds = dur.nanosecond) - -proc fromUnix*(unix: int64): Time {.benign, tags: [], raises: [], noSideEffect.} = - ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) to a ``Time``. +proc fromUnix*(unix: int64): Time + {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) + ## to a ``Time``. runnableExamples: doAssert $fromUnix(0).utc == "1970-01-01T00:00:00Z" initTime(unix, 0) @@ -421,15 +539,16 @@ proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = t.seconds proc fromWinTime*(win: int64): Time = - ## Convert a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``) - ## to a ``Time``. + ## Convert a Windows file time (100-nanosecond intervals since + ## ``1601-01-01T00:00:00Z``) to a ``Time``. const hnsecsPerSec = convert(Seconds, Nanoseconds, 1) div 100 let nanos = floorMod(win, hnsecsPerSec) * 100 let seconds = floorDiv(win - epochDiff, hnsecsPerSec) result = initTime(seconds, nanos) proc toWinTime*(t: Time): int64 = - ## Convert ``t`` to a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``). + ## Convert ``t`` to a Windows file time (100-nanosecond intervals + ## since ``1601-01-01T00:00:00Z``). result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 proc isLeapYear*(year: int): bool = @@ -437,7 +556,7 @@ proc isLeapYear*(year: int): bool = year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) proc getDaysInMonth*(month: Month, year: int): int = - ## Get the number of days in a ``month`` of a ``year``. + ## Get the number of days in ``month`` of ``year``. # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month case month of mFeb: result = if isLeapYear(year): 29 else: 28 @@ -448,15 +567,18 @@ proc getDaysInYear*(year: int): int = ## Get the number of days in a ``year`` result = 365 + (if isLeapYear(year): 1 else: 0) -proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = +proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) + {.inline.} = assert monthday <= getDaysInMonth(month, year), - $year & "-" & intToStr(ord(month), 2) & "-" & $monthday & " is not a valid date" + $year & "-" & intToStr(ord(month), 2) & "-" & $monthday & + " is not a valid date" proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = ## Get the epoch day from a year/month/day date. - ## The epoch day is the number of days since 1970/01/01 (it might be negative). - assertValidDate monthday, month, year + ## The epoch day is the number of days since 1970/01/01 + ## (it might be negative). # Based on http://howardhinnant.github.io/date_algorithms.html + assertValidDate monthday, month, year var (y, m, d) = (year, ord(month), monthday.int) if m <= 2: y.dec @@ -467,9 +589,11 @@ proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = let doe = yoe * 365 + yoe div 4 - yoe div 100 + doy return era * 146097 + doe - 719468 -proc fromEpochDay(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = +proc fromEpochDay(epochday: int64): + tuple[monthday: MonthdayRange, month: Month, year: int] = ## Get the year/month/day date from a epoch day. - ## The epoch day is the number of days since 1970/01/01 (it might be negative). + ## The epoch day is the number of days since 1970/01/01 + ## (it might be negative). # Based on http://howardhinnant.github.io/date_algorithms.html var z = epochday z.inc 719468 @@ -483,19 +607,23 @@ proc fromEpochDay(epochday: int64): tuple[monthday: MonthdayRange, month: Month, let m = mp + (if mp < 10: 3 else: -9) return (d.MonthdayRange, m.Month, (y + ord(m <= 2)).int) -proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign .} = +proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): + YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).yearday``. assertValidDate monthday, month, year - const daysUntilMonth: array[Month, int] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] - const daysUntilMonthLeap: array[Month, int] = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] + const daysUntilMonth: array[Month, int] = + [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] + const daysUntilMonthLeap: array[Month, int] = + [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] if isLeapYear(year): result = daysUntilMonthLeap[month] + monthday - 1 else: result = daysUntilMonth[month] + monthday - 1 -proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign .} = +proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay + {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).weekday``. assertValidDate monthday, month, year @@ -507,8 +635,7 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {. # so we must correct for the WeekDay type. result = if wd == 0: dSun else: WeekDay(wd - 1) - -{. pragma: operator, rtl, noSideEffect, benign .} +{.pragma: operator, rtl, noSideEffect, benign.} template subImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = normalize[T](a.seconds - b.seconds, a.nanosecond - b.nanosecond) @@ -526,28 +653,6 @@ template lqImpl(a: Duration|Time, b: Duration|Time): bool = template eqImpl(a: Duration|Time, b: Duration|Time): bool = a.seconds == b.seconds and a.nanosecond == b.nanosecond - -proc initDuration*(nanoseconds, microseconds, milliseconds, - seconds, minutes, hours, days, weeks: int64 = 0): Duration = - runnableExamples: - let dur = initDuration(seconds = 1, milliseconds = 1) - doAssert dur.milliseconds == 1 - doAssert dur.seconds == 1 - - let seconds = convert(Weeks, Seconds, weeks) + - convert(Days, Seconds, days) + - convert(Minutes, Seconds, minutes) + - convert(Hours, Seconds, hours) + - convert(Seconds, Seconds, seconds) + - convert(Milliseconds, Seconds, milliseconds) + - convert(Microseconds, Seconds, microseconds) + - convert(Nanoseconds, Seconds, nanoseconds) - let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + - convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + - nanoseconds mod 1_000_000_000).int - # Nanoseconds might be negative so we must normalize. - result = normalize[Duration](seconds, nanoseconds) - const DurationZero* = initDuration() ## \ ## Zero value for durations. Useful for comparisons. ## @@ -564,7 +669,7 @@ proc toParts*(dur: Duration): DurationParts = ## ## This procedure is useful for converting ``Duration`` values to strings. runnableExamples: - var dp = toParts(initDuration(weeks=2, days=1)) + var dp = toParts(initDuration(weeks = 2, days = 1)) doAssert dp[Days] == 1 doAssert dp[Weeks] == 2 dp = toParts(initDuration(days = -1)) @@ -616,12 +721,14 @@ proc humanizeParts(parts: seq[string]): string = result.add "and " & parts[high(parts)] proc `$`*(dur: Duration): string = - ## Human friendly string representation of ``Duration``. + ## Human friendly string representation of a ``Duration``. runnableExamples: doAssert $initDuration(seconds = 2) == "2 seconds" doAssert $initDuration(weeks = 1, days = 2) == "1 week and 2 days" - doAssert $initDuration(hours = 1, minutes = 2, seconds = 3) == "1 hour, 2 minutes, and 3 seconds" - doAssert $initDuration(milliseconds = -1500) == "-1 second and -500 milliseconds" + doAssert $initDuration(hours = 1, minutes = 2, seconds = 3) == + "1 hour, 2 minutes, and 3 seconds" + doAssert $initDuration(milliseconds = -1500) == + "-1 second and -500 milliseconds" var parts = newSeq[string]() var numParts = toParts(dur) @@ -659,7 +766,7 @@ proc `<`*(a, b: Duration): bool {.operator.} = ## Use ``abs(a) < abs(b)`` to compare the absolute ## duration. runnableExamples: - doAssert initDuration(seconds = 1) < initDuration(seconds = 2) + doAssert initDuration(seconds = 1) < initDuration(seconds = 2) doAssert initDuration(seconds = -2) < initDuration(seconds = 1) ltImpl(a, b) @@ -669,23 +776,25 @@ proc `<=`*(a, b: Duration): bool {.operator.} = proc `==`*(a, b: Duration): bool {.operator.} = eqImpl(a, b) -proc `*`*(a: int64, b: Duration): Duration {.operator} = +proc `*`*(a: int64, b: Duration): Duration {.operator.} = ## Multiply a duration by some scalar. runnableExamples: doAssert 5 * initDuration(seconds = 1) == initDuration(seconds = 5) normalize[Duration](a * b.seconds, a * b.nanosecond) -proc `*`*(a: Duration, b: int64): Duration {.operator} = +proc `*`*(a: Duration, b: int64): Duration {.operator.} = ## Multiply a duration by some scalar. runnableExamples: doAssert initDuration(seconds = 1) * 5 == initDuration(seconds = 5) b * a -proc `div`*(a: Duration, b: int64): Duration {.operator} = +proc `div`*(a: Duration, b: int64): Duration {.operator.} = ## Integer division for durations. runnableExamples: - doAssert initDuration(seconds = 3) div 2 == initDuration(milliseconds = 1500) - doAssert initDuration(nanoseconds = 3) div 2 == initDuration(nanoseconds = 1) + doAssert initDuration(seconds = 3) div 2 == + initDuration(milliseconds = 1500) + doAssert initDuration(nanoseconds = 3) div 2 == + initDuration(nanoseconds = 1) let carryOver = convert(Seconds, Nanoseconds, a.seconds mod b) normalize[Duration](a.seconds div b, (a.nanosecond + carryOver) div b) @@ -714,7 +823,7 @@ proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} = ## Returns true iff ``a < b``, that is iff a happened before b. ltImpl(a, b) -proc `<=` * (a, b: Time): bool {.operator, extern: "ntLeTime".} = +proc `<=`*(a, b: Time): bool {.operator, extern: "ntLeTime".} = ## Returns true iff ``a <= b``. lqImpl(a, b) @@ -783,8 +892,10 @@ proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = proc newTimezone*( name: string, - zonedTimeFromTimeImpl: proc (time: Time): ZonedTime {.tags: [], raises: [], benign.}, - zonedTimeFromAdjTimeImpl: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} + zonedTimeFromTimeImpl: proc (time: Time): ZonedTime + {.tags: [], raises: [], benign.}, + zonedTimeFromAdjTimeImpl: proc (adjTime: Time): ZonedTime + {.tags: [], raises: [], benign.} ): Timezone = ## Create a new ``Timezone``. ## @@ -847,11 +958,13 @@ proc `==`*(zone1, zone2: Timezone): bool = doAssert local() != utc() zone1.name == zone2.name -proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = +proc inZone*(time: Time, zone: Timezone): DateTime + {.tags: [], raises: [], benign.} = ## Convert ``time`` into a ``DateTime`` using ``zone`` as the timezone. result = initDateTime(zone.zonedTimeFromTime(time), zone) -proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = +proc inZone*(dt: DateTime, zone: Timezone): DateTime + {.tags: [], raises: [], benign.} = ## Returns a ``DateTime`` representing the same point in time as ``dt`` but ## using ``zone`` as the timezone. dt.toTime.inZone(zone) @@ -865,94 +978,38 @@ proc toAdjTime(dt: DateTime): Time = result = initTime(seconds, dt.nanosecond) when defined(JS): - type JsDate = object - proc newDate(year, month, date, hours, minutes, seconds, milliseconds: int): JsDate {.tags: [], raises: [], importc: "new Date".} - proc newDate(): JsDate {.importc: "new Date".} - proc newDate(value: float): JsDate {.importc: "new Date".} - proc getTimezoneOffset(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getTime(js: JsDate): int {.tags: [], raises: [], noSideEffect, benign, importcpp.} - proc getDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc getYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} - proc setFullYear(js: JsDate, year: int): void {.tags: [], raises: [], benign, importcpp.} + proc localZonedTimeFromTime(time: Time): ZonedTime = + let jsDate = newDate(time.seconds * 1000) + let offset = jsDate.getTimezoneOffset() * secondsInMin + result.time = time + result.utcOffset = offset + result.isDst = false - proc localZonedTimeFromTime(time: Time): ZonedTime = - let jsDate = newDate(time.seconds.float * 1000) - let offset = jsDate.getTimezoneOffset() * secondsInMin - result.time = time - result.utcOffset = offset - result.isDst = false + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + let utcDate = newDate(adjTime.seconds * 1000) + let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), + utcDate.getUTCDate(), utcDate.getUTCHours(), utcDate.getUTCMinutes(), + utcDate.getUTCSeconds(), 0) - proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = - let utcDate = newDate(adjTime.seconds.float * 1000) - let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), - utcDate.getUTCHours(), utcDate.getUTCMinutes(), utcDate.getUTCSeconds(), 0) + # This is as dumb as it looks - JS doesn't support years in the range + # 0-99 in the constructor because they are assumed to be 19xx... + # Because JS doesn't support timezone history, + # it doesn't really matter in practice. + if utcDate.getUTCFullYear() in 0 .. 99: + localDate.setFullYear(utcDate.getUTCFullYear()) - # This is as dumb as it looks - JS doesn't support years in the range 0-99 in the constructor - # because they are assumed to be 19xx... - # Because JS doesn't support timezone history, it doesn't really matter in practice. - if utcDate.getUTCFullYear() in 0 .. 99: - localDate.setFullYear(utcDate.getUTCFullYear()) - - result.utcOffset = localDate.getTimezoneOffset() * secondsInMin - result.time = adjTime + initDuration(seconds = result.utcOffset) - result.isDst = false + result.utcOffset = localDate.getTimezoneOffset() * secondsInMin + result.time = adjTime + initDuration(seconds = result.utcOffset) + result.isDst = false else: - when defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(macosx): - type - StructTm {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - gmtoff {.importc: "tm_gmtoff".}: clong - else: - type - StructTm {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - when defined(linux) and defined(amd64) or defined(haiku): - gmtoff {.importc: "tm_gmtoff".}: clong - zone {.importc: "tm_zone".}: cstring - type - StructTmPtr = ptr StructTm - - proc localtime(timer: ptr CTime): StructTmPtr {. importc: "localtime", header: "", tags: [].} - - proc toAdjUnix(tm: StructTm): int64 = - let epochDay = toEpochday(tm.monthday, (tm.month + 1).Month, tm.year.int + 1900) + proc toAdjUnix(tm: Tm): int64 = + let epochDay = toEpochday(tm.tm_mday, (tm.tm_mon + 1).Month, + tm.tm_year.int + 1900) result = epochDay * secondsInDay - result.inc tm.hour * secondsInHour - result.inc tm.minute * 60 - result.inc tm.second + result.inc tm.tm_hour * secondsInHour + result.inc tm.tm_min * 60 + result.inc tm.tm_sec proc getLocalOffsetAndDst(unix: int64): tuple[offset: int, dst: bool] = # Windows can't handle unix < 0, so we fall back to unix = 0. @@ -960,7 +1017,7 @@ else: when defined(windows): if unix < 0: var a = 0.CTime - let tmPtr = localtime(addr(a)) + let tmPtr = localtime(a) if not tmPtr.isNil: let tm = tmPtr[] return ((0 - tm.toAdjUnix).int, false) @@ -969,10 +1026,10 @@ else: # In case of a 32-bit time_t, we fallback to the closest available # timezone information. var a = clamp(unix, low(CTime), high(CTime)).CTime - let tmPtr = localtime(addr(a)) + let tmPtr = localtime(a) if not tmPtr.isNil: let tm = tmPtr[] - return ((a.int64 - tm.toAdjUnix).int, tm.isdst > 0) + return ((a.int64 - tm.toAdjUnix).int, tm.tm_isdst > 0) return (0, false) proc localZonedTimeFromTime(time: Time): ZonedTime = @@ -981,7 +1038,7 @@ else: result.utcOffset = offset result.isDst = dst - proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = var adjUnix = adjTime.seconds let past = adjUnix - secondsInDay let (pastOffset, _) = getLocalOffsetAndDst(past) @@ -991,7 +1048,7 @@ else: var utcOffset: int if pastOffset == futureOffset: - utcOffset = pastOffset.int + utcOffset = pastOffset.int else: if pastOffset > futureOffset: adjUnix -= secondsInHour @@ -1025,8 +1082,8 @@ proc utc*(): TimeZone = proc local*(): TimeZone = ## Get the ``Timezone`` implementation for the local timezone. runnableExamples: - doAssert now().timezone == local() - doAssert local().name == "LOCAL" + doAssert now().timezone == local() + doAssert local().name == "LOCAL" if localInstance.isNil: localInstance = newTimezone("LOCAL", localZonedTimeFromTime, localZonedTimeFromAdjTime) @@ -1060,7 +1117,8 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = elif defined(macosx) or defined(freebsd): var a: Timeval gettimeofday(a) - result = initTime(a.tv_sec.int64, convert(Microseconds, Nanoseconds, a.tv_usec.int)) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) elif defined(posix): var ts: Timespec discard clock_gettime(realTimeClockId, ts) @@ -1081,13 +1139,17 @@ proc initTimeInterval*(nanoseconds, microseconds, milliseconds, days, weeks, months, years: int = 0): TimeInterval = ## Creates a new ``TimeInterval``. ## + ## This proc doesn't perform any normalization! For example, + ## ``initTimeInterval(hours = 24)`` and ``initTimeInterval(days = 1)`` are + ## not equal. + ## ## You can also use the convenience procedures called ``milliseconds``, ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. - ## runnableExamples: - let day = initTimeInterval(hours=24) + let day = initTimeInterval(hours = 24) let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) doAssert $(dt + day) == "2000-01-02T12:00:00Z" + doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) result.nanoseconds = nanoseconds result.microseconds = microseconds result.milliseconds = milliseconds @@ -1115,7 +1177,7 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval = proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval runnableExamples: - let day = -initTimeInterval(hours=24) + let day = -initTimeInterval(hours = 24) doAssert day.hours == -24 result = TimeInterval( @@ -1136,9 +1198,9 @@ proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## ## Time components are subtracted one-by-one, see output: runnableExamples: - let ti1 = initTimeInterval(hours=24) - let ti2 = initTimeInterval(hours=4) - doAssert (ti1 - ti2) == initTimeInterval(hours=20) + let ti1 = initTimeInterval(hours = 24) + let ti2 = initTimeInterval(hours = 4) + doAssert (ti1 - ti2) == initTimeInterval(hours = 20) result = ti1 + (-ti2) @@ -1155,13 +1217,13 @@ proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ':' & intToStr(dt.second, 2) proc toParts* (ti: TimeInterval): TimeIntervalParts = - ## Converts a `TimeInterval` into an array consisting of its time units, - ## starting with nanoseconds and ending with years + ## Converts a ``TimeInterval`` into an array consisting of its time units, + ## starting with nanoseconds and ending with years. ## ## This procedure is useful for converting ``TimeInterval`` values to strings. ## E.g. then you need to implement custom interval printing runnableExamples: - var tp = toParts(initTimeInterval(years=1, nanoseconds=123)) + var tp = toParts(initTimeInterval(years = 1, nanoseconds = 123)) doAssert tp[Years] == 1 doAssert tp[Nanoseconds] == 123 @@ -1171,9 +1233,10 @@ proc toParts* (ti: TimeInterval): TimeIntervalParts = index += 1 proc `$`*(ti: TimeInterval): string = - ## Get string representation of `TimeInterval` + ## Get string representation of ``TimeInterval``. runnableExamples: - doAssert $initTimeInterval(years=1, nanoseconds=123) == "1 year and 123 nanoseconds" + doAssert $initTimeInterval(years = 1, nanoseconds = 123) == + "1 year and 123 nanoseconds" doAssert $initTimeInterval() == "0 nanoseconds" var parts: seq[string] = @[] @@ -1199,7 +1262,7 @@ proc milliseconds*(ms: int): TimeInterval {.inline.} = proc seconds*(s: int): TimeInterval {.inline.} = ## TimeInterval of ``s`` seconds. ## - ## ``echo getTime() + 5.second`` + ## ``echo getTime() + 5.seconds`` initTimeInterval(seconds = s) proc minutes*(m: int): TimeInterval {.inline.} = @@ -1238,7 +1301,8 @@ proc years*(y: int): TimeInterval {.inline.} = ## ``echo getTime() + 2.years`` initTimeInterval(years = y) -proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDur, absDur: Duration] = +proc evaluateInterval(dt: DateTime, interval: TimeInterval): + tuple[adjDur, absDur: Duration] = ## Evaluates how many nanoseconds the interval is worth ## in the context of ``dt``. ## The result in split into an adjusted diff and an absolute diff. @@ -1277,10 +1341,10 @@ proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDur, absDu minutes = interval.minutes, hours = interval.hours) - proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, hour: HourRange, minute: MinuteRange, second: SecondRange, - nanosecond: NanosecondRange, zone: Timezone = local()): DateTime = + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime = ## Create a new ``DateTime`` in the specified timezone. runnableExamples: let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) @@ -1288,12 +1352,12 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, assertValidDate monthday, month, year let dt = DateTime( - monthday: monthday, - year: year, - month: month, - hour: hour, - minute: minute, - second: second, + monthday: monthday, + year: year, + month: month, + hour: hour, + minute: minute, + second: second, nanosecond: nanosecond ) result = initDateTime(zone.zonedTimeFromAdjTime(dt.toAdjTime), zone) @@ -1310,14 +1374,15 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## Adds ``interval`` to ``dt``. Components from ``interval`` are added - ## in the order of their size, i.e first the ``years`` component, then the ``months`` - ## component and so on. The returned ``DateTime`` will have the same timezone as the input. - ## - ## Note that when adding months, monthday overflow is allowed. This means that if the resulting - ## month doesn't have enough days it, the month will be incremented and the monthday will be - ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, - ## which will overflow and result in `1 December`. + ## in the order of their size, i.e first the ``years`` component, then the + ## ``months`` component and so on. The returned ``DateTime`` will have the + ## same timezone as the input. ## + ## Note that when adding months, monthday overflow is allowed. This means that + ## if the resulting month doesn't have enough days it, the month will be + ## incremented and the monthday will be set to the number of days overflowed. + ## So adding one month to `31 October` will result in `31 November`, which + ## will overflow and result in `1 December`. runnableExamples: let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" @@ -1337,9 +1402,10 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = result = initDateTime(zt, dt.timezone) proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = - ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted - ## in the order of their size, i.e first the ``years`` component, then the ``months`` - ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + ## Subtract ``interval`` from ``dt``. Components from ``interval`` are + ## subtracted in the order of their size, i.e first the ``years`` component, + ## then the ``months`` component and so on. The returned ``DateTime`` will + ## have the same timezone as the input. runnableExamples: let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" @@ -1373,15 +1439,15 @@ proc `-`*(dt1, dt2: DateTime): Duration = dt1.toTime - dt2.toTime proc `<`*(a, b: DateTime): bool = - ## Returns true iff ``a < b``, that is iff a happened before b. + ## Returns true iff ``a`` happened before ``b``. return a.toTime < b.toTime -proc `<=` * (a, b: DateTime): bool = - ## Returns true iff ``a <= b``. +proc `<=`*(a, b: DateTime): bool = + ## Returns true iff ``a`` happened before or at the same time as ``b``. return a.toTime <= b.toTime proc `==`*(a, b: DateTime): bool = - ## Returns true if ``a == b``, that is if both dates represent the same point in time. + ## Returns true iff ``a`` and ``b`` represent the same point in time. return a.toTime == b.toTime proc isStaticInterval(interval: TimeInterval): bool = @@ -1482,8 +1548,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = #Months if endDt.month < startDt.month: - result.months = endDt.month.int() + 12 - startDt.month.int() - endDt.year -= 1 + result.months = endDt.month.int() + 12 - startDt.month.int() + endDt.year -= 1 else: result.months = endDt.month.int() - startDt.month.int() @@ -1604,7 +1670,7 @@ type TimeFormatParseError* = object of ValueError ## \ ## Raised when parsing a ``TimeFormat`` string fails. -const FormatLiterals = { ' ', '-', '/', ':', '(', ')', '[', ']', ',' } +const FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ','} proc `$`*(f: TimeFormat): string = ## Returns the format string that was used to construct ``f``. @@ -1672,9 +1738,9 @@ iterator tokens(f: string): tuple[kind: FormatTokenKind, token: string] = i.inc yield (tkLiteral, token) of FormatLiterals: - yieldCurrToken() - yield (tkLiteral, $f[i]) - i.inc + yieldCurrToken() + yield (tkLiteral, $f[i]) + i.inc else: # Check if the letter being added matches previous accumulated buffer. if currToken.len == 0 or currToken[0] == f[i]: @@ -1832,12 +1898,12 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string) = else: result.add '+' & $year of UUUU: - result.add $dt.year + result.add $dt.year of z, zz, zzz, zzzz: if dt.timezone != nil and dt.timezone.name == "Etc/UTC": result.add 'Z' else: - result.add if -dt.utcOffset >= 0: '+' else: '-' + result.add if -dt.utcOffset >= 0: '+' else: '-' let absOffset = abs(dt.utcOffset) case pattern: of z: @@ -1856,7 +1922,7 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string) = result.add h & ":" & m & ":" & s else: assert false of g: - result.add if dt.year < 1: "BC" else: "AD" + result.add if dt.year < 1: "BC" else: "AD" of Lit: assert false # Can't happen proc parsePattern(input: string, pattern: FormatPattern, i: var int, @@ -2017,7 +2083,7 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = year > 0 of yyyy: let year = - if input[i] in { '+', '-' }: + if input[i] in {'+', '-'}: takeInt(4..high(int), allowSign = true) else: takeInt(4..4) @@ -2029,7 +2095,7 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = year > 0 of uuuu: let year = - if input[i] in { '+', '-' }: + if input[i] in {'+', '-'}: takeInt(4..high(int), allowSign = true) else: takeInt(4..4) @@ -2244,7 +2310,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local()): DateTime if patIdx <= f.patterns.high: raiseParseException(f, input, - "Parsing ended but there was still patterns remaining") + "Parsing ended but there was still patterns remaining") result = toDateTime(parsed, zone, f, input) @@ -2261,8 +2327,8 @@ proc parse*(input, f: string, tz: Timezone = local()): DateTime let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz) -proc parse*(input: string, f: static[string], zone: Timezone = local()): DateTime - {.raises: [TimeParseError, Defect].} = +proc parse*(input: string, f: static[string], zone: Timezone = local()): + DateTime {.raises: [TimeParseError, Defect].} = ## Overload that validates ``f`` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone) @@ -2298,7 +2364,7 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = - ## converts a `Time` value to a string representation. It will use the local + ## Converts a `Time` value to a string representation. It will use the local ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. runnableExamples: let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) @@ -2347,7 +2413,8 @@ when not defined(JS): type Clock {.importc: "clock_t".} = distinct int - proc getClock(): Clock {.importc: "clock", header: "", tags: [TimeEffect].} + proc getClock(): Clock + {.importc: "clock", header: "", tags: [TimeEffect].} var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int @@ -2405,59 +2472,68 @@ when defined(JS): # Deprecated procs when not defined(JS): - proc unixTimeToWinTime*(time: CTime): int64 {.deprecated: "Use toWinTime instead".} = + proc unixTimeToWinTime*(time: CTime): int64 + {.deprecated: "Use toWinTime instead".} = ## Converts a UNIX `Time` (``time_t``) to a Windows file time ## ## **Deprecated:** use ``toWinTime`` instead. result = int64(time) * rateDiff + epochDiff - proc winTimeToUnixTime*(time: int64): CTime {.deprecated: "Use fromWinTime instead".} = + proc winTimeToUnixTime*(time: int64): CTime + {.deprecated: "Use fromWinTime instead".} = ## Converts a Windows time to a UNIX `Time` (``time_t``) ## ## **Deprecated:** use ``fromWinTime`` instead. result = CTime((time - epochDiff) div rateDiff) -proc initInterval*(seconds, minutes, hours, days, months, - years: int = 0): TimeInterval {.deprecated.} = +proc initInterval*(seconds, minutes, hours, days, months, years: int = 0): + TimeInterval {.deprecated.} = ## **Deprecated since v0.18.0:** use ``initTimeInterval`` instead. initTimeInterval(0, 0, 0, seconds, minutes, hours, days, 0, months, years) -proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = +proc fromSeconds*(since1970: float): Time + {.tags: [], raises: [], benign, deprecated.} = ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - let nanos = ((since1970 - since1970.int64.float) * convert(Seconds, Nanoseconds, 1).float).int + let nanos = ((since1970 - since1970.int64.float) * + convert(Seconds, Nanoseconds, 1).float).int initTime(since1970.int64, nanos) -proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = +proc fromSeconds*(since1970: int64): Time + {.tags: [], raises: [], benign, deprecated.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead fromUnix(since1970) -proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = +proc toSeconds*(time: Time): float + {.tags: [], raises: [], benign, deprecated.} = ## Returns the time in seconds since the unix epoch. ## ## **Deprecated since v0.18.0:** use ``toUnix`` instead time.seconds.float + time.nanosecond / convert(Seconds, Nanoseconds, 1) -proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = +proc getLocalTime*(time: Time): DateTime + {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-time representation, ## expressed relative to the user's specified time zone. ## ## **Deprecated since v0.18.0:** use ``local`` instead time.local -proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = +proc getGMTime*(time: Time): DateTime + {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). ## ## **Deprecated since v0.18.0:** use ``utc`` instead time.utc -proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} = +proc getTimezone*(): int + {.tags: [TimeEffect], raises: [], benign, deprecated.} = ## Returns the offset of the local (non-DST) timezone in seconds west of UTC. ## ## **Deprecated since v0.18.0:** use ``now().utcOffset`` to get the current @@ -2465,45 +2541,14 @@ proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} when defined(JS): return newDate().getTimezoneOffset() * 60 elif defined(freebsd) or defined(netbsd) or defined(openbsd): - var a: CTime - discard time(a) - let lt = localtime(addr(a)) - # BSD stores in `gmtoff` offset east of UTC in seconds, - # but posix systems using west of UTC in seconds - return -(lt.gmtoff) + # This is wrong since it will include DST offsets, but the behavior has + # always been wrong for bsd and the proc is deprecated so lets ignore it. + return now().utcOffset else: return timezone -proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = - ## Converts a broken-down time structure to calendar time representation. - ## - ## **Deprecated since v0.14.0:** use ``toTime`` instead. - dt.toTime - -when defined(JS): - var start = getTime() - proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = - let dur = getTime() - start - result = (convert(Seconds, Milliseconds, dur.seconds) + - convert(Nanoseconds, Milliseconds, dur.nanosecond)).int -else: - proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = - ## get the milliseconds from the start of the program. - ## - ## **Deprecated since v0.8.10:** use ``epochTime`` or ``cpuTime`` instead. - when defined(macosx): - result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) - else: - result = int(getClock()) div (clocksPerSec div 1000) - -proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = - ## Converts a Time to a TimeInterval. - ## - ## **Deprecated since v0.14.0:** use ``toTimeInterval`` instead. - # Milliseconds not available from Time - t.toTimeInterval() - -proc getDayOfWeek*(day, month, year: int): WeekDay {.tags: [], raises: [], benign, deprecated.} = +proc getDayOfWeek*(day, month, year: int): WeekDay + {.tags: [], raises: [], benign, deprecated.} = ## **Deprecated since v0.18.0:** use ## ``getDayOfWeek(monthday: MonthdayRange; month: Month; year: int)`` instead. getDayOfWeek(day, month.Month, year) From d998cb58dd9a29cb7b739ee4daa6a0528810790d Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Thu, 10 Jan 2019 17:19:35 +0530 Subject: [PATCH 067/350] void object fields are now ignored by codegen and fields/fieldPairs iterator (#10144) * Codegen now ignores object fields of type void * Fix `$` bug for objects/tuples where it does not add a comma * fields/fieldPairs iterators now ignore void types * Use `isEmptyType` instead of checking for `tyVoid` directly --- compiler/ccgtypes.nim | 2 ++ compiler/jsgen.nim | 2 ++ compiler/semfields.nim | 3 +++ lib/system.nim | 1 + tests/objects/t3734.nim | 17 +++++++++++++++++ 5 files changed, 25 insertions(+) create mode 100644 tests/objects/t3734.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 243aa87ded..23e16cb934 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1041,6 +1041,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; else: internalError(m.config, n.info, "genObjectFields(nkRecCase)") of nkSym: var field = n.sym + # Do not produce code for void types + if isEmptyType(field.typ): return if field.bitsize == 0: if field.loc.r == nil: fillObjectFields(m, typ) if field.loc.t == nil: diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 83d205bc29..2f5e202e0c 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1502,6 +1502,8 @@ proc createRecordVarAux(p: PProc, rec: PNode, excludedFieldIDs: IntSet, output: for i in countup(1, sonsLen(rec) - 1): createRecordVarAux(p, lastSon(rec.sons[i]), excludedFieldIDs, output) of nkSym: + # Do not produce code for void types + if isEmptyType(rec.sym.typ): return if rec.sym.id notin excludedFieldIDs: if output.len > 0: output.add(", ") output.addf("$#: ", [mangleName(p.module, rec.sym)]) diff --git a/compiler/semfields.nim b/compiler/semfields.nim index 07321f4778..d65d962cb9 100644 --- a/compiler/semfields.nim +++ b/compiler/semfields.nim @@ -19,6 +19,9 @@ type c: PContext proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode = + if c.field != nil and isEmptyType(c.field.typ): + result = newNode(nkEmpty) + return case n.kind of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = n of nkIdent, nkSym: diff --git a/lib/system.nim b/lib/system.nim index dff1954021..b8ff85f494 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2700,6 +2700,7 @@ proc `$`*[T: tuple|object](x: T): string = firstElement = false else: result.add("...") + firstElement = false when not isNamed: if count == 1: result.add(",") # $(1,) should print as the semantically legal (1,) diff --git a/tests/objects/t3734.nim b/tests/objects/t3734.nim new file mode 100644 index 0000000000..cebef60811 --- /dev/null +++ b/tests/objects/t3734.nim @@ -0,0 +1,17 @@ +discard """ +output: "i0" +""" + +type + Application = object + config: void + i: int + f: void + +proc printFields(rec: Application) = + for k, v in fieldPairs(rec): + echo k, v + +var app: Application + +printFields(app) From 8e600b78ca965117a053751f38ccaffa3e16780a Mon Sep 17 00:00:00 2001 From: Vindaar Date: Thu, 10 Jan 2019 13:44:44 +0100 Subject: [PATCH 068/350] add unsetControlCHook to remove a Ctrl-C hook after it was set (#7267) * add unsetControlCHook to remove a Ctrl-C hook after it was set Adds the inverse proc to setControlCHook in order to lift a Ctrl-C hook after it has been set. * remove check for noSignalHandler in system/excpt.nim --- lib/system.nim | 4 ++++ lib/system/embedded.nim | 1 + lib/system/excpt.nim | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index b8ff85f494..fb52ee9eb2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3458,6 +3458,10 @@ when not defined(JS): #and not defined(nimscript): ## allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. + when not defined(useNimRtl): + proc unsetControlCHook*() + ## reverts a call to setControlCHook + proc writeStackTrace*() {.tags: [], gcsafe.} ## writes the current stack trace to ``stderr``. This is only works ## for debug builds. Since it's usually used for debugging, this diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index 4d453fcca4..fb89e7f0f3 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -40,6 +40,7 @@ proc reraiseException() {.compilerRtl.} = proc writeStackTrace() = discard +proc unsetControlCHook() = discard proc setControlCHook(hook: proc () {.noconv.}) = discard proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 84a1da3438..3efefead4a 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -536,3 +536,8 @@ proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: type SignalHandler = proc (sign: cint) {.noconv, benign.} c_signal(SIGINT, cast[SignalHandler](hook)) + +when not defined(useNimRtl): + proc unsetControlCHook() = + # proc to unset a hook set by setControlCHook + c_signal(SIGINT, signalHandler) From 0bb76dde547ff9f3e80f7421d4f2c556d7aefc4d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jan 2019 08:49:57 -0800 Subject: [PATCH 069/350] fixes #10030 bootstrap is insulated from user config (#10244) --- build_all.sh | 6 +++++- koch.nim | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/build_all.sh b/build_all.sh index 6c28e7407d..6828bbbd8f 100644 --- a/build_all.sh +++ b/build_all.sh @@ -26,6 +26,10 @@ build_nim_csources(){ [ -f $nim_csources ] || echo_run build_nim_csources -echo_run bin/nim c koch # Note: if fails, may need to `cd csources && git pull` +# Note: if fails, may need to `cd csources && git pull` +# Note: --skipUserCfg is to prevent newer flags from +# breaking bootstrap phase +echo_run bin/nim c --skipUserCfg koch + echo_run ./koch boot -d:release echo_run ./koch tools # Compile Nimble and other tools. diff --git a/koch.nim b/koch.nim index 845dffadac..a77ad3144a 100644 --- a/koch.nim +++ b/koch.nim @@ -285,8 +285,13 @@ proc boot(args: string) = copyExe(findStartNim(), 0.thVersion) for i in 0..2: echo "iteration: ", i+1 - exec i.thVersion & " $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, args, - smartNimcache] + let extraOption = if i == 0: + "--skipUserCfg" + # forward compatibility: for bootstrap (1st iteration), avoid user flags + # that could break things, see #10030 + else: "" + exec i.thVersion & " $# $# $# --nimcache:$# compiler" / "nim.nim" % + [bootOptions, extraOption, args, smartNimcache] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) echo "executables are equal: SUCCESS!" From a5fe676f37cfabfec0b4ea1a2008fe82058a7059 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Thu, 10 Jan 2019 17:57:21 +0100 Subject: [PATCH 070/350] Added 'Limitations' section to nimscript manual listing the restrictions of the VM (#10209) --- doc/nims.rst | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/doc/nims.rst b/doc/nims.rst index 034ad1fda8..6cc894b913 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -26,9 +26,28 @@ previous settings): ``$project.nim``. This file can be skipped with the same ``--skipProjCfg`` command line option. -The VM cannot deal with ``importc`` because the FFI is not -available. So the stdlib modules using ``importc`` cannot be used with -Nim's VM. However, at least the following modules are available: +Limitations +================================= + +NimScript is subject to some limitations caused by the implementation of the VM +(virtual machine): + +* Nim's FFI (foreign function interface) is not available in NimScript. This + means that any stdlib module which relies on ``importc`` can not be used in + the VM. + +* ``ptr`` operations are are hard to emulate with the symbolic representation + the VM uses. They are available and tested extensively but there are bugs left. + +* ``var T`` function arguments rely on ``ptr`` operations internally and might + also be problematic in some cases. + +* More than one level of `ref` is generally not supported (for example, the type + `ref ref int`). + +* multimethods are not available. + +Given the above restrictions, at least the following modules are available: * `macros `_ * `os `_ From ab425d793af1a5c0b221bfbeab382b4781479c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Wollenschl=C3=A4ger?= Date: Fri, 11 Jan 2019 03:18:00 +0900 Subject: [PATCH 071/350] [RFC] Better atomics (#8620) * Initial version of C++11 style atomics * Make Atomic[T] always concrete --- lib/pure/concurrency/atomics.nim | 378 +++++++++++++++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100644 lib/pure/concurrency/atomics.nim diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim new file mode 100644 index 0000000000..5a3709f87f --- /dev/null +++ b/lib/pure/concurrency/atomics.nim @@ -0,0 +1,378 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Jörg Wollenschläger +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Types and operations for atomic operations and lockless algorithms. + +import macros + +when defined(cpp) or defined(nimdoc): + # For the C++ backend, types and operations map directly to C++11 atomics. + + {.push, header: "".} + + type + MemoryOrder* {.importcpp: "std::memory_order".} = enum + ## Specifies how non-atomic operations can be reordered around atomic + ## operations. + + moRelaxed + ## No ordering constraints. Only the atomicity and ordering against + ## other atomic operations is guaranteed. + + moConsume + ## This ordering is currently discouraged as it's semantics are + ## being revised. Acquire operations should be preferred. + + moAcquire + ## When applied to a load operation, no reads or writes in the + ## current thread can be reordered before this operation. + + moRelease + ## When applied to a store operation, no reads or writes in the + ## current thread can be reorderd after this operation. + + moAcquireRelease + ## When applied to a read-modify-write operation, this behaves like + ## both an acquire and a release operation. + + moSequentiallyConsistent + ## Behaves like Acquire when applied to load, like Release when + ## applied to a store and like AcquireRelease when applied to a + ## read-modify-write operation. + ## Also garantees that all threads observe the same total ordering + ## with other moSequentiallyConsistent operations. + + type + Atomic* {.importcpp: "std::atomic".} [T] = object + ## An atomic object with underlying type `T`. + + AtomicFlag* {.importcpp: "std::atomic_flag".} = object + ## An atomic boolean state. + + # Access operations + + proc load*[T](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.load(@)".} + ## Atomically obtains the value of the atomic object. + + proc store*[T](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importcpp: "#.store(@)".} + ## Atomically replaces the value of the atomic object with the `desired` + ## value. + + proc exchange*[T](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.exchange(@)".} + ## Atomically replaces the value of the atomic object with the `desired` + ## value and returns the old value. + + proc compareExchange*[T](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.compare_exchange_strong(@)".} + ## Atomically compares the value of the atomic object with the `expected` + ## value and performs exchange with the `desired` one if equal or load if + ## not. Returns true if the exchange was successful. + + proc compareExchange*[T](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.importcpp: "#.compare_exchange_strong(@)".} + ## Same as above, but allows for different memory orders for success and + ## failure. + + proc compareExchangeWeak*[T](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.compare_exchange_weak(@)".} + ## Same as above, but is allowed to fail spuriously. + + proc compareExchangeWeak*[T](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.importcpp: "#.compare_exchange_weak(@)".} + ## Same as above, but allows for different memory orders for success and + ## failure. + + # Numerical operations + + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_add(@)".} + ## Atomically adds a `value` to the atomic integer and returns the + ## original value. + + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_sub(@)".} + ## Atomically subtracts a `value` to the atomic integer and returns the + ## original value. + + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_and(@)".} + ## Atomically replaces the atomic integer with it's bitwise AND + ## with the specified `value` and returns the original value. + + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_or(@)".} + ## Atomically replaces the atomic integer with it's bitwise OR + ## with the specified `value` and returns the original value. + + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_xor(@)".} + ## Atomically replaces the atomic integer with it's bitwise XOR + ## with the specified `value` and returns the original value. + + # Flag operations + + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.test_and_set(@)".} + ## Atomically sets the atomic flag to true and returns the original value. + + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importcpp: "#.clear(@)".} + ## Atomically sets the value of the atomic flag to false. + + proc fence*(order: MemoryOrder) {.importcpp: "std::atomic_thread_fence(@)".} + ## Ensures memory ordering without using atomic operations. + + proc signalFence*(order: MemoryOrder) {.importcpp: "std::atomic_signal_fence(@)".} + ## Prevents reordering of accesses by the compiler as would fence, but + ## inserts no CPU instructions for memory ordering. + + {.pop.} + +else: + # For the C backend, atomics map to C11 built-ins on GCC and Clang for + # trivial Nim types. Other types are implemented using spin locks. + # This could be overcome by supporting advanced importc-patterns. + + # Since MSVC does not implement C11, we fall back to MS intrinsics + # where available. + + type + Trivial = SomeNumber | bool | ptr | pointer + # A type that is known to be atomic and whose size is known at + # compile time to be 8 bytes or less + + template nonAtomicType(T: typedesc[Trivial]): typedesc = + # Maps types to integers of the same size + when sizeof(T) == 1: int8 + elif sizeof(T) == 2: int16 + elif sizeof(T) == 4: int32 + elif sizeof(T) == 8: int64 + + when defined(vcc): + + # TODO: Trivial types should be volatile and use VC's special volatile + # semantics for store and loads. + + type + MemoryOrder* = enum + moRelaxed + moConsume + moAcquire + moRelease + moAcquireRelease + moSequentiallyConsistent + + Atomic*[T] = object + when T is Trivial: + value: T.nonAtomicType + else: + nonAtomicValue: T + guard: AtomicFlag + + AtomicFlag* = distinct int8 + + {.push header: "".} + + # MSVC intrinsics + proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} + + proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} + proc interlockedCompareExchange(location: pointer; desired, expected: int16): int16 {.importc: "_InterlockedCompareExchange16".} + proc interlockedCompareExchange(location: pointer; desired, expected: int32): int32 {.importc: "_InterlockedCompareExchange".} + proc interlockedCompareExchange(location: pointer; desired, expected: int64): int64 {.importc: "_InterlockedCompareExchange64".} + + proc interlockedAnd(location: pointer; value: int8): int8 {.importc: "_InterlockedAnd8".} + proc interlockedAnd(location: pointer; value: int16): int16 {.importc: "_InterlockedAnd16".} + proc interlockedAnd(location: pointer; value: int32): int32 {.importc: "_InterlockedAnd".} + proc interlockedAnd(location: pointer; value: int64): int64 {.importc: "_InterlockedAnd64".} + + proc interlockedOr(location: pointer; value: int8): int8 {.importc: "_InterlockedOr8".} + proc interlockedOr(location: pointer; value: int16): int16 {.importc: "_InterlockedOr16".} + proc interlockedOr(location: pointer; value: int32): int32 {.importc: "_InterlockedOr".} + proc interlockedOr(location: pointer; value: int64): int64 {.importc: "_InterlockedOr64".} + + proc interlockedXor(location: pointer; value: int8): int8 {.importc: "_InterlockedXor8".} + proc interlockedXor(location: pointer; value: int16): int16 {.importc: "_InterlockedXor16".} + proc interlockedXor(location: pointer; value: int32): int32 {.importc: "_InterlockedXor".} + proc interlockedXor(location: pointer; value: int64): int64 {.importc: "_InterlockedXor64".} + + proc fence(order: MemoryOrder): int64 {.importc: "_ReadWriteBarrier()".} + proc signalFence(order: MemoryOrder): int64 {.importc: "_ReadWriteBarrier()".} + + {.pop.} + + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool = + interlockedOr(addr(location), 1'i8) == 1'i8 + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) = + discard interlockedAnd(addr(location), 0'i8) + + proc load*[T: Trivial](location: var T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location), (nonAtomicType(T))0)) + proc store*[T: Trivial](location: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + discard interlockedExchange(addr(location), cast[nonAtomicType(T)](desired)) + + proc exchange*[T: Trivial](location: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedExchange(addr(location), cast[int64](desired))) + proc compareExchange*[T: Trivial](location: var T; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + cast[T](interlockedCompareExchange(addr(location), cast[nonAtomicType(T)](desired), cast[nonAtomicType(T)](expected))) == expected + proc compareExchange*[T: Trivial](location: var T; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + proc compareExchangeWeak*[T: Trivial](location: var T; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + compareExchange(location, expected, desired, success, failure) + proc compareExchangeWeak*[T: Trivial](location: var T; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + + proc fetchAdd*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + let currentValue = location.load() + while not compareExchangeWeak(location, currentValue, currentValue + value): discard + proc fetchSub*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + fetchAdd(location, -value, order) + proc fetchAnd*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedAnd(addr(location), cast[nonAtomicType(T)](value))) + proc fetchOr*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location), cast[nonAtomicType(T)](value))) + proc fetchXor*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedXor(addr(location), cast[nonAtomicType(T)](value))) + + else: + {.push, header: "".} + + type + MemoryOrder* {.importc: "memory_order".} = enum + moRelaxed + moConsume + moAcquire + moRelease + moAcquireRelease + moSequentiallyConsistent + + type + # Atomic* {.importcpp: "_Atomic('0)".} [T] = object + + AtomicInt8 {.importc: "_Atomic NI8".} = object + AtomicInt16 {.importc: "_Atomic NI16".} = object + AtomicInt32 {.importc: "_Atomic NI32".} = object + AtomicInt64 {.importc: "_Atomic NI64".} = object + + template atomicType(T: typedesc[Trivial]): untyped = + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: AtomicInt8 + elif sizeof(T) == 2: AtomicInt16 + elif sizeof(T) == 4: AtomicInt32 + elif sizeof(T) == 8: AtomicInt64 + + type + AtomicFlag* {.importc: "atomic_flag".} = object + + Atomic*[T] = object + when T is Trivial: + value: T.atomicType + else: + nonAtomicValue: T + guard: AtomicFlag + + #proc init*[T](location: var Atomic[T]; value: T): T {.importcpp: "atomic_init(@)".} + proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc.} + proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc.} + proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} + proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} + + # Numerical operations + proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + + # Flag operations + # var ATOMIC_FLAG_INIT {.importc, nodecl.}: AtomicFlag + # proc init*(location: var AtomicFlag) {.inline.} = location = ATOMIC_FLAG_INIT + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".} + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".} + + proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".} + proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".} + + {.pop.} + + proc load*[T: Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_load_explicit[nonAtomicType(T), type(location.value)](addr(location.value), order)) + proc store*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + atomic_store_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order) + proc exchange*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_exchange_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order)) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + atomic_compare_exchange_strong_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + atomic_compare_exchange_weak_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + + # Numerical operations + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_add_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_sub_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_and_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_or_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + + template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = + while location.guard.testAndSet(moAcquire): discard + body + location.guard.clear(moRelease) + + proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + withLock(location, order): + result = location.nonAtomicValue + + proc store*[T: not Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + withLock(location, order): + location.nonAtomicValue = desired + + proc exchange*[T: not Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + withLock(location, order): + result = location.nonAtomicValue + location.nonAtomicValue = desired + + proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + withLock(location, success): + if location.nonAtomicValue != expected: + return false + swap(location.nonAtomicValue, expected) + return true + + proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + withLock(location, success): + if location.nonAtomicValue != expected: + return false + swap(location.nonAtomicValue, expected) + return true + + proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + + proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + +proc atomicInc*[T: SomeInteger](location: var Atomic[T]; value: T = 1) {.inline.} = + ## Atomically increments the atomic integer by some `value`. + discard location.fetchAdd(value) + +proc atomicDec*[T: SomeInteger](location: var Atomic[T]; value: T = 1) {.inline.} = + ## Atomically decrements the atomic integer by some `value`. + discard location.fetchSub(value) + +proc `+=`*[T: SomeInteger](location: var Atomic[T]; value: T) {.inline.} = + ## Atomically increments the atomic integer by some `value`. + discard location.fetchAdd(value) + +proc `-=`*[T: SomeInteger](location: var Atomic[T]; value: T) {.inline.} = + ## Atomically decrements the atomic integer by some `value`. + discard location.fetchSub(value) + \ No newline at end of file From 540d102d87645394d1f6e1c5116f9507a269882c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jan 2019 22:59:20 -0800 Subject: [PATCH 072/350] fix #10261 unittest now returns 1 on error instead of buggy behavior (#10264) --- lib/pure/unittest.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 135a24e9a8..ce147cccce 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -499,7 +499,7 @@ template test*(name, body) {.dirty.} = finally: if testStatusIMPL == FAILED: - programResult += 1 + programResult = 1 let testResult = TestResult( suiteName: when declared(testSuiteName): testSuiteName else: "", testName: name, @@ -540,7 +540,7 @@ template fail* = when declared(testStatusIMPL): testStatusIMPL = FAILED else: - programResult += 1 + programResult = 1 ensureInitialized() From 0a2f711b9e91ae0b188bc9616598d2fcbd8b29b3 Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 11 Jan 2019 08:51:19 +0100 Subject: [PATCH 073/350] revert moving `oids` and `smtp` to graveyard --- changelog.md | 5 +- doc/lib.rst | 9 ++ lib/pure/oids.nim | 93 ++++++++++++++++ lib/pure/smtp.nim | 253 ++++++++++++++++++++++++++++++++++++++++++ lib/pure/smtp.nim.cfg | 1 + tools/kochdocs.nim | 2 + 6 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 lib/pure/oids.nim create mode 100644 lib/pure/smtp.nim create mode 100644 lib/pure/smtp.nim.cfg diff --git a/changelog.md b/changelog.md index da06f3daa9..2ac668cc65 100644 --- a/changelog.md +++ b/changelog.md @@ -62,8 +62,9 @@ - several deprecated modules were removed: `ssl`, `matchers`, `httpserver`, `unsigned`, `actors`, `parseurl` -- several poorly documented and not used modules were moved to graveyard - (they are available as Nimble packages): `subexes`, `scgi`, `smtp`, `oids` +- two poorly documented and not used modules (`subexes`, `scgi`) were moved to + graveyard (they are available as Nimble packages) + #### Breaking changes in the compiler diff --git a/doc/lib.rst b/doc/lib.rst index d6c296e8d4..6b1af43e16 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -238,6 +238,9 @@ Internet Protocols and Support This module implements a simple HTTP client which supports both synchronous and asynchronous retrieval of web pages. +* `smtp `_ + This module implement a simple SMTP client. + * `cookies `_ This module contains helper procs for parsing and generating cookies. @@ -372,6 +375,12 @@ Multimedia support Miscellaneous ------------- +* `oids `_ + An OID is a global ID that consists of a timestamp, + a unique counter and a random value. This combination should suffice to + produce a globally distributed unique ID. This implementation was extracted + from the Mongodb interface and it thus binary compatible with a Mongo OID. + * `endians `_ This module contains helpers that deal with different byte orders. diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim new file mode 100644 index 0000000000..d6369b5f9f --- /dev/null +++ b/lib/pure/oids.nim @@ -0,0 +1,93 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2013 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim OID support. An OID is a global ID that consists of a timestamp, +## a unique counter and a random value. This combination should suffice to +## produce a globally distributed unique ID. This implementation was extracted +## from the Mongodb interface and it thus binary compatible with a Mongo OID. +## +## This implementation calls ``math.randomize()`` for the first call of +## ``genOid``. + +import times, endians + +type + Oid* = object ## an OID + time: int32 ## + fuzz: int32 ## + count: int32 ## + +proc `==`*(oid1: Oid, oid2: Oid): bool = + ## Compare two Mongo Object IDs for equality + return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) + +proc hexbyte*(hex: char): int = + case hex + of '0'..'9': result = (ord(hex) - ord('0')) + of 'a'..'f': result = (ord(hex) - ord('a') + 10) + of 'A'..'F': result = (ord(hex) - ord('A') + 10) + else: discard + +proc parseOid*(str: cstring): Oid = + ## parses an OID. + var bytes = cast[cstring](addr(result.time)) + var i = 0 + while i < 12: + bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) + inc(i) + +proc oidToString*(oid: Oid, str: cstring) = + const hex = "0123456789abcdef" + # work around a compiler bug: + var str = str + var o = oid + var bytes = cast[cstring](addr(o)) + var i = 0 + while i < 12: + let b = bytes[i].ord + str[2 * i] = hex[(b and 0xF0) shr 4] + str[2 * i + 1] = hex[b and 0xF] + inc(i) + str[24] = '\0' + +proc `$`*(oid: Oid): string = + result = newString(24) + oidToString(oid, result) + +var + incr: int + fuzz: int32 + +proc genOid*(): Oid = + ## generates a new OID. + proc rand(): cint {.importc: "rand", header: "", nodecl.} + proc srand(seed: cint) {.importc: "srand", header: "", nodecl.} + + var t = getTime().toUnix.int32 + + var i = int32(atomicInc(incr)) + + if fuzz == 0: + # racy, but fine semantically: + srand(t) + fuzz = rand() + bigEndian32(addr result.time, addr(t)) + result.fuzz = fuzz + bigEndian32(addr result.count, addr(i)) + +proc generatedTime*(oid: Oid): Time = + ## returns the generated timestamp of the OID. + var tmp: int32 + var dummy = oid.time + bigEndian32(addr(tmp), addr(dummy)) + result = fromUnix(tmp) + +when not defined(testing) and isMainModule: + let xo = genOid() + echo xo.generatedTime diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim new file mode 100644 index 0000000000..5f4b09f801 --- /dev/null +++ b/lib/pure/smtp.nim @@ -0,0 +1,253 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the SMTP client protocol as specified by RFC 5321, +## this can be used to send mail to any SMTP Server. +## +## This module also implements the protocol used to format messages, +## as specified by RFC 2822. +## +## Example gmail use: +## +## +## .. code-block:: Nim +## var msg = createMessage("Hello from Nim's SMTP", +## "Hello!.\n Is this awesome or what?", +## @["foo@gmail.com"]) +## let smtpConn = newSmtp(useSsl = true, debug=true) +## smtpConn.connect("smtp.gmail.com", Port 465) +## smtpConn.auth("username", "password") +## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) +## +## +## For SSL support this module relies on OpenSSL. If you want to +## enable SSL, compile with ``-d:ssl``. + +import net, strutils, strtabs, base64, os +import asyncnet, asyncdispatch + +export Port + +type + Message* = object + msgTo: seq[string] + msgCc: seq[string] + msgSubject: string + msgOtherHeaders: StringTableRef + msgBody: string + + ReplyError* = object of IOError + + SmtpBase[SocketType] = ref object + sock: SocketType + debug: bool + + Smtp* = SmtpBase[Socket] + AsyncSmtp* = SmtpBase[AsyncSocket] + +proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = + if smtp.debug: + echo("C:" & cmd) + await smtp.sock.send(cmd) + +proc debugRecv(smtp: Smtp | AsyncSmtp): Future[TaintedString] {.multisync.} = + result = await smtp.sock.recvLine() + if smtp.debug: + echo("S:" & result.string) + +proc quitExcpt(smtp: Smtp, msg: string) = + smtp.debugSend("QUIT") + raise newException(ReplyError, msg) + +const compiledWithSsl = defined(ssl) + +when not defined(ssl): + type PSSLContext = ref object + let defaultSSLContext: PSSLContext = nil +else: + var defaultSSLContext {.threadvar.}: SSLContext + + proc getSSLContext(): SSLContext = + if defaultSSLContext == nil: + defaultSSLContext = newContext(verifyMode = CVerifyNone) + result = defaultSSLContext + +proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], + otherHeaders: openarray[tuple[name, value: string]]): Message = + ## Creates a new MIME compliant message. + result.msgTo = mTo + result.msgCc = mCc + result.msgSubject = mSubject + result.msgBody = mBody + result.msgOtherHeaders = newStringTable() + for n, v in items(otherHeaders): + result.msgOtherHeaders[n] = v + +proc createMessage*(mSubject, mBody: string, mTo, + mCc: seq[string] = @[]): Message = + ## Alternate version of the above. + result.msgTo = mTo + result.msgCc = mCc + result.msgSubject = mSubject + result.msgBody = mBody + result.msgOtherHeaders = newStringTable() + +proc `$`*(msg: Message): string = + ## stringify for ``Message``. + result = "" + if msg.msgTo.len() > 0: + result = "TO: " & msg.msgTo.join(", ") & "\c\L" + if msg.msgCc.len() > 0: + result.add("CC: " & msg.msgCc.join(", ") & "\c\L") + # TODO: Folding? i.e when a line is too long, shorten it... + result.add("Subject: " & msg.msgSubject & "\c\L") + for key, value in pairs(msg.msgOtherHeaders): + result.add(key & ": " & value & "\c\L") + + result.add("\c\L") + result.add(msg.msgBody) + +proc newSmtp*(useSsl = false, debug=false, + sslContext: SSLContext = nil): Smtp = + ## Creates a new ``Smtp`` instance. + new result + result.debug = debug + result.sock = newSocket() + if useSsl: + when compiledWithSsl: + if sslContext == nil: + getSSLContext().wrapSocket(result.sock) + else: + sslContext.wrapSocket(result.sock) + else: + {.error: "SMTP module compiled without SSL support".} + +proc newAsyncSmtp*(useSsl = false, debug=false, + sslContext: SSLContext = nil): AsyncSmtp = + ## Creates a new ``AsyncSmtp`` instance. + new result + result.debug = debug + + result.sock = newAsyncSocket() + if useSsl: + when compiledWithSsl: + if sslContext == nil: + getSSLContext().wrapSocket(result.sock) + else: + sslContext.wrapSocket(result.sock) + else: + {.error: "SMTP module compiled without SSL support".} + +proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = + var retFuture = newFuture[void]() + var sendFut = smtp.debugSend("QUIT") + sendFut.callback = + proc () = + # TODO: Fix this in async procs. + raise newException(ReplyError, msg) + return retFuture + +proc checkReply(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = + var line = await smtp.debugRecv() + if not line.startswith(reply): + await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) + +proc connect*(smtp: Smtp | AsyncSmtp, + address: string, port: Port) {.multisync.} = + ## Establishes a connection with a SMTP server. + ## May fail with ReplyError or with a socket error. + await smtp.sock.connect(address, port) + + await smtp.checkReply("220") + await smtp.debugSend("HELO " & address & "\c\L") + await smtp.checkReply("250") + +proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = + ## Sends an AUTH command to the server to login as the `username` + ## using `password`. + ## May fail with ReplyError. + + await smtp.debugSend("AUTH LOGIN\c\L") + await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" + # i.e "334 VXNlcm5hbWU6" + await smtp.debugSend(encode(username) & "\c\L") + await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) + + await smtp.debugSend(encode(password) & "\c\L") + await smtp.checkReply("235") # Check whether the authentification was successful. + +proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, + toAddrs: seq[string], msg: string) {.multisync.} = + ## Sends ``msg`` from ``fromAddr`` to the addresses specified in ``toAddrs``. + ## Messages may be formed using ``createMessage`` by converting the + ## Message into a string. + + await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") + await smtp.checkReply("250") + for address in items(toAddrs): + await smtp.debugSend("RCPT TO:<" & address & ">\c\L") + await smtp.checkReply("250") + + # Send the message + await smtp.debugSend("DATA " & "\c\L") + await smtp.checkReply("354") + await smtp.sock.send(msg & "\c\L") + await smtp.debugSend(".\c\L") + await smtp.checkReply("250") + +proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = + ## Disconnects from the SMTP server and closes the socket. + await smtp.debugSend("QUIT\c\L") + smtp.sock.close() + +when not defined(testing) and isMainModule: + # To test with a real SMTP service, create a smtp.ini file, e.g.: + # username = "" + # password = "" + # smtphost = "smtp.gmail.com" + # port = 465 + # use_tls = true + # sender = "" + # recipient = "" + + import parsecfg + + proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) + + let + conf = loadConfig("smtp.ini") + msg = createMessage("Hello from Nim's SMTP!", + "Hello!\n Is this awesome or what?", @[conf["recipient"]]) + + assert conf["smtphost"] != "" + + proc async_test() {.async.} = + let client = newAsyncSmtp( + conf["use_tls"].parseBool, + debug=true + ) + await client.connect(conf["smtphost"], conf["port"].parseInt.Port) + await client.auth(conf["username"], conf["password"]) + await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) + await client.close() + echo "async email sent" + + proc sync_test() = + var smtpConn = newSmtp( + conf["use_tls"].parseBool, + debug=true + ) + smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) + smtpConn.auth(conf["username"], conf["password"]) + smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) + smtpConn.close() + echo "sync email sent" + + waitFor async_test() + sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg new file mode 100644 index 0000000000..521e21de41 --- /dev/null +++ b/lib/pure/smtp.nim.cfg @@ -0,0 +1 @@ +-d:ssl diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index b348d3172a..3d146188fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -161,6 +161,7 @@ lib/impure/db_sqlite.nim lib/impure/db_odbc.nim lib/pure/db_common.nim lib/pure/httpclient.nim +lib/pure/smtp.nim lib/pure/ropes.nim lib/pure/unidecode/unidecode.nim lib/pure/xmlparser.nim @@ -190,6 +191,7 @@ lib/pure/memfiles.nim lib/pure/collections/critbits.nim lib/core/locks.nim lib/core/rlocks.nim +lib/pure/oids.nim lib/pure/endians.nim lib/pure/uri.nim lib/pure/nimprof.nim From ceabbeadd7807dd21f274271acc779a7a8e69132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Wollenschl=C3=A4ger?= Date: Fri, 11 Jan 2019 20:07:46 +0900 Subject: [PATCH 074/350] Fixes new atomics for C + vcc (#10267) --- lib/pure/concurrency/atomics.nim | 42 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 5a3709f87f..9e716bdf4b 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -136,7 +136,7 @@ else: # A type that is known to be atomic and whose size is known at # compile time to be 8 bytes or less - template nonAtomicType(T: typedesc[Trivial]): typedesc = + template nonAtomicType(T: typedesc[Trivial]): untyped = # Maps types to integers of the same size when sizeof(T) == 1: int8 elif sizeof(T) == 2: int16 @@ -204,33 +204,33 @@ else: proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) = discard interlockedAnd(addr(location), 0'i8) - proc load*[T: Trivial](location: var T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](interlockedOr(addr(location), (nonAtomicType(T))0)) - proc store*[T: Trivial](location: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = - discard interlockedExchange(addr(location), cast[nonAtomicType(T)](desired)) + proc load*[T: Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location.value), (nonAtomicType(T))0)) + proc store*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + discard interlockedExchange(addr(location.value), cast[nonAtomicType(T)](desired)) - proc exchange*[T: Trivial](location: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](interlockedExchange(addr(location), cast[int64](desired))) - proc compareExchange*[T: Trivial](location: var T; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = - cast[T](interlockedCompareExchange(addr(location), cast[nonAtomicType(T)](desired), cast[nonAtomicType(T)](expected))) == expected - proc compareExchange*[T: Trivial](location: var T; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + proc exchange*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedExchange(addr(location.value), cast[int64](desired))) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + cast[T](interlockedCompareExchange(addr(location.value), cast[nonAtomicType(T)](desired), cast[nonAtomicType(T)](expected))) == expected + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = compareExchange(location, expected, desired, order, order) - proc compareExchangeWeak*[T: Trivial](location: var T; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = compareExchange(location, expected, desired, success, failure) - proc compareExchangeWeak*[T: Trivial](location: var T; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = compareExchangeWeak(location, expected, desired, order, order) - proc fetchAdd*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - let currentValue = location.load() + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + var currentValue = location.load() while not compareExchangeWeak(location, currentValue, currentValue + value): discard - proc fetchSub*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = fetchAdd(location, -value, order) - proc fetchAnd*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](interlockedAnd(addr(location), cast[nonAtomicType(T)](value))) - proc fetchOr*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](interlockedOr(addr(location), cast[nonAtomicType(T)](value))) - proc fetchXor*[T: SomeInteger](location: var T; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](interlockedXor(addr(location), cast[nonAtomicType(T)](value))) + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedAnd(addr(location.value), cast[nonAtomicType(T)](value))) + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location.value), cast[nonAtomicType(T)](value))) + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedXor(addr(location.value), cast[nonAtomicType(T)](value))) else: {.push, header: "".} From d1b7aa28e7b5b847a44eda11ff7aadbe3bb0b49a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 11 Jan 2019 15:55:16 +0100 Subject: [PATCH 075/350] fixes #10240 (#10269) * kochdocs.nim: code cleanup * fixes #10420 --- compiler/scriptconfig.nim | 6 ++++-- tools/kochdocs.nim | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index bfff864797..e3a9478d13 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -63,8 +63,10 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; os.removeFile getString(a, 0) cbos createDir: os.createDir getString(a, 0) - cbos getOsError: - setResult(a, errorMsg) + + result.registerCallback "stdlib.system.getOsError", + proc (a: VmArgs) = setResult(a, errorMsg) + cbos setCurrentDir: os.setCurrentDir getString(a, 0) cbos getCurrentDir: diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 3d146188fa..68662bab24 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -56,12 +56,8 @@ proc nimexec*(cmd: string) = exec findNim() & " " & cmd proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = - # TODO: simplify pending https://github.com/nim-lang/Nim/issues/9513 - var cmd = findNim() & " " & mode let output = outputDir / input.splitFile.name.exe - cmd.add " -o:" & output - cmd.add " " & options - cmd.add " " & input + let cmd = findNim() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd const From 5397c5d31d58f14ad39e8009882a87ceda6253aa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 11 Jan 2019 16:34:22 +0100 Subject: [PATCH 076/350] fixes #10216 (#10270) --- compiler/pragmas.nim | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index bb5707cd52..b1a88ace7e 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -64,10 +64,10 @@ const varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl, wMagic, wHeader, wDeprecated, wCompilerProc, wCore, wDynlib, wExtern, wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal, - wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed, wRaises} + wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims, - wIntDefine, wStrDefine, wUsed, wCompilerProc, wCore, wRaises} + wIntDefine, wStrDefine, wUsed, wCompilerProc, wCore} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, wThread, wRaises, wLocks, wTags, wGcSafe} @@ -1119,13 +1119,16 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, else: sym.flags.incl sfUsed of wLiftLocals: discard else: invalidPragma(c, it) - elif sym == nil or (sym != nil and sym.kind in {skVar, skLet, skParam, - skField, skProc, skFunc, skConverter, skMethod, skType}): - n.sons[i] = semCustomPragma(c, it) - elif sym != nil: - illegalCustomPragma(c, it, sym) + elif comesFromPush and whichKeyword(ident) in {wTags, wRaises}: + discard "ignore the .push pragma; it doesn't apply" else: - invalidPragma(c, it) + if sym == nil or (sym != nil and sym.kind in {skVar, skLet, skParam, + skField, skProc, skFunc, skConverter, skMethod, skType}): + n.sons[i] = semCustomPragma(c, it) + elif sym != nil: + illegalCustomPragma(c, it, sym) + else: + invalidPragma(c, it) proc mergePragmas(n, pragmas: PNode) = if n[pragmasPos].kind == nkEmpty: From 09ef25810aa7de55c32dfe7025668f1f889d00d6 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 11 Jan 2019 08:03:19 -0800 Subject: [PATCH 077/350] refs #10249 ; more debug info to diagnose failures (#10266) --- testament/tester.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/testament/tester.nim b/testament/tester.nim index 1867871c55..04552b7c9f 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -104,15 +104,19 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp var line = newStringOfCap(120).TaintedString while true: if outp.readLine(line): - result[0].string.add(line.string) - result[0].string.add("\n") + result.output.string.add(line.string) + result.output.string.add("\n") if onStdout != nil: onStdout(line.string) else: - result[1] = peekExitCode(p) - if result[1] != -1: break + result.exitCode = peekExitCode(p) + if result.exitCode != -1: break close(p) - - + if result.exitCode != 0: + var command2 = command + if args.len > 0: command2.add " " & args.quoteShellCommand + echo (msg: "execCmdEx2 failed", + command: command2, + options: options) proc nimcacheDir(filename, options: string, target: TTarget): string = ## Give each test a private nimcache dir so they don't clobber each other's. From b0979c8b1c3b43c285fc06ba492b20607e264560 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Fri, 11 Jan 2019 23:46:03 +0530 Subject: [PATCH 078/350] Cleanup comesFromPush logic (#10278) --- compiler/pragmas.nim | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index b1a88ace7e..4adf6032b8 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1054,14 +1054,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, noVal(c, it) if sym == nil: invalidPragma(c, it) of wLine: pragmaLine(c, it) - of wRaises, wTags: - if not sym.isNil and sym.kind in {skVar, skLet, skConst}: - if comesFromPush: - return - else: - invalidPragma(c, it) - else: - pragmaRaisesOrTags(c, it) + of wRaises, wTags: pragmaRaisesOrTags(c, it) of wLocks: if sym == nil: pragmaLockStmt(c, it) elif sym.typ == nil: invalidPragma(c, it) From c3d80647aec6b09673e452d9f451e0549bb10dbc Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2019 12:52:31 +0100 Subject: [PATCH 079/350] docgen: render 'interpreted text' the same as inline literals for markdown compat --- lib/packages/docutils/rstgen.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 232da5c938..766ce6ffd3 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1134,11 +1134,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTripleEmphasis: renderAux(d, n, "$1", "\\textbf{emph{$1}}", result) - of rnInterpretedText: - renderAux(d, n, "$1", "\\emph{$1}", result) of rnIdx: renderIndexTerm(d, n, result) - of rnInlineLiteral: + of rnInlineLiteral, rnInterpretedText: renderAux(d, n, "$1", "\\texttt{$1}", result) From 5ef5dc86c523c760b83050b6192f00b6a0816bce Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2019 14:20:34 +0100 Subject: [PATCH 080/350] docgen: support markdown link syntax; enable markdown extensions --- compiler/docgen.nim | 4 +-- lib/packages/docutils/rst.nim | 28 ++++++++++++++++++++ lib/packages/docutils/rstast.nim | 3 +++ nimdoc/tester.nim | 2 +- nimdoc/testproject/expected/testproject.html | 2 +- nimdoc/testproject/subdir/subdir_b/utils.nim | 2 ++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 1af9c06b8f..33cd98f38f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -119,7 +119,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.conf = conf result.cache = cache initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex), - conf.configVars, filename.string, {roSupportRawDirective}, + conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown}, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -991,7 +991,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; d.isPureRst = true var rst = parseRst(readFile(filen.string), filen.string, 0, 1, d.hasToc, - {roSupportRawDirective}, conf) + {roSupportRawDirective, roSupportMarkdown}, conf) var modDesc = newStringOfCap(30_000) renderRstToOut(d[], rst, modDesc) d.modDesc = rope(modDesc) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 161509afe6..6a03e6bc17 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -780,6 +780,31 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode = add(result, nil) add(result, lb) +proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = + result = true + var desc, link = "" + var i = p.idx + + template parse(endToken, dest) = + inc i # skip begin token + while true: + if p.tok[i].kind in {tkEof, tkIndent}: return false + if p.tok[i].symbol == endToken: break + dest.add p.tok[i].symbol + inc i + inc i # skip end token + + parse("]", desc) + if p.tok[i].symbol != "(": return false + parse(")", link) + let child = newRstNode(rnHyperlink) + child.add desc + child.add link + # only commit if we detected no syntax error: + father.add child + p.idx = i + result = true + proc parseInline(p: var RstParser, father: PRstNode) = case p.tok[p.idx].kind of tkPunct: @@ -811,6 +836,9 @@ proc parseInline(p: var RstParser, father: PRstNode) = var n = newRstNode(rnSubstitutionReferences) parseUntil(p, n, "|", false) add(father, n) + elif roSupportMarkdown in p.s.options and p.tok[p.idx].symbol == "[" and + parseMarkdownLink(p, father): + discard "parseMarkdownLink already processed it" else: if roSupportSmilies in p.s.options: let n = parseSmiley(p) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 4a77b4f34d..fee824b092 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -89,6 +89,9 @@ proc lastSon*(n: PRstNode): PRstNode = proc add*(father, son: PRstNode) = add(father.sons, son) +proc add*(father: PRstNode; s: string) = + add(father.sons, newRstNode(rnLeaf, s)) + proc addIfNotNil*(father, son: PRstNode) = if son != nil: add(father, son) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index e0afe6b94f..db20951283 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -28,5 +28,5 @@ proc test(dir: string; fixup = false) = echo "SUCCESS: files identical: ", produced removeDir(dir / "htmldocs") -test("nimdoc/testproject", false) +test("nimdoc/testproject", defined(fixup)) if failures > 0: quit($failures & " failures occurred.") diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 2e23a64d50..dd1541a56f 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1369,7 +1369,7 @@ The enum B. 
func someFunc() {...}{.raises: [], tags: [].}
-My someFunc. +My someFunc. Stuff in quotes here. Some link
diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index e2ec80dc25..7b529a617c 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -23,4 +23,6 @@ template bEnum*(): untyped = func someFunc*() = ## My someFunc. + ## Stuff in `quotes` here. + ## [Some link](https://nim-lang.org) discard From ee7f2d97cc5e07af3f73a1a8e93086f2e86385bc Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2019 15:06:45 +0100 Subject: [PATCH 081/350] docgen: support markdown headings too --- lib/packages/docutils/rst.nim | 39 +++++++++++++++---- .../expected/subdir/subdir_b/utils.html | 14 ++++++- nimdoc/testproject/expected/theindex.html | 2 +- nimdoc/testproject/subdir/subdir_b/utils.nim | 13 +++++++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 6a03e6bc17..25cea59431 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1065,15 +1065,32 @@ proc isOptionList(p: RstParser): bool = result = match(p, p.idx, "-w") or match(p, p.idx, "--w") or match(p, p.idx, "/w") or match(p, p.idx, "//w") +proc isMarkdownHeadlinePattern(s: string): bool = + if s.len >= 1 and s.len <= 6: + for c in s: + if c != '#': return false + result = true + +proc isMarkdownHeadline(p: RstParser): bool = + if roSupportMarkdown in p.s.options: + if isMarkdownHeadlinePattern(p.tok[p.idx].symbol) and p.tok[p.idx+1].kind == tkWhite: + if p.tok[p.idx+2].kind in {tkWord, tkOther, tkPunct}: + result = true + proc whichSection(p: RstParser): RstNodeKind = case p.tok[p.idx].kind of tkAdornment: if match(p, p.idx + 1, "ii"): result = rnTransition elif match(p, p.idx + 1, " a"): result = rnTable elif match(p, p.idx + 1, "i"): result = rnOverline - else: result = rnLeaf + elif isMarkdownHeadline(p): + result = rnHeadline + else: + result = rnLeaf of tkPunct: - if match(p, tokenAfterNewline(p), "ai"): + if isMarkdownHeadline(p): + result = rnHeadline + elif match(p, tokenAfterNewline(p), "ai"): result = rnHeadline elif p.tok[p.idx].symbol == "::": result = rnLiteralBlock @@ -1158,12 +1175,18 @@ proc parseParagraph(p: var RstParser, result: PRstNode) = proc parseHeadline(p: var RstParser): PRstNode = result = newRstNode(rnHeadline) - parseUntilNewline(p, result) - assert(p.tok[p.idx].kind == tkIndent) - assert(p.tok[p.idx + 1].kind == tkAdornment) - var c = p.tok[p.idx + 1].symbol[0] - inc(p.idx, 2) - result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c) + if isMarkdownHeadline(p): + result.level = p.tok[p.idx].symbol.len + assert(p.tok[p.idx+1].kind == tkWhite) + inc p.idx, 2 + parseUntilNewline(p, result) + else: + parseUntilNewline(p, result) + assert(p.tok[p.idx].kind == tkIndent) + assert(p.tok[p.idx + 1].kind == tkAdornment) + var c = p.tok[p.idx + 1].symbol[0] + inc(p.idx, 2) + result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c) type IntSeq = seq[int] diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 2c89acce45..f2f6264594 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1238,7 +1238,12 @@ function main() {
    -
  • +
  • This is now a header
    • +
  • More headers
    • +
  • Types
    • -

      +

      +

      This is now a header

      +

      Next header

      +

      And so on

      +

      More headers

      +
      Up to level 6

      Types

      diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 652077be87..95c667acca 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1220,7 +1220,7 @@ function main() {

      Index

      - Modules: subdir/subdir_b/utils, testproject.

      API symbols

      + Modules: testproject, utils.

      API symbols

      A:
      • testproject: A
        • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 7b529a617c..28c00c755c 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -1,3 +1,16 @@ +##[ + +# This is now a header + +## Next header + +### And so on + +# More headers + +###### Up to level 6 + +]## type SomeType* = enum From 56b804a283a69baa4887eaf300c353a5748f86a4 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2019 16:31:24 +0100 Subject: [PATCH 082/350] RST parser: fixes #8158 --- lib/packages/docutils/rst.nim | 6 +++--- nimdoc/testproject/expected/subdir/subdir_b/utils.html | 9 ++++++++- nimdoc/testproject/subdir/subdir_b/utils.nim | 9 +++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 25cea59431..551df99194 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -560,9 +560,9 @@ proc match(p: RstParser, start: int, expr: string): bool = result = (p.tok[j].kind == tkWord) or (p.tok[j].symbol == "#") if result: case p.tok[j].symbol[0] - of 'a'..'z', 'A'..'Z': result = len(p.tok[j].symbol) == 1 + of 'a'..'z', 'A'..'Z', '#': result = len(p.tok[j].symbol) == 1 of '0'..'9': result = allCharsInSet(p.tok[j].symbol, {'0'..'9'}) - else: discard + else: result = false else: var c = expr[i] var length = 0 @@ -1105,7 +1105,7 @@ proc whichSection(p: RstParser): RstNodeKind = elif match(p, p.idx, ":w:") and predNL(p): # (p.tok[p.idx].symbol == ":") result = rnFieldList - elif match(p, p.idx, "(e) "): + elif match(p, p.idx, "(e) ") or match(p, p.idx, "e. "): result = rnEnumList elif match(p, p.idx, "+a+"): result = rnGridTable diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f2f6264594..9ba3c24e15 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1281,7 +1281,14 @@ function main() {

        Next header

        And so on

        More headers

        -
        Up to level 6

        +
        Up to level 6
        1. An enumeration
          2. +
        2. Second idea here.
          3. +
        +

        More text.

        +
        1. Other case value
          2. +
        2. Second case.
          3. +
        +

        Types

        diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 28c00c755c..0576f194f3 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -10,6 +10,15 @@ ###### Up to level 6 + +#. An enumeration +#. Second idea here. + +More text. + +1. Other case value +2. Second case. + ]## type From 647066e378bdbc85646dd88ea90eb15eddbbbc50 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2019 18:36:20 +0100 Subject: [PATCH 083/350] make the stdlib work with the changed docgen --- doc/lib.rst | 2 +- lib/core/macros.nim | 2 +- lib/impure/nre.nim | 8 ++++---- lib/pure/collections/critbits.nim | 2 +- lib/pure/collections/sets.nim | 2 +- lib/pure/streams.nim | 2 +- lib/system/gc_common.nim | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 6b1af43e16..395b242e69 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -108,7 +108,7 @@ String handling * `strformat `_ Macro based standard string interpolation / formatting. Inpired by - Python's ```f``-strings. + Python's ``f``-strings. * `strmisc `_ This module contains uncommon string handling operations that do not diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 64334161e6..92a35193f5 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -151,7 +151,7 @@ proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.} proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect, deprecated.} ## compares two Nim symbols - ## **Deprecated since version 0.18.1**; Use ```==`(NimNode,NimNode)`` instead. + ## **Deprecated since version 0.18.1**; Use ``==(NimNode, NimNode)`` instead. proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} = diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 94dd89db5e..5c5125ba10 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -540,7 +540,7 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt raise RegexInternalError(msg : "Unknown internal error: " & $execRet) proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = - ## Like ```find(...)`` <#proc-find>`_, but anchored to the start of the + ## Like ` ``find(...)`` <#proc-find>`_, but anchored to the start of the ## string. ## runnableExamples: @@ -550,11 +550,11 @@ proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[R return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = - ## Works the same as ```find(...)`` <#proc-find>`_, but finds every + ## Works the same as ` ``find(...)`` <#proc-find>`_, but finds every ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not ## ``"22", "22", "22"``. ## - ## Arguments are the same as ```find(...)`` <#proc-find>`_ + ## Arguments are the same as ` ``find(...)`` <#proc-find>`_ ## ## Variants: ## @@ -633,7 +633,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## Splits the string with the given regex. This works according to the ## rules that Perl and Javascript use. ## - ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`_. + ## ``start`` behaves the same as in ` ``find(...)`` <#proc-find>`_. ## runnableExamples: # - If the match is zero-width, then the string is still split: diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 32e0299ba7..1bd13920d4 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -205,7 +205,7 @@ proc `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline, proc mget*[T](c: var CritBitTree[T], key: string): var T {.inline, deprecated.} = ## retrieves the value at ``c[key]``. The value can be modified. ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. + ## Use ``[]`` instead. get(c, key) iterator leaves[T](n: Node[T]): Node[T] = diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 1273cbc33a..07fcfe6763 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -195,7 +195,7 @@ proc mget*[A](s: var HashSet[A], key: A): var A {.deprecated.} = ## returns the element that is actually stored in 's' which has the same ## value as 'key' or raises the ``KeyError`` exception. This is useful ## when one overloaded 'hash' and '==' but still needs reference semantics - ## for sharing. Use ```[]``` instead. + ## for sharing. Use ``[]`` instead. s[key] proc contains*[A](s: HashSet[A], key: A): bool = diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index b5254b4f70..10de86e9f3 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -271,7 +271,7 @@ proc peekStr*(s: Stream, length: int): TaintedString = proc readLine*(s: Stream, line: var TaintedString): bool = ## reads a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ```LF`` or ``CRLF``. + ## A line of text may be delimited by ``LF`` or ``CRLF``. ## The newline character(s) are not part of the returned string. ## Returns ``false`` if the end of the file has been reached, ``true`` ## otherwise. If ``false`` is returned `line` contains no new data. diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 565453298e..ebd3dada25 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -418,7 +418,7 @@ proc prepareDealloc(cell: PCell) = decTypeSize(cell, t) proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = - ## Frees the thread local heap. Runs every finalizer if ``runFinalizers``` + ## Frees the thread local heap. Runs every finalizer if ``runFinalizers`` ## is true. If ``allowGcAfterwards`` is true, a minimal amount of allocation ## happens to ensure the GC can continue to work after the call ## to ``deallocHeap``. From 7e7603ed2b6774454689978a08b46adb2fb953da Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Sat, 12 Jan 2019 08:17:20 +0100 Subject: [PATCH 084/350] Documented error source argument of macros.error() (#10279) --- lib/core/macros.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 92a35193f5..08ee051522 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -377,7 +377,9 @@ proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.} - ## writes an error message at compile time + ## writes an error message at compile time. The optional ``n: NimNode`` + ## parameter is used as the source for file and line number information in + ## the compilation error message. proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.} ## writes a warning message at compile time From 98ef545bedd27959a568a8e99f889a32c3ae1f72 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 11 Jan 2019 23:18:04 -0800 Subject: [PATCH 085/350] fix #10281 (#10282) --- lib/wrappers/openssl.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 447592a686..9ee73a44d7 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -38,7 +38,11 @@ when useWinVersion: from winlean import SocketHandle else: - const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.46|.45|.44|.43|.41|.39|.38|.10|)" + when defined(osx): + # todo: find a better workaround for #10281 (caused by #10230) + const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + else: + const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.46|.45|.44|.43|.41|.39|.38|.10|)" when defined(macosx): const From f5cc2e2de567b36d33778ddf263b7a440f1d3b11 Mon Sep 17 00:00:00 2001 From: rec <44084068+recloser@users.noreply.github.com> Date: Sat, 12 Jan 2019 19:49:31 +0100 Subject: [PATCH 086/350] Fixes 10202 (#10283) * Add a test case for #10202 * Fix asgn for object tyVars; fixes #10202 * Check the variant kind before accessing the sym field --- compiler/jsgen.nim | 4 ++-- tests/js/t9410.nim | 37 +++++++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 2f5e202e0c..8625f2fe1d 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -925,7 +925,7 @@ const proc needsNoCopy(p: PProc; y: PNode): bool = return y.kind in nodeKindsNeedNoCopy or - ((mapType(y.typ) != etyBaseIndex or y.sym.kind == skParam) and + ((mapType(y.typ) != etyBaseIndex or (y.kind == nkSym and y.sym.kind == skParam)) and (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc} + IntegralTypes)) @@ -950,7 +950,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = lineF(p, "$1 = nimCopy(null, $2, $3);$n", [a.rdLoc, b.res, genTypeInfo(p, y.typ)]) of etyObject: - if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + if x.typ.kind == tyVar or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) else: useMagic(p, "nimCopy") diff --git a/tests/js/t9410.nim b/tests/js/t9410.nim index 9aca6d45b5..78c329a24c 100644 --- a/tests/js/t9410.nim +++ b/tests/js/t9410.nim @@ -1,13 +1,3 @@ -template doAssert(exp: untyped) = - when defined(echot9410): - let r = exp - echo $(instantiationInfo().line) & ":\n " & astToStr(exp) & "\n was " & repr(r) - when not defined(noassertt9410): - system.doAssert r - else: - when not defined(noassertt9410): - system.doAssert exp - template tests = block: var i = 0 @@ -428,6 +418,33 @@ template tests = let xptr2 = cast[type(xptr)](p2) doAssert xptr == xptr2 + + block: # var types + block t10202: + type Point = object + x: float + y: float + + var points: seq[Point] + + points.add(Point(x:1, y:2)) + + for i, p in points.mpairs: + p.x += 1 + + doAssert points[0].x == 2 + + block: + var ints = @[1, 2, 3] + for i, val in mpairs ints: + val *= 10 + doAssert ints == @[10, 20, 30] + + block: + var seqOfSeqs = @[@[1, 2], @[3, 4]] + for i, val in mpairs seqOfSeqs: + val[0] *= 10 + doAssert seqOfSeqs == @[@[10, 2], @[30, 4]] when false: block: # openarray From 9af85fb69f26dcae5fba7881a597d78f6bc7ffc8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jan 2019 00:00:39 -0800 Subject: [PATCH 087/350] fixes #10273 execShellCmd now returns nonzero when child killed with signal + other fixes (#10274) * s/exitStatus(...)/exitStatusLikeShell(...)/ * fix #10273 execShellCmd now returns nonzero when child exits with signal * test case for #10249 and explanation for the bug * fix test failure * add tests/nim.cfg --- .gitignore | 2 +- lib/pure/os.nim | 21 +++-- lib/pure/osproc.nim | 21 ++--- testament/lib/stdtest/specialpaths.nim | 31 +++++++ tests/nim.cfg | 1 + tests/stdlib/t10231.nim | 2 + tests/stdlib/tos.nim | 2 + tests/stdlib/tosproc.nim | 109 +++++++++++++++++++++---- 8 files changed, 148 insertions(+), 41 deletions(-) create mode 100644 testament/lib/stdtest/specialpaths.nim create mode 100644 tests/nim.cfg diff --git a/.gitignore b/.gitignore index 132cd7f60c..2bf0bf30c3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,7 @@ doc/*.html doc/*.pdf doc/*.idx /web/upload -build/* +/build/* bin/* # iOS specific wildcards. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 3a959d4e2c..32fa2885ea 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1287,6 +1287,17 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", discard tryRemoveFile(dest) raise +proc exitStatusLikeShell*(status: cint): cint = + ## converts exit code from `c_system` into a shell exit code + when defined(posix) and not weirdTarget: + if WIFSIGNALED(status): + # like the shell! + 128 + WTERMSIG(status) + else: + WEXITSTATUS(status) + else: + status + proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", tags: [ExecIOEffect], noNimScript.} = ## Executes a `shell command`:idx:. @@ -1295,14 +1306,8 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## line arguments given to program. The proc returns the error code ## of the shell when it has finished. The proc does not return until ## the process has finished. To execute a program without having a - ## shell involved, use the `execProcess` proc of the `osproc` - ## module. - when defined(macosx): - result = c_system(command) shr 8 - elif defined(posix): - result = WEXITSTATUS(c_system(command)) - else: - result = c_system(command) + ## shell involved, use `osproc.execProcess`. + result = exitStatusLikeShell(c_system(command)) # Templates for filtering directories and files when defined(windows) and not weirdTarget: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 72581f47cc..dfe75d9982 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -724,13 +724,6 @@ elif not defined(useNimRtl): proc isExitStatus(status: cint): bool = WIFEXITED(status) or WIFSIGNALED(status) - proc exitStatus(status: cint): cint = - if WIFSIGNALED(status): - # like the shell! - 128 + WTERMSIG(status) - else: - WEXITSTATUS(status) - proc envToCStringArray(t: StringTableRef): cstringArray = result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring))) var i = 0 @@ -1054,7 +1047,7 @@ elif not defined(useNimRtl): proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: - return exitStatus(p.exitStatus) + return exitStatusLikeShell(p.exitStatus) if timeout == -1: var status: cint = 1 @@ -1109,7 +1102,7 @@ elif not defined(useNimRtl): finally: discard posix.close(kqFD) - result = exitStatus(p.exitStatus) + result = exitStatusLikeShell(p.exitStatus) else: import times @@ -1142,7 +1135,7 @@ elif not defined(useNimRtl): s.tv_nsec = b.tv_nsec if p.exitFlag: - return exitStatus(p.exitStatus) + return exitStatusLikeShell(p.exitStatus) if timeout == -1: var status: cint = 1 @@ -1220,20 +1213,20 @@ elif not defined(useNimRtl): if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: raiseOSError(osLastError()) - result = exitStatus(p.exitStatus) + result = exitStatusLikeShell(p.exitStatus) proc peekExitCode(p: Process): int = var status = cint(0) result = -1 if p.exitFlag: - return exitStatus(p.exitStatus) + return exitStatusLikeShell(p.exitStatus) var ret = waitpid(p.id, status, WNOHANG) if ret > 0: if isExitStatus(status): p.exitFlag = true p.exitStatus = status - result = exitStatus(status) + result = exitStatusLikeShell(status) proc createStream(stream: var Stream, handle: var FileHandle, fileMode: FileMode) = @@ -1265,7 +1258,7 @@ elif not defined(useNimRtl): proc execCmd(command: string): int = when defined(linux): let tmp = csystem(command) - result = if tmp == -1: tmp else: exitStatus(tmp) + result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: result = csystem(command) diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim new file mode 100644 index 0000000000..3c8b2338cd --- /dev/null +++ b/testament/lib/stdtest/specialpaths.nim @@ -0,0 +1,31 @@ +#[ +todo: move findNimStdLibCompileTime, findNimStdLib here +]# + +import os + +# Note: all the const paths defined here are known at compile time and valid +# so long Nim repo isn't relocated after compilation. +# This means the binaries they produce will embed hardcoded paths, which +# isn't appropriate for some applications that need to be relocatable. + +const sourcePath = currentSourcePath() + # robust way to derive other paths here + # We don't depend on PATH so this is robust to having multiple nim + # binaries + +const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir + ## root of Nim repo + +const stdlibDir* = nimRootDir / "lib" + # todo: make nimeval.findNimStdLibCompileTime use this + +const systemPath* = stdlibDir / "system.nim" + +const buildDir* = nimRootDir / "build" + ## refs #10268: all testament generated files should go here to avoid + ## polluting .gitignore + +static: + # sanity check + doAssert fileExists(systemPath) diff --git a/tests/nim.cfg b/tests/nim.cfg new file mode 100644 index 0000000000..577baaacda --- /dev/null +++ b/tests/nim.cfg @@ -0,0 +1 @@ +--path:"../testament/lib" # so we can `import stdtest/foo` in this dir diff --git a/tests/stdlib/t10231.nim b/tests/stdlib/t10231.nim index 5d1101aa42..2bb64b475c 100644 --- a/tests/stdlib/t10231.nim +++ b/tests/stdlib/t10231.nim @@ -6,6 +6,8 @@ discard """ import os +# consider moving this inside tosproc (taking care that it's for cpp mode) + if paramCount() == 0: # main process doAssert execShellCmd(getAppFilename().quoteShell & " test") == 1 diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index a7cf5d5b6f..e4e14d5a14 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -263,3 +263,5 @@ block splitFile: doAssert splitFile("abc/.") == ("abc", ".", "") doAssert splitFile("..") == ("", "..", "") doAssert splitFile("a/..") == ("a", "..", "") + +# execShellCmd is tested in tosproc diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index 9d57d45742..7a7c1836f9 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -1,23 +1,96 @@ -discard """ - output: "" -""" # test the osproc module -import os, osproc +import stdtest/specialpaths +import "../.." / compiler/unittest_light -block execProcessTest: - let dir = parentDir(currentSourcePath()) - let (outp, err) = execCmdEx("nim c " & quoteShell(dir / "osproctest.nim")) - doAssert err == 0 - let exePath = dir / addFileExt("osproctest", ExeExt) - let outStr1 = execProcess(exePath, workingDir=dir, args=["foo", "b A r"], options={}) - doAssert outStr1 == dir & "\nfoo\nb A r\n" +when defined(case_testfile): # compiled test file for child process + from posix import exitnow + proc c_exit2(code: c_int): void {.importc: "_exit", header: "".} + import os + var a = 0 + proc fun(b = 0) = + a.inc + if a mod 10000000 == 0: # prevents optimizing it away + echo a + fun(b+1) - const testDir = "t e st" - createDir(testDir) - doAssert dirExists(testDir) - let outStr2 = execProcess(exePath, workingDir=testDir, args=["x yz"], options={}) - doAssert outStr2 == absolutePath(testDir) & "\nx yz\n" + proc main() = + let args = commandLineParams() + echo (msg: "child binary", pid: getCurrentProcessId()) + let arg = args[0] + echo (arg: arg) + case arg + of "exit_0": + if true: quit(0) + of "exitnow_139": + if true: exitnow(139) + of "c_exit2_139": + if true: c_exit2(139) + of "quit_139": + # `exitStatusLikeShell` doesn't distinguish between a process that + # exit(139) and a process that gets killed with `SIGSEGV` because + # 139 = 11 + 128 = SIGSEGV + 128. + # However, as #10249 shows, this leads to bad debugging experience + # when a child process dies with SIGSEGV, leaving no trace of why it + # failed. The shell (and lldb debugger) solves that by inserting a + # helpful msg: `segmentation fault` when it detects a signal killed + # the child. + # todo: expose an API that will show more diagnostic, returing + # (exitCode, signal) instead of just `shellExitCode`. + if true: quit(139) + of "exit_recursion": # stack overflow by infinite recursion + fun() + echo a + of "exit_array": # bad array access + echo args[1] + main() - removeDir(testDir) - removeFile(exePath) +else: + + import os, osproc, strutils, posix + + block execShellCmdTest: + ## first, compile child program + const nim = getCurrentCompilerExe() + const sourcePath = currentSourcePath() + let output = buildDir / "D20190111T024543".addFileExt(ExeExt) + let cmd = "$# c -o:$# -d:release -d:case_testfile $#" % [nim, output, + sourcePath] + # we're testing `execShellCmd` so don't rely on it to compile test file + # note: this should be exported in posix.nim + proc c_system(cmd: cstring): cint {.importc: "system", + header: "".} + assertEquals c_system(cmd), 0 + + ## use it + template runTest(arg: string, expected: int) = + echo (arg2: arg, expected2: expected) + assertEquals execShellCmd(output & " " & arg), expected + + runTest("exit_0", 0) + runTest("exitnow_139", 139) + runTest("c_exit2_139", 139) + runTest("quit_139", 139) + runTest("exit_array", 1) + when defined(posix): # on windows, -1073741571 + runTest("exit_recursion", SIGSEGV.int + 128) # bug #10273: was returning 0 + assertEquals exitStatusLikeShell(SIGSEGV), SIGSEGV + 128.cint + + block execProcessTest: + let dir = parentDir(currentSourcePath()) + let (outp, err) = execCmdEx("nim c " & quoteShell(dir / "osproctest.nim")) + doAssert err == 0 + let exePath = dir / addFileExt("osproctest", ExeExt) + let outStr1 = execProcess(exePath, workingDir = dir, args = ["foo", + "b A r"], options = {}) + doAssert outStr1 == dir & "\nfoo\nb A r\n" + + const testDir = "t e st" + createDir(testDir) + doAssert dirExists(testDir) + let outStr2 = execProcess(exePath, workingDir = testDir, args = ["x yz"], + options = {}) + doAssert outStr2 == absolutePath(testDir) & "\nx yz\n" + + removeDir(testDir) + removeFile(exePath) From 2823da5e528b55cb5178e1bc72183bf4638fbf28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Str=C3=B8mberg?= Date: Sun, 13 Jan 2019 09:04:40 +0100 Subject: [PATCH 088/350] setCommand nop update documentation (#10254) When I used Warning: Using `setCommand 'nop'` is not necessary. --- doc/nims.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/doc/nims.rst b/doc/nims.rst index 6cc894b913..eaf20a6dbd 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -117,17 +117,6 @@ Task Description ========= =================================================== -If the task runs an external command via ``exec`` it should afterwards call -``setCommand "nop"`` to tell the Nim compiler that nothing else needs to be -done: - -.. code-block:: nim - - task tests, "test regular expressions": - exec "nim c -r tests" - setCommand "nop" - - Look at the module `distros `_ for some support of the OS's native package managers. From 5cd18b0f240ff111e8892aef7bbab5037c3888bf Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 09:04:13 +0100 Subject: [PATCH 089/350] make system.task compatible with NimScript; refs #10254 --- lib/system/nimscript.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 34daf30a9e..ae97e88fa2 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -341,14 +341,15 @@ when not defined(nimble): ## .. code-block:: nim ## task build, "default build is via the C backend": ## setCommand "c" - proc `name Task`*() = body + proc `name Task`*() = + setCommand "nop" + body let cmd = getCommand() if cmd.len == 0 or cmd ==? "help": setCommand "help" writeTask(astToStr(name), description) elif cmd ==? astToStr(name): - setCommand "nop" `name Task`() # nimble has its own implementation for these things. From 40115cd6458457d071600526aa1037eaffe99f79 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 13:30:01 +0100 Subject: [PATCH 090/350] improve error messages quality for '.push: raises []' --- compiler/pragmas.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 4adf6032b8..3967fa22dc 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1123,7 +1123,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, else: invalidPragma(c, it) +proc overwriteLineInfo(n: PNode; info: TLineInfo) = + n.info = info + for i in 0.. Date: Sun, 13 Jan 2019 13:54:44 +0100 Subject: [PATCH 091/350] make tests more robust; tests should be deterministic, no randomize() calls in tests --- tests/stdlib/tmath.nim | 24 ++++++++++++------------ tests/stdlib/tosproc.nim | 5 ++++- tests/stdlib/tunittest.nim | 1 - 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 544fa8dc09..bdb5aa3329 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -20,29 +20,29 @@ import sets, tables suite "random int": test "there might be some randomness": var set = initSet[int](128) - randomize() + for i in 1..1000: incl(set, random(high(int))) check len(set) == 1000 test "single number bounds work": - randomize() + var rand: int for i in 1..1000: rand = random(1000) check rand < 1000 check rand > -1 test "slice bounds work": - randomize() + var rand: int for i in 1..1000: rand = random(100..1000) check rand < 1000 check rand >= 100 - test "randomize() again gives new numbers": - randomize() + test " again gives new numbers": + var rand1 = random(1000000) os.sleep(200) - randomize() + var rand2 = random(1000000) check rand1 != rand2 @@ -50,29 +50,29 @@ suite "random int": suite "random float": test "there might be some randomness": var set = initSet[float](128) - randomize() + for i in 1..100: incl(set, random(1.0)) check len(set) == 100 test "single number bounds work": - randomize() + var rand: float for i in 1..1000: rand = random(1000.0) check rand < 1000.0 check rand > -1.0 test "slice bounds work": - randomize() + var rand: float for i in 1..1000: rand = random(100.0..1000.0) check rand < 1000.0 check rand >= 100.0 - test "randomize() again gives new numbers": - randomize() + test " again gives new numbers": + var rand1:float = random(1000000.0) os.sleep(200) - randomize() + var rand2:float = random(1000000.0) check rand1 != rand2 diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index 7a7c1836f9..b8d3be9bbf 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -93,4 +93,7 @@ else: doAssert outStr2 == absolutePath(testDir) & "\nx yz\n" removeDir(testDir) - removeFile(exePath) + try: + removeFile(exePath) + except OSError: + discard diff --git a/tests/stdlib/tunittest.nim b/tests/stdlib/tunittest.nim index c8656bbff5..65baefef0b 100644 --- a/tests/stdlib/tunittest.nim +++ b/tests/stdlib/tunittest.nim @@ -50,7 +50,6 @@ test "unittest multiple requires": import math, random from strutils import parseInt proc defectiveRobot() = - randomize() case random(1..4) of 1: raise newException(OSError, "CANNOT COMPUTE!") of 2: discard parseInt("Hello World!") From d740a5b79a004632f90da6702cd351ae214e273d Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 13 Jan 2019 13:06:16 +0000 Subject: [PATCH 092/350] Implements BackwardsIndex `[]` for deque. (#10105) --- lib/pure/collections/deques.nim | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index e8342e2084..5243b96a64 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -88,11 +88,23 @@ proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = xBoundsCheck(deq, i) return deq.data[(deq.head + i) and deq.mask] -proc `[]=`* [T] (deq: var Deque[T], i: Natural, val : T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: Natural, val : T) {.inline.} = ## Change the i-th element of `deq`. xBoundsCheck(deq, i) deq.data[(deq.head + i) and deq.mask] = val +proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = + ## Access the backwards indexed i-th element. + return deq[deq.len - int(i)] + +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = + ## Access the backwards indexed i-th element. + return deq[deq.len - int(i)] + +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = + ## Change the backwards indexed i-th element. + deq[deq.len - int(i)] = x + iterator items*[T](deq: Deque[T]): T = ## Yield every element of `deq`. var i = deq.head From 0c10fc67eb259ede0de4501cee5fe81463064294 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 14:05:09 +0100 Subject: [PATCH 093/350] fixes #10104 --- compiler/vm.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index e2586e615b..c001981f89 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -85,7 +85,7 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = add(s, x.prc.name.s) msgWriteln(c.config, s) -proc stackTrace(c: PCtx, tos: PStackFrame, pc: int, +proc stackTraceImpl(c: PCtx, tos: PStackFrame, pc: int, msg: string, lineInfo: TLineInfo) = msgWriteln(c.config, "stack trace: (most recent call last)") stackTraceAux(c, tos, pc) @@ -93,8 +93,14 @@ proc stackTrace(c: PCtx, tos: PStackFrame, pc: int, if c.mode == emRepl: globalError(c.config, lineInfo, msg) else: localError(c.config, lineInfo, msg) -proc stackTrace(c: PCtx, tos: PStackFrame, pc: int, msg: string) = - stackTrace(c, tos, pc, msg, c.debug[pc]) +template stackTrace(c: PCtx, tos: PStackFrame, pc: int, + msg: string, lineInfo: TLineInfo) = + stackTraceImpl(c, tos, pc, msg, lineInfo) + return + +template stackTrace(c: PCtx, tos: PStackFrame, pc: int, msg: string) = + stackTraceImpl(c, tos, pc, msg, c.debug[pc]) + return proc bailOut(c: PCtx; tos: PStackFrame) = stackTrace(c, tos, c.exceptionInstr, "unhandled exception: " & @@ -955,13 +961,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeBC(rkInt) let a = regs[rb].node let b = regs[rc].node - if a.kind == nkSym and a.sym.kind in skProcKinds and + if a.kind == nkSym and a.sym.kind in skProcKinds and b.kind == nkSym and b.sym.kind in skProcKinds: regs[ra].intVal = if sfFromGeneric in a.sym.flags and a.sym.owner == b.sym: 1 else: 0 - else: - stackTrace(c, tos, pc, "node is not a proc symbol") + else: + stackTrace(c, tos, pc, "node is not a proc symbol") of opcEcho: let rb = instr.regB if rb == 1: From 7f0559b93fbd21283119f545ac8bf805d1a6eeb4 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 14:08:49 +0100 Subject: [PATCH 094/350] fixes #7241 --- lib/impure/db_sqlite.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index c7e3730984..3910992bb1 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -166,10 +166,12 @@ iterator fastRows*(db: DbConn, query: SqlQuery, var stmt = setupQuery(db, query, args) var L = (column_count(stmt)) var result = newRow(L) - while step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - yield result - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + setRow(stmt, result, L) + yield result + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow @@ -177,9 +179,11 @@ iterator instantRows*(db: DbConn, query: SqlQuery, ## same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the iterator body. var stmt = setupQuery(db, query, args) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + yield stmt + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) proc toTypeKind(t: var DbType; x: int32) = case x @@ -210,9 +214,11 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, ## on demand using []. Returned handle is valid only within the iterator body. var stmt = setupQuery(db, query, args) setColumns(columns, stmt) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + yield stmt + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) proc `[]`*(row: InstantRow, col: int32): string {.inline.} = ## returns text for given column of the row From 451377f737ab4b21a96c1ca3f3646c86c275a2b3 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 14:24:38 +0100 Subject: [PATCH 095/350] fixes #10271 [backport] --- lib/system/gc_ms.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d8cb3e2d0e..aa5fb6aea1 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -485,8 +485,9 @@ proc collectCTBody(gch: var GcHeap) = sysAssert(allocInv(gch.region), "collectCT: end") proc collectCT(gch: var GcHeap; size: int) = + let fmem = getFreeMem(gch.region) if (getOccupiedMem(gch.region) >= gch.cycleThreshold or - size > getFreeMem(gch.region)) and gch.recGcLock == 0: + size > fmem and fmem > InitialThreshold) and gch.recGcLock == 0: collectCTBody(gch) when not defined(useNimRtl): From ab99bdfc408e56a2e8a1a3ada21effe983433851 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 14:56:50 +0100 Subject: [PATCH 096/350] fixes #10136 --- compiler/semexprs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8852cb6535..657df36ddc 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -403,8 +403,8 @@ proc semIs(c: PContext, n: PNode, flags: TExprFlags): PNode = n[1] = makeTypeSymNode(c, lhsType, n[1].info) lhsType = n[1].typ else: - internalAssert c.config, lhsType.base.kind != tyNone - if c.inGenericContext > 0 and lhsType.base.containsGenericType: + if lhsType.base.kind == tyNone or + (c.inGenericContext > 0 and lhsType.base.containsGenericType): # BUGFIX: don't evaluate this too early: ``T is void`` return From aa7ad8897895acaee9e2999652050d5bc52921a7 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 15:52:50 +0100 Subject: [PATCH 097/350] fixes #10075 [backport] --- compiler/sigmatch.nim | 3 +-- tests/macros/tvarargsuntyped.nim | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 0915f303b3..9aae254f33 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2267,8 +2267,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, if n.sons[a].kind == nkHiddenStdConv: doAssert n.sons[a].sons[0].kind == nkEmpty and - n.sons[a].sons[1].kind == nkArgList and - n.sons[a].sons[1].len == 0 + n.sons[a].sons[1].kind in {nkBracket, nkArgList} # Steal the container and pass it along setSon(m.call, formal.position + 1, n.sons[a].sons[1]) else: diff --git a/tests/macros/tvarargsuntyped.nim b/tests/macros/tvarargsuntyped.nim index 657ed47d68..f0fcff662c 100644 --- a/tests/macros/tvarargsuntyped.nim +++ b/tests/macros/tvarargsuntyped.nim @@ -3,7 +3,9 @@ discard """ (left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8) (left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 1, g: 7, b: 8) (left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 4, g: 7, b: 8) -(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8)''' +(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8) +10 +hello 18.0''' """ import macros @@ -78,3 +80,30 @@ let width: cint = 3 let height: cint = 4 bar(rect(top, left, width, height), "test", point(8, 9), color(7,7,8)) + + +# bug #10075 + +import macros + +proc convert_hidden_stdconv(args: NimNode): NimNode = + var n = args + while n.len == 1 and n[0].kind == nnkHiddenStdConv: + n = n[0][1] + return n + +macro t2(s: int, v: varargs[untyped]): untyped = + let v = convert_hidden_stdconv(v) + echo v.treeRepr + let (v1, v2) = (v[0], v[1]) + quote do: + echo `v1`, " ", `v2` + +template t1(s: int, v: varargs[typed]) = + #static: + # dumpTree v + echo s + t2(s, v) + +when isMainModule: + t1(10, "hello", 18.0) From 2ccc9db59d849f3dc5613236eea4a47c9ab19be1 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 16:05:42 +0100 Subject: [PATCH 098/350] closes #3744 --- tests/macros/tquotedo.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/macros/tquotedo.nim b/tests/macros/tquotedo.nim index cd1f691161..0aae87bf04 100644 --- a/tests/macros/tquotedo.nim +++ b/tests/macros/tquotedo.nim @@ -3,6 +3,7 @@ output: ''' 123 Hallo Welt Hallo Welt +1 ''' """ @@ -23,3 +24,13 @@ macro foobar(arg: untyped): untyped = foobar: echo "Hallo Welt" + +# bug #3744 +import macros +macro t(): untyped = + return quote do: + proc tp(): int = + result = 1 +t() + +echo tp() From bd5dd2b95bb17a07d1400d677e80c0514fef69d0 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 16:22:19 +0100 Subject: [PATCH 099/350] nimgrep: fixes #989 --- tools/nimgrep.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index 8c6353e319..b26d862b6a 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -160,7 +160,7 @@ proc processFile(pattern; filename: string; counter: var int) = var reallyReplace = true while i < buffer.len: let t = findBounds(buffer, pattern, matches, i) - if t.first < 0: break + if t.first < 0 or t.last < t.first: break inc(line, countLines(buffer, i, t.first-1)) var wholeMatch = buffer.substr(t.first, t.last) From 80c69ffd31ec896497479203d3c282a5ba08d9c4 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 16:23:08 +0100 Subject: [PATCH 100/350] nimgrep: use standard regular expression syntax, not reExtended --- tools/nimgrep.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index b26d862b6a..0a835b038a 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -336,7 +336,7 @@ else: for f in items(filenames): walker(pegp, f, counter) else: - var reflags = {reStudy, reExtended} + var reflags = {reStudy} if optIgnoreStyle in options: pattern = styleInsensitive(pattern) if optWord in options: From 3c2d82eaae8054e1f52201d1429214dd15cfef95 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 17:19:36 +0100 Subject: [PATCH 101/350] make megatest green --- tests/macros/tvarargsuntyped.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/macros/tvarargsuntyped.nim b/tests/macros/tvarargsuntyped.nim index f0fcff662c..5a06adcca3 100644 --- a/tests/macros/tvarargsuntyped.nim +++ b/tests/macros/tvarargsuntyped.nim @@ -105,5 +105,4 @@ template t1(s: int, v: varargs[typed]) = echo s t2(s, v) -when isMainModule: - t1(10, "hello", 18.0) +t1(10, "hello", 18.0) From d983d998b25dc1eda004b2483d620315e9ea64b2 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 13 Jan 2019 18:53:27 +0100 Subject: [PATCH 102/350] fixes #9218 --- lib/nimbase.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index 2cb632787a..ba42737260 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -166,13 +166,13 @@ __clang__ # define N_STDCALL(rettype, name) rettype __stdcall name # define N_SYSCALL(rettype, name) rettype __syscall name # define N_FASTCALL(rettype, name) rettype __fastcall name -# define N_SAFECALL(rettype, name) rettype __safecall name +# define N_SAFECALL(rettype, name) rettype __stdcall name /* function pointers with calling convention: */ # define N_CDECL_PTR(rettype, name) rettype (__cdecl *name) # define N_STDCALL_PTR(rettype, name) rettype (__stdcall *name) # define N_SYSCALL_PTR(rettype, name) rettype (__syscall *name) # define N_FASTCALL_PTR(rettype, name) rettype (__fastcall *name) -# define N_SAFECALL_PTR(rettype, name) rettype (__safecall *name) +# define N_SAFECALL_PTR(rettype, name) rettype (__stdcall *name) # ifdef __cplusplus # define N_LIB_EXPORT extern "C" __declspec(dllexport) From 2b37bf5d5bf9e9ee7cd387fa455df8931f8e6666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Mon, 14 Jan 2019 09:20:31 +0100 Subject: [PATCH 103/350] added proc rand for typedesc (#10259) --- lib/pure/random.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index ee728ad4aa..378ca6f87b 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -158,10 +158,14 @@ proc rand*[T](x: HSlice[T, T]): T = result = rand(state, x) proc rand*[T](r: var Rand; a: openArray[T]): T {.deprecated.} = - ## returns a random element from the openarray `a`. + ## Returns a random element from the openarray `a`. ## **Deprecated since v0.20.0:** use ``sample`` instead. result = a[rand(r, a.low..a.high)] +proc rand*[T: SomeInteger](t: typedesc[T]): T = + ## Returns a random integer in the range `low(T)..high(T)`. + result = cast[T](state.next) + proc rand*[T](a: openArray[T]): T {.deprecated.} = ## returns a random element from the openarray `a`. ## **Deprecated since v0.20.0:** use ``sample`` instead. From 29e0a792ffff7a08636f0bb50709a4d491cdf19b Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 14 Jan 2019 09:24:12 +0100 Subject: [PATCH 104/350] [backport] fix documentation leak to `os.nim`[ci skip] --- lib/pure/includes/osenv.nim | 2 +- lib/pure/includes/oserr.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 4acc36b93d..945555540f 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -1,4 +1,4 @@ -## Include file that implements 'getEnv' and friends. Do not import it! +# Include file that implements 'getEnv' and friends. Do not import it! when not declared(os): {.error: "This is an include file for os.nim!".} diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index abd0bf501f..9ae06f5bc5 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -1,4 +1,4 @@ -## Include file that implements 'osErrorMsg' and friends. Do not import it! +# Include file that implements 'osErrorMsg' and friends. Do not import it! when not declared(os): {.error: "This is an include file for os.nim!".} From c707267212a8e5cd5056296acf6502d58c9e5de9 Mon Sep 17 00:00:00 2001 From: genotrance Date: Mon, 14 Jan 2019 02:26:10 -0600 Subject: [PATCH 105/350] Add stdin read support to nimscrit #3983 (#10292) --- compiler/scriptconfig.nim | 15 ++++++++++++--- lib/system/nimscript.nim | 29 ++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index e3a9478d13..db60dafcce 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -42,15 +42,18 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; proc (a: VmArgs) = body - template cbos(name, body) {.dirty.} = + template cbexc(name, exc, body) {.dirty.} = result.registerCallback "stdlib.system." & astToStr(name), proc (a: VmArgs) = errorMsg = "" try: body - except OSError: + except exc: errorMsg = getCurrentExceptionMsg() + template cbos(name, body) {.dirty.} = + cbexc(name, OSError, body) + # Idea: Treat link to file as a file, but ignore link to directory to prevent # endless recursions out of the box. cbos listFiles: @@ -64,7 +67,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; cbos createDir: os.createDir getString(a, 0) - result.registerCallback "stdlib.system.getOsError", + result.registerCallback "stdlib.system.getError", proc (a: VmArgs) = setResult(a, errorMsg) cbos setCurrentDir: @@ -157,6 +160,12 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; setResult(a, os.getAppFilename()) cbconf cppDefine: options.cppDefine(conf, a.getString(0)) + cbexc stdinReadLine, EOFError: + setResult(a, "") + setResult(a, stdin.readLine()) + cbexc stdinReadAll, EOFError: + setResult(a, "") + setResult(a, stdin.readAll()) proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; freshDefines=true; conf: ConfigRef) = diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index ae97e88fa2..e879fda833 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -46,7 +46,7 @@ proc copyDir(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin -proc getOsError: string = builtin +proc getError: string = builtin proc setCurrentDir(dir: string) = builtin proc getCurrentDir*(): string = ## Retrieves the current working directory. @@ -178,9 +178,12 @@ var mode*: ScriptMode ## Set this to influence how mkDir, rmDir, rmFile etc. ## behave +template checkError(exc: untyped): untyped = + let err = getError() + if err.len > 0: raise newException(exc, err) + template checkOsError = - let err = getOsError() - if err.len > 0: raise newException(OSError, err) + checkError(OSError) template log(msg: string, body: untyped) = if mode in {ScriptMode.Verbose, ScriptMode.Whatif}: @@ -332,6 +335,26 @@ proc cppDefine*(define: string) = ## needs to be mangled. builtin +proc stdinReadLine(): TaintedString {. + tags: [ReadIOEffect], raises: [IOError].} = + builtin + +proc stdinReadAll(): TaintedString {. + tags: [ReadIOEffect], raises: [IOError].} = + builtin + +proc readLineFromStdin*(): TaintedString {.raises: [IOError].} = + ## Reads a line of data from stdin - blocks until \n or EOF which happens when stdin is closed + log "readLineFromStdin": + result = stdinReadLine() + checkError(EOFError) + +proc readAllFromStdin*(): TaintedString {.raises: [IOError].} = + ## Reads all data from stdin - blocks until EOF which happens when stdin is closed + log "readAllFromStdin": + result = stdinReadAll() + checkError(EOFError) + when not defined(nimble): template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 template task*(name: untyped; description: string; body: untyped): untyped = From 65593e76f44176e3a28d4d60aff79e58042244ad Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 14 Jan 2019 02:58:16 -0800 Subject: [PATCH 106/350] fixes leftover from #9647 : use nimf as standardized extension for source code filters (#10294) --- testament/htmlgen.nim | 2 +- testament/{testamenthtml.templ => testamenthtml.nimf} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename testament/{testamenthtml.templ => testamenthtml.nimf} (100%) diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim index 34902f71e1..641c1c32c9 100644 --- a/testament/htmlgen.nim +++ b/testament/htmlgen.nim @@ -11,7 +11,7 @@ import cgi, backend, strutils, json, os, tables, times -import "testamenthtml.templ" +import "testamenthtml.nimf" proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = let diff --git a/testament/testamenthtml.templ b/testament/testamenthtml.nimf similarity index 100% rename from testament/testamenthtml.templ rename to testament/testamenthtml.nimf From 825e08b04603a3839ba6d0b915aedfb08418b7d3 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 14 Jan 2019 12:15:29 +0100 Subject: [PATCH 107/350] fixes #7524 --- compiler/parampatterns.nim | 15 +++++++-------- compiler/semtypes.nim | 7 ++++++- tests/trmacros/trmacros_various2.nim | 10 ++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index bbaf7a0698..db79e3eb98 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -115,20 +115,19 @@ proc compileConstraints(p: PNode, result: var TPatternCode; conf: ConfigRef) = else: patternError(p, conf) -proc semNodeKindConstraints*(p: PNode; conf: ConfigRef): PNode = +proc semNodeKindConstraints*(n: PNode; conf: ConfigRef; start: Natural): PNode = ## does semantic checking for a node kind pattern and compiles it into an ## efficient internal format. - assert p.kind == nkCurlyExpr - result = newNodeI(nkStrLit, p.info) + result = newNodeI(nkStrLit, n.info) result.strVal = newStringOfCap(10) result.strVal.add(chr(aqNone.ord)) - if p.len >= 2: - for i in 1..= 2: + for i in start.. MaxStackSize-1: - internalError(conf, p.info, "parameter pattern too complex") + internalError(conf, n.info, "parameter pattern too complex") else: - patternError(p, conf) + patternError(n, conf) result.strVal.add(ppEof) type diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f4936a71aa..7056eab5f2 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1028,7 +1028,12 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, proc semParamType(c: PContext, n: PNode, constraint: var PNode): PType = if n.kind == nkCurlyExpr: result = semTypeNode(c, n.sons[0], nil) - constraint = semNodeKindConstraints(n, c.config) + constraint = semNodeKindConstraints(n, c.config, 1) + elif n.kind == nkCall and + n[0].kind in {nkIdent, nkSym, nkOpenSymChoice, nkClosedSymChoice} and + considerQuotedIdent(c, n[0]).s == "{}": + result = semTypeNode(c, n[1], nil) + constraint = semNodeKindConstraints(n, c.config, 2) else: result = semTypeNode(c, n, nil) diff --git a/tests/trmacros/trmacros_various2.nim b/tests/trmacros/trmacros_various2.nim index d500c49dea..c1367cb1be 100644 --- a/tests/trmacros/trmacros_various2.nim +++ b/tests/trmacros/trmacros_various2.nim @@ -77,3 +77,13 @@ block tstar: # check that it's been optimized properly: doAssert calls == 1 + +# bug #7524 +template in_to_out(typIn, typOut: typedesc) = + proc to_out(x: typIn{lit}): typOut = result = ord(x) + +# Generating the proc via template doesn't work +in_to_out(char, int) + +# This works +proc to_out2(x: char{lit}): int = result = ord(x) From d69a7842fa0836ca18b8057a33bca81a771a9e5a Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 14 Jan 2019 12:36:34 +0100 Subject: [PATCH 108/350] fixes #7878 --- compiler/ccgexprs.nim | 2 +- tests/stdlib/trepr.nim | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index d00371dd83..ed62550040 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1501,7 +1501,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = addrLoc(p.config, a), genTypeInfo(p.module, t, e.info)]), a.storage) of tyOpenArray, tyVarargs: var b: TLoc - case a.t.kind + case skipTypes(a.t, abstractVarRange).kind of tyOpenArray, tyVarargs: putIntoDest(p, b, e, "$1, $1Len_0" % [rdLoc(a)], a.storage) of tyString, tySequence: diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 33cb581ef9..c1941bd38e 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -1,5 +1,6 @@ discard """ - output: "{a, b}{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}" + output: '''{a, b}{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'} +[1, 2, 3, 4, 5, 6]''' """ type @@ -25,3 +26,11 @@ when false: # "a", "b", "c", "d", "e" #] #echo(repr(testseq)) + +# bug #7878 +proc test(variable: var openarray[int]) = + echo repr(variable) + +var arr = [1, 2, 3, 4, 5, 6] + +test(arr) From b78af990b8f656cb17044c96609f58e1f01e6a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Mon, 14 Jan 2019 17:16:17 +0100 Subject: [PATCH 109/350] Fixes #10065 (#10260) CountTable now returns 0 instead of 'key not found' for get requests. --- lib/pure/collections/tables.nim | 97 +++++++++++++++------------------ tests/stdlib/tmget.nim | 8 +-- 2 files changed, 47 insertions(+), 58 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index f46a368b1a..1fa2ca0a60 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -933,52 +933,6 @@ proc rawGet[A](t: CountTable[A], key: A): int = h = nextTry(h, high(t.data)) result = -1 - h # < 0 => MISSING; insert idx = -1 - result -template ctget(t, key: untyped): untyped = - var index = rawGet(t, key) - if index >= 0: result = t.data[index].val - else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") - -proc `[]`*[A](t: CountTable[A], key: A): int {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If ``key`` is not in ``t``, - ## the ``KeyError`` exception is raised. One can check with ``hasKey`` - ## whether the key exists. - ctget(t, key) - -proc `[]`*[A](t: var CountTable[A], key: A): var int {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ctget(t, key) - -proc mget*[A](t: var CountTable[A], key: A): var int {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - ctget(t, key) - -proc getOrDefault*[A](t: CountTable[A], key: A): int = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, 0 (the - ## default initialization value of ``int``), is returned. - var index = rawGet(t, key) - if index >= 0: result = t.data[index].val - -proc getOrDefault*[A](t: CountTable[A], key: A, default: int): int = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the - ## integer value of ``default`` is returned. - var index = rawGet(t, key) - result = if index >= 0: t.data[index].val else: default - -proc hasKey*[A](t: CountTable[A], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - result = rawGet(t, key) >= 0 - -proc contains*[A](t: CountTable[A], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A](t, key) - proc rawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]], key: A, val: int) = var h: Hash = hash(key) and high(data) @@ -996,16 +950,40 @@ proc enlarge[A](t: var CountTable[A]) = proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = ## puts a ``(key, value)`` pair into ``t``. assert val >= 0 - var h = rawGet(t, key) + let h = rawGet(t, key) if h >= 0: t.data[h].val = val else: if mustRehash(len(t.data), t.counter): enlarge(t) rawInsert(t, t.data, key, val) inc(t.counter) - #h = -1 - h - #t.data[h].key = key - #t.data[h].val = val + +template ctget(t, key, default: untyped): untyped = + var index = rawGet(t, key) + result = if index >= 0: t.data[index].val else: default + +proc `[]`*[A](t: CountTable[A], key: A): int = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise ``0`` is returned. + ctget(t, key, 0) + +proc mget*[A](t: var CountTable[A], key: A): var int = + ## Retrieves the value at ``t[key]``. The value can be modified. + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + get(t, key) + +proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = + ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the + ## integer value of ``default`` is returned. + ctget(t, key, default) + +proc hasKey*[A](t: CountTable[A], key: A): bool = + ## returns true iff ``key`` is in the table ``t``. + result = rawGet(t, key) >= 0 + +proc contains*[A](t: CountTable[A], key: A): bool = + ## Alias of ``hasKey`` for use with the ``in`` operator. + return hasKey[A](t, key) proc inc*[A](t: var CountTable[A], key: A, val = 1) = ## increments ``t[key]`` by ``val``. @@ -1113,16 +1091,15 @@ iterator mvalues*[A](t: CountTableRef[A]): var int = for h in 0..high(t.data): if t.data[h].val != 0: yield t.data[h].val -proc `[]`*[A](t: CountTableRef[A], key: A): var int {.deprecatedGet.} = +proc `[]`*[A](t: CountTableRef[A], key: A): int = ## retrieves the value at ``t[key]``. The value can be modified. ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. result = t[][key] -proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} = +proc mget*[A](t: CountTableRef[A], key: A): var int = ## retrieves the value at ``t[key]``. The value can be modified. ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - result = t[][key] + mget(t[], key) proc getOrDefault*[A](t: CountTableRef[A], key: A): int = ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, 0 (the @@ -1394,6 +1371,18 @@ when isMainModule: let t = toCountTable([0, 0, 5, 5, 5]) doAssert t.smallest == (0, 2) + block: #10065 + let t = toCountTable("abracadabra") + doAssert t['z'] == 0 + + var t_mut = toCountTable("abracadabra") + doAssert t_mut['z'] == 0 + # the previous read may not have modified the table. + doAssert t_mut.hasKey('z') == false + t_mut['z'] = 1 + doAssert t_mut['z'] == 1 + doAssert t_mut.hasKey('z') == true + block: var tp: Table[string, string] = initTable[string, string]() doAssert "test1" == tp.getOrDefault("test1", "test1") diff --git a/tests/stdlib/tmget.nim b/tests/stdlib/tmget.nim index 5792b62827..5e2e327f44 100644 --- a/tests/stdlib/tmget.nim +++ b/tests/stdlib/tmget.nim @@ -11,10 +11,10 @@ Can't access 6 Can't access 6 10 11 -Can't access 6 +0 10 11 -Can't access 6 +0 10 11 Can't access 6 @@ -85,7 +85,7 @@ block: except KeyError: echo "Can't access 6" echo x[5] - x[5] += 1 + x.inc 5, 1 var c = x[5] echo c @@ -97,7 +97,7 @@ block: except KeyError: echo "Can't access 6" echo x[5] - x[5] += 1 + x.inc 5, 1 var c = x[5] echo c From 1899d8d107e2f5d991fd3cad321c959768bdf8c8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 14 Jan 2019 20:11:33 +0100 Subject: [PATCH 110/350] make tests green again --- tests/metatype/tmetatype_issues.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/metatype/tmetatype_issues.nim b/tests/metatype/tmetatype_issues.nim index 08d486de2e..c184689a12 100644 --- a/tests/metatype/tmetatype_issues.nim +++ b/tests/metatype/tmetatype_issues.nim @@ -9,6 +9,7 @@ asd Foo Bar ''' +joinable: false """ import typetraits, macros From dd1f23f6fc9c3d57f5d3e2a44d2abe4dc1b57a2a Mon Sep 17 00:00:00 2001 From: genotrance Date: Mon, 14 Jan 2019 15:16:11 -0600 Subject: [PATCH 111/350] Expose CTime (#10301) --- lib/pure/times.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 8fe04a4b98..166dfd446c 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -217,7 +217,7 @@ when defined(JS): elif defined(posix): import posix - type CTime = posix.Time + type CTime* = posix.Time var realTimeClockId {.importc: "CLOCK_REALTIME", header: "".}: Clockid @@ -236,10 +236,10 @@ elif defined(windows): import winlean when defined(i386) and defined(gcc): - type CTime {.importc: "time_t", header: "".} = distinct int32 + type CTime* {.importc: "time_t", header: "".} = distinct int32 else: # newest version of Visual C++ defines time_t to be of 64 bits - type CTime {.importc: "time_t", header: "".} = distinct int64 + type CTime* {.importc: "time_t", header: "".} = distinct int64 # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "".}: int From e17321aa2429c6bed97bef28a149fd21166b90a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 12 Jan 2019 15:51:44 -0800 Subject: [PATCH 112/350] improve formatting in assertEquals --- compiler/unittest_light.nim | 15 ++++++++------- tests/compiler/tunittest_light.nim | 22 +++++++++++----------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/compiler/unittest_light.nim b/compiler/unittest_light.nim index bcba6f7c73..d9842b3994 100644 --- a/compiler/unittest_light.nim +++ b/compiler/unittest_light.nim @@ -14,8 +14,8 @@ proc mismatch*[T](lhs: T, rhs: T): string = proc quoted(s: string): string = result.addQuoted s result.add "\n" - result.add "lhs:{\n" & replaceInvisible( - $lhs) & "}\nrhs:{\n" & replaceInvisible($rhs) & "}\n" + result.add "lhs:{" & replaceInvisible( + $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n" when compiles(lhs.len): if lhs.len != rhs.len: result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & "\n" @@ -26,12 +26,13 @@ proc mismatch*[T](lhs: T, rhs: T): string = i.inc result.add "first mismatch index: " & $i & "\n" if i < lhs.len and i < rhs.len: - result.add "lhs[i]: {" & quoted($lhs[i]) & "} rhs[i]: {" & quoted( - $rhs[i]) & "}" - result.add "lhs[0.. Date: Sat, 12 Jan 2019 15:26:22 -0800 Subject: [PATCH 113/350] fix #9842 #9951: `nim -r` and parseopt.cmdLineRest are now correct --- compiler/lineinfos.nim | 2 +- lib/pure/parseopt.nim | 24 +------ tests/misc/tparseopt.nim | 136 ++++++++++++++++++++++++++------------- 3 files changed, 94 insertions(+), 68 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 21ce44406e..165d758218 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -171,7 +171,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] = warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats} result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf, - hintProcessing, hintPattern, hintExecuting, hintLinking} + hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC} const NotesVerbosity* = computeNotesVerbosity() diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index eba915604e..06f32f0326 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -73,24 +73,6 @@ proc parseWord(s: string, i: int, w: var string, inc(result) when declared(os.paramCount): - proc quote(s: string): string = - if find(s, {' ', '\t'}) >= 0 and s.len > 0 and s[0] != '"': - if s[0] == '-': - result = newStringOfCap(s.len) - var i = parseWord(s, 0, result, {' ', '\t', ':', '='}) - if i < s.len and s[i] in {':','='}: - result.add s[i] - inc i - result.add '"' - while i < s.len: - result.add s[i] - inc i - result.add '"' - else: - result = '"' & s & '"' - else: - result = s - # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! @@ -228,11 +210,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = when declared(os.paramCount): proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = ## retrieves the rest of the command line that has not been parsed yet. - var res = "" - for i in p.idx.. p.idx: res.add ' ' - res.add quote(p.cmds[i]) - result = res.TaintedString + result = p.cmds[p.idx .. ^1].quoteShellCommand.TaintedString proc remainingArgs*(p: OptParser): seq[TaintedString] {.rtl, extern: "npo$1".} = ## retrieves the rest of the command line that has not been parsed yet. diff --git a/tests/misc/tparseopt.nim b/tests/misc/tparseopt.nim index cbed5d4768..39c17869cc 100644 --- a/tests/misc/tparseopt.nim +++ b/tests/misc/tparseopt.nim @@ -30,54 +30,102 @@ kind: cmdLongOption key:val -- debug:3 kind: cmdShortOption key:val -- l:4 kind: cmdShortOption key:val -- r:2''' """ -from parseopt import nil -from parseopt2 import nil +when defined(testament_tparseopt): + import os + proc main() = + let args = commandLineParams() + echo args + for i, ai in args: + echo "arg ", i, " ai.len:", ai.len, " :{", ai, "}" + main() +else: + from parseopt import nil + from parseopt2 import nil -block: - echo "parseopt" - for kind, key, val in parseopt.getopt(): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val + block: + echo "parseopt" + for kind, key, val in parseopt.getopt(): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val - # pass custom cmdline arguments - echo "first round" - var argv = "--left --debug:3 -l=4 -r:2" - var p = parseopt.initOptParser(argv) - for kind, key, val in parseopt.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val - break - # reset getopt iterator and check arguments are returned correctly. - echo "second round" - for kind, key, val in parseopt.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val + # pass custom cmdline arguments + echo "first round" + var argv = "--left --debug:3 -l=4 -r:2" + var p = parseopt.initOptParser(argv) + for kind, key, val in parseopt.getopt(p): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val + break + # reset getopt iterator and check arguments are returned correctly. + echo "second round" + for kind, key, val in parseopt.getopt(p): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val - # bug #9619 - var x = parseopt.initOptParser(@["--foo:", "--path"], allowWhitespaceAfterColon = false) - for kind, key, val in parseopt.getopt(x): - echo kind, " ", key + # bug #9619 + var x = parseopt.initOptParser(@["--foo:", "--path"], + allowWhitespaceAfterColon = false) + for kind, key, val in parseopt.getopt(x): + echo kind, " ", key -block: - echo "parseoptNoVal" - # test NoVal mode with custom cmdline arguments - var argv = "--left --debug:3 -l -r:2 --debug 2 --debug=1 -r1 -r=0 -lr4" - var p = parseopt.initOptParser(argv, - shortNoVal = {'l'}, longNoVal = @["left"]) - for kind, key, val in parseopt.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val + block: + echo "parseoptNoVal" + # test NoVal mode with custom cmdline arguments + var argv = "--left --debug:3 -l -r:2 --debug 2 --debug=1 -r1 -r=0 -lr4" + var p = parseopt.initOptParser(argv, + shortNoVal = {'l'}, longNoVal = @["left"]) + for kind, key, val in parseopt.getopt(p): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val -block: - echo "parseopt2" - for kind, key, val in parseopt2.getopt(): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val + block: + echo "parseopt2" + for kind, key, val in parseopt2.getopt(): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val - # pass custom cmdline arguments - echo "first round" - var argv: seq[string] = @["--left", "--debug:3", "-l=4", "-r:2"] - var p = parseopt2.initOptParser(argv) - for kind, key, val in parseopt2.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val - break - # reset getopt iterator and check arguments are returned correctly. - echo "second round" - for kind, key, val in parseopt2.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val + # pass custom cmdline arguments + echo "first round" + var argv: seq[string] = @["--left", "--debug:3", "-l=4", "-r:2"] + var p = parseopt2.initOptParser(argv) + for kind, key, val in parseopt2.getopt(p): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val + break + # reset getopt iterator and check arguments are returned correctly. + echo "second round" + for kind, key, val in parseopt2.getopt(p): + echo "kind: ", kind, "\tkey:val -- ", key, ":", val + + import osproc, os, strutils + from stdtest/specialpaths import buildDir + import "../.." / compiler/unittest_light + + block: # fix #9951 + var p = parseopt.initOptParser(@["echo \"quoted\""]) + assertEquals parseopt.cmdLineRest(p), """'echo "quoted"'""" + let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] + var p2 = parseopt.initOptParser(args) + assertEquals parseopt.cmdLineRest(p2), + """a1b 'a2 b' '' 'a4"b' 'a5'"'"'b' 'a6\b' 'a7'"'"'b'""" + doAssert "a5'b" == "a5\'b" + + block: # fix #9842 + let exe = buildDir / "D20190112T145450".addFileExt(ExeExt) + defer: removeFile exe + let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] + let cmd = "$# c -r --verbosity:0 -o:$# -d:testament_tparseopt $# $#" % + [getCurrentCompilerExe(), exe, currentSourcePath(), + args.quoteShellCommand] + var ret = execCmdEx(cmd, options = {}) + if ret.exitCode != 0: + # before bug fix, running cmd would show: + # sh: -c: line 0: unexpected EOF while looking for matching `"'\n + echo "exitCode: ", ret.exitCode, " cmd:", cmd + doAssert false + stripLineEnd(ret.output) + assertEquals ret.output, + """ +@["a1b", "a2 b", "", "a4\"b", "a5\'b", "a6\\b", "a7\'b"] +arg 0 ai.len:3 :{a1b} +arg 1 ai.len:4 :{a2 b} +arg 2 ai.len:0 :{} +arg 3 ai.len:4 :{a4"b} +arg 4 ai.len:4 :{a5'b} +arg 5 ai.len:4 :{a6\b} +arg 6 ai.len:4 :{a7'b}""" From 9e68b2ce5d836fc269be9735d97aebb99adbfa2d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jan 2019 23:53:50 -0800 Subject: [PATCH 114/350] fix test --- tests/misc/tparseopt.nim | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/misc/tparseopt.nim b/tests/misc/tparseopt.nim index 39c17869cc..57253389da 100644 --- a/tests/misc/tparseopt.nim +++ b/tests/misc/tparseopt.nim @@ -29,6 +29,7 @@ kind: cmdLongOption key:val -- left: kind: cmdLongOption key:val -- debug:3 kind: cmdShortOption key:val -- l:4 kind: cmdShortOption key:val -- r:2''' +joinable: false """ when defined(testament_tparseopt): @@ -98,12 +99,22 @@ else: block: # fix #9951 var p = parseopt.initOptParser(@["echo \"quoted\""]) - assertEquals parseopt.cmdLineRest(p), """'echo "quoted"'""" + let expected = when defined(windows): + """"echo \"quoted\""""" + else: + """'echo "quoted"'""" + assertEquals parseopt.cmdLineRest(p), expected + + doAssert "a5'b" == "a5\'b" + let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] var p2 = parseopt.initOptParser(args) - assertEquals parseopt.cmdLineRest(p2), + let expected2 = when defined(windows): + """a1b "a2 b" "" a4\"b a5'b a6\b a7'b""" + else: """a1b 'a2 b' '' 'a4"b' 'a5'"'"'b' 'a6\b' 'a7'"'"'b'""" doAssert "a5'b" == "a5\'b" + assertEquals parseopt.cmdLineRest(p2), expected2 block: # fix #9842 let exe = buildDir / "D20190112T145450".addFileExt(ExeExt) From c8ba9ffdba9165b2c952f579005fdbb6871a1380 Mon Sep 17 00:00:00 2001 From: Jjp137 Date: Mon, 14 Jan 2019 22:05:05 -0800 Subject: [PATCH 115/350] lib.rst: fix a typo (#10308) [ci skip] --- doc/lib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lib.rst b/doc/lib.rst index 395b242e69..1f19f9bf42 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -107,7 +107,7 @@ String handling substrings, replacing substrings. * `strformat `_ - Macro based standard string interpolation / formatting. Inpired by + Macro based standard string interpolation / formatting. Inspired by Python's ``f``-strings. * `strmisc `_ From 8922063bd84e9109a5ba493188f201ebda74f66d Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Tue, 15 Jan 2019 12:38:12 +0530 Subject: [PATCH 116/350] typed/untyped return type is invalid for everything except templates and macros (#10275) --- compiler/semstmts.nim | 3 --- compiler/semtypes.nim | 7 +++++-- tests/proc/tillegalreturntype.nim | 12 ++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 tests/proc/tillegalreturntype.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 12283e0422..3fdbb85dbc 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1253,9 +1253,6 @@ proc semTypeSection(c: PContext, n: PNode): PNode = proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) = s.typ = semProcTypeNode(c, n, genericParams, nil, s.kind) - if s.kind notin {skMacro, skTemplate}: - if s.typ.sons[0] != nil and s.typ.sons[0].kind == tyStmt: - localError(c.config, n.info, "invalid return type: 'stmt'") proc addParams(c: PContext, n: PNode, kind: TSymKind) = for i in countup(1, sonsLen(n)-1): diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 7056eab5f2..ddc42c5b4c 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1158,12 +1158,15 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, # turn explicit 'void' return type into 'nil' because the rest of the # compiler only checks for 'nil': if skipTypes(r, {tyGenericInst, tyAlias, tySink}).kind != tyVoid: + if kind notin {skMacro, skTemplate} and r.kind in {tyStmt, tyExpr}: + localError(c.config, n.sons[0].info, "return type '" & typeToString(r) & + "' is only valid for macros and templates") # 'auto' as a return type does not imply a generic: - if r.kind == tyAnything: + elif r.kind == tyAnything: # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the # compiler is hardly aware of 'auto': r = newTypeS(tyExpr, c) - elif r.kind != tyExpr: + else: if r.sym == nil or sfAnon notin r.sym.flags: let lifted = liftParamType(c, kind, genericParams, r, "result", n.sons[0].info) diff --git a/tests/proc/tillegalreturntype.nim b/tests/proc/tillegalreturntype.nim new file mode 100644 index 0000000000..be9e2147e2 --- /dev/null +++ b/tests/proc/tillegalreturntype.nim @@ -0,0 +1,12 @@ +discard """ + cmd: "nim check $file" + errmsg: "" + nimout: '''tillegalreturntype.nim(8, 11) Error: return type 'typed' is only valid for macros and templates +tillegalreturntype.nim(11, 11) Error: return type 'untyped' is only valid for macros and templates''' +""" + +proc x(): typed = + discard + +proc y(): untyped = + discard From 1d941b87352dc675fdedac274ddf0b9d43c8d5c5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 00:53:58 -0800 Subject: [PATCH 117/350] refs #10249 #10266; each command logged only if --verboseCommands:true (#10303) --- testament/tester.nim | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/testament/tester.nim b/testament/tester.nim index 04552b7c9f..ad0d227425 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -18,6 +18,7 @@ var useColors = true var backendLogging = true var simulate = false var verboseMegatest = false # very verbose but can be useful +var verboseCommands = false const testsDir = "tests" & DirSep @@ -37,6 +38,7 @@ Arguments: Options: --print also print results to the console --verboseMegatest log to stdout megatetest compilation + --verboseCommands log to stdout info about commands being run --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests --targets:"c c++ js objc" run tests for specified targets (default: all) @@ -111,12 +113,14 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp result.exitCode = peekExitCode(p) if result.exitCode != -1: break close(p) - if result.exitCode != 0: + + if verboseCommands: var command2 = command if args.len > 0: command2.add " " & args.quoteShellCommand - echo (msg: "execCmdEx2 failed", + echo (msg: "execCmdEx2", command: command2, - options: options) + options: options, + exitCode: result.exitCode) proc nimcacheDir(filename, options: string, target: TTarget): string = ## Give each test a private nimcache dir so they don't clobber each other's. @@ -540,6 +544,7 @@ proc main() = case p.key.string.normalize of "print", "verbose": optPrintResults = true of "verbosemegatest": verboseMegatest = true + of "verbosecommands": verboseCommands = true of "failing": optFailing = true of "pedantic": discard "now always enabled" of "targets": From fbf9b1f88af5f56e095574b6d3bddbe4246b0a16 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 15 Jan 2019 10:01:45 +0100 Subject: [PATCH 118/350] fixed indentation --- testament/categories.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 4ff50af971..54b4272ee8 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -600,7 +600,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = get from Nim cmd put outputGotten.txt, outputGotten.txt, megatest.nim there too delete upon completion, maybe - ]# + ]# var outDir = nimcacheDir(testsDir / "megatest", "", targetC) const marker = "megatest:processing: " @@ -707,10 +707,9 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string, runJoinedTest(r, cat, testsDir) else: var testsRun = 0 - var files: seq[string] for file in walkDirRec(testsDir &.? cat.string): - if isTestFile(file): files.add file + if isTestFile(file): files.add file files.sort # give reproducible order for i, name in files: From 06a8b488111b6d3858706bec6b31b1e4609620ea Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 15 Jan 2019 10:01:57 +0100 Subject: [PATCH 119/350] disable one more test for C++ --- tests/iter/titervaropenarray.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/iter/titervaropenarray.nim b/tests/iter/titervaropenarray.nim index 701f652df5..4469fdcf55 100644 --- a/tests/iter/titervaropenarray.nim +++ b/tests/iter/titervaropenarray.nim @@ -1,5 +1,6 @@ discard """ output: "123" + targets: "C" """ # Try to break the transformation pass: iterator iterAndZero(a: var openArray[int]): int = From 05c52ff34f996bb44425bcc17ae3ae60ed896bef Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 15 Jan 2019 10:15:27 +0100 Subject: [PATCH 120/350] fixes #10203 (#10290) * fixes #10203 * make typredef test green again * fixes the regressions differently --- compiler/semtypes.nim | 28 +++++++++++++--------------- compiler/types.nim | 2 +- tests/objects/tobject.nim | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index ddc42c5b4c..c28902b1f2 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1318,7 +1318,19 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = if tx != result and tx.kind == tyObject and tx.sons[0] != nil: semObjectTypeForInheritedGenericInst(c, n, tx) -proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType +proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType = + if typeExpr.kind in {tyObject, tyEnum, tyDistinct, tyForward} and prev != nil: + result = newTypeS(tyAlias, c) + result.rawAddSon typeExpr + result.sym = prev.sym + assignType(prev, result) + +proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) = + if prev != nil: + let result = newTypeS(tyAlias, c) + result.rawAddSon typExpr.typ + result.sym = prev.sym + assignType(prev, result) proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType = var n = semExprWithType(c, n, {efDetermineType}) @@ -1422,20 +1434,6 @@ proc semProcTypeWithScope(c: PContext, n: PNode, when useEffectSystem: setEffectsForProcType(c.graph, result, n.sons[1]) closeScope(c) -proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType = - if typeExpr.kind in {tyObject, tyEnum, tyDistinct} and prev != nil: - result = newTypeS(tyAlias, c) - result.rawAddSon typeExpr - result.sym = prev.sym - assignType(prev, result) - -proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) = - if prev != nil: - let result = newTypeS(tyAlias, c) - result.rawAddSon typExpr.typ - result.sym = prev.sym - assignType(prev, result) - proc symFromExpectedTypeNode(c: PContext, n: PNode): PSym = if n.kind == nkType: result = symFromType(c, n.typ, n.info) diff --git a/compiler/types.nim b/compiler/types.nim index 5b8b9e6027..797336ddf7 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -432,7 +432,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = sfAnon notin t.sym.flags: if t.kind == tyInt and isIntLit(t): result = t.sym.name.s & " literal(" & $t.n.intVal & ")" - elif t.kind == tyAlias: + elif t.kind == tyAlias and t.sons[0].kind != tyAlias: result = typeToString(t.sons[0]) elif prefer in {preferName, preferTypeName} or t.sym.owner.isNil: result = t.sym.name.s diff --git a/tests/objects/tobject.nim b/tests/objects/tobject.nim index 61ef7442e2..fbf531c3da 100644 --- a/tests/objects/tobject.nim +++ b/tests/objects/tobject.nim @@ -17,3 +17,23 @@ suite "object basic methods": check($obj == "(foo: 1)") test "it should test equality based on fields": check(makeObj(1) == makeObj(1)) + +# bug #10203 + +type + TMyObj = TYourObj + TYourObj = object of RootObj + x, y: int + +proc init: TYourObj = + result.x = 0 + result.y = -1 + +proc f(x: var TYourObj) = + discard + +var m: TMyObj = init() +f(m) + +var a: TYourObj = m +var b: TMyObj = a From c05e9c7c736e471895794a397e677400822b0672 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 05:46:01 -0800 Subject: [PATCH 121/350] fix twrong_refcounts in nim cpp mode (#10313) --- tests/parallel/twrong_refcounts.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/parallel/twrong_refcounts.nim b/tests/parallel/twrong_refcounts.nim index ac428762df..ed3c1b894b 100644 --- a/tests/parallel/twrong_refcounts.nim +++ b/tests/parallel/twrong_refcounts.nim @@ -1,7 +1,10 @@ discard """ output: "Success" + target: "c" """ +# Note: target: "cpp" fails because we can't yet have `extern "C"` mangling in +# `exportc` procs. import math, random, threadPool # --- From 9a92bc15b38b60a889815a1be9f87cca5038f747 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 05:46:29 -0800 Subject: [PATCH 122/350] [CI] runCI runs `kochExec "boot"` regardless of nim mode (#10312) --- koch.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index a77ad3144a..5e95376074 100644 --- a/koch.nim +++ b/koch.nim @@ -444,8 +444,7 @@ proc runCI(cmd: string) = # note(@araq): Do not replace these commands with direct calls (eg boot()) # as that would weaken our testing efforts. when defined(posix): # appveyor (on windows) didn't run this - # todo: implement `execWithEnv` - exec("env NIM_COMPILE_TO_CPP=false $1 boot" % kochExe.quoteShell) + kochExec "boot" kochExec "boot -d:release" ## build nimble early on to enable remainder to depend on it if needed From 4355f23ee5fd53acfdaa8ecb5dbb50f9b43f98b2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 05:50:28 -0800 Subject: [PATCH 123/350] fix #10305 nim cpp is now nan-correct at CT (#10310) * fix #10305 nim cpp is now nan-correct at CT * add example where simply `nim cpp -d:release` would exhibit nan bug --- compiler/extccomp.nim | 8 ++++---- doc/nimc.rst | 3 +++ tests/float/tfloatnan.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index ef371d5d04..2c5af6433b 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -63,8 +63,8 @@ compiler gcc: result = ( name: "gcc", objExt: "o", - optSpeed: " -O3 -ffast-math ", - optSize: " -Os -ffast-math ", + optSpeed: " -O3 ", + optSize: " -Os ", compilerExe: "gcc", cppCompiler: "g++", compileTmpl: "-c $options $include -o $objfile $file", @@ -88,8 +88,8 @@ compiler nintendoSwitchGCC: result = ( name: "switch_gcc", objExt: "o", - optSpeed: " -O3 -ffast-math ", - optSize: " -Os -ffast-math ", + optSpeed: " -O3 ", + optSize: " -Os ", compilerExe: "aarch64-none-elf-gcc", cppCompiler: "aarch64-none-elf-g++", compileTmpl: "-w -MMD -MP -MF $dfile -c $options $include -o $objfile $file", diff --git a/doc/nimc.rst b/doc/nimc.rst index 6f6d38d665..4ffb595c0e 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -347,6 +347,9 @@ complete list. Define Effect ====================== ========================================================= ``release`` Turns off runtime checks and turns on the optimizer. + More aggressive optimizations are possible, eg: + ``--passC:-ffast-math`` (but see issue #10305) + ``--stacktrace:off`` ``useWinAnsi`` Modules like ``os`` and ``osproc`` use the Ansi versions of the Windows API. The default build uses the Unicode version. diff --git a/tests/float/tfloatnan.nim b/tests/float/tfloatnan.nim index 29937a8624..8f384c3d91 100644 --- a/tests/float/tfloatnan.nim +++ b/tests/float/tfloatnan.nim @@ -14,3 +14,31 @@ echo "Nim: ", f32, " (float)" let f64: float64 = NaN echo "Nim: ", f64, " (double)" + +block: # issue #10305 + # with `-O3 -ffast-math`, generated C/C++ code is not nan compliant + # user can pass `--passC:-ffast-math` if he doesn't care. + proc fun() = + # this was previously failing at compile time with a nim compiler + # that was compiled with `nim cpp -d:release` + let a1 = 0.0 + let a = 0.0/a1 + let b1 = a == 0.0 + let b2 = a == a + doAssert not b1 + doAssert not b2 + + proc fun2(i: int) = + # this was previously failing simply with `nim cpp -d:release`; the + # difference with above example is that optimization (const folding) can't + # take place in this example to hide the non-compliant nan bug. + let a = 0.0/(i.float) + let b1 = a == 0.0 + let b2 = a == a + doAssert not b1 + doAssert not b2 + + static: fun() + fun() + fun2(0) + From 06e3d3ab4dea4aa11a1728c849799ce6aeecb3a4 Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Tue, 15 Jan 2019 19:00:02 +0100 Subject: [PATCH 124/350] Disable interior pointer checking by default for the Boehm GC. (#10316) The Boehm GC only needs interior pointer checking if pointers in global variables or on the heap point to the interior of an object rather than the beginning. If this does not happen, then checking for interior pointers causes additional overhead, in particular because any objects whose sizes are an exact multiple of two words gain another two words of padding, wasting memory. If checking of interior pointers is still desired, this can be achieved by setting the environment variable GC_ALL_INTERIOR_POINTERS. Pointers on the stack will always be treated as potential interior pointers, as compiler optimizations may advance the only live reference to a point past the beginning of an object. --- lib/system/mmdisp.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 89bc11d2c3..9cc7ab3231 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -72,6 +72,8 @@ when defined(boehmgc): proc boehmGCincremental {. importc: "GC_enable_incremental", boehmGC.} proc boehmGCfullCollect {.importc: "GC_gcollect", boehmGC.} + proc boehmGC_set_all_interior_pointers(flag: cint) {. + importc: "GC_set_all_interior_pointers", boehmGC.} proc boehmAlloc(size: int): pointer {.importc: "GC_malloc", boehmGC.} proc boehmAllocAtomic(size: int): pointer {. importc: "GC_malloc_atomic", boehmGC.} @@ -148,6 +150,7 @@ when defined(boehmgc): proc nimGC_setStackBottom(theStackBottom: pointer) = discard proc initGC() = + boehmGC_set_all_interior_pointers(0) boehmGCinit() when hasThreadSupport: boehmGC_allow_register_threads() From beed27b75d27118324d89b83974b7cf3065769cf Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 10:02:01 -0800 Subject: [PATCH 125/350] improve vmgen.codeListing formatting (#10306) * improve vmgen.codeListing formatting * address comments --- compiler/dfa.nim | 2 +- compiler/vm.nim | 7 +++---- compiler/vmgen.nim | 31 ++++++++++++++++++++++--------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index cd32d95d55..df9584576a 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -86,7 +86,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) = result.add("\n") inc i if i in jumpTargets: result.add("L" & $i & ": End\n") - + # consider calling `asciitables.alignTable` proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} = ## echos the ControlFlowGraph for debugging purposes. diff --git a/compiler/vm.nim b/compiler/vm.nim index c001981f89..180f3800b2 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,10 +10,6 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -const - debugEchoCode = false - traceCode = debugEchoCode - import ast except getstr import @@ -26,6 +22,9 @@ from evaltempl import evalTemplate from modulegraphs import ModuleGraph, PPassContext +const + traceCode = debugEchoCode + when hasFFI: import evalffi diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index afadb41698..033cc81f0d 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -33,6 +33,11 @@ import import platform from os import splitFile +const + debugEchoCode* = defined(nimVMDebug) + +when debugEchoCode: + import asciitables when hasFFI: import evalffi @@ -43,9 +48,10 @@ type TGenFlags = set[TGenFlag] proc debugInfo(c: PCtx; info: TLineInfo): string = - result = toFilename(c.config, info).splitFile.name & ":" & $info.line + result = toFileLineCol(c.config, info) proc codeListing(c: PCtx, result: var string, start=0; last = -1) = + ## for debugging purposes # first iteration: compute all necessary labels: var jumpTargets = initIntSet() let last = if last < 0: c.code.len-1 else: min(last, c.code.len-1) @@ -54,7 +60,9 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) = if x.opcode in relativeJumps: jumpTargets.incl(i+x.regBx-wordExcess) - # for debugging purposes + template toStr(opc: TOpcode): string = ($opc).substr(3) + + result.add "code listing:\n" var i = start while i <= last: if i in jumpTargets: result.addf("L$1:\n", i) @@ -62,34 +70,39 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) = result.add($i) let opc = opcode(x) - if opc in {opcConv, opcCast}: + if opc in {opcIndCall, opcIndCallAsgn}: + result.addf("\t$#\tr$#, r$#, nargs:$#", opc.toStr, x.regA, + x.regB, x.regC) + elif opc in {opcConv, opcCast}: let y = c.code[i+1] let z = c.code[i+2] - result.addf("\t$#\tr$#, r$#, $#, $#", ($opc).substr(3), x.regA, x.regB, + result.addf("\t$#\tr$#, r$#, $#, $#", opc.toStr, x.regA, x.regB, c.types[y.regBx-wordExcess].typeToString, c.types[z.regBx-wordExcess].typeToString) inc i, 2 elif opc < firstABxInstr: - result.addf("\t$#\tr$#, r$#, r$#", ($opc).substr(3), x.regA, + result.addf("\t$#\tr$#, r$#, r$#", opc.toStr, x.regA, x.regB, x.regC) elif opc in relativeJumps: - result.addf("\t$#\tr$#, L$#", ($opc).substr(3), x.regA, + result.addf("\t$#\tr$#, L$#", opc.toStr, x.regA, i+x.regBx-wordExcess) elif opc in {opcLdConst, opcAsgnConst}: let idx = x.regBx-wordExcess - result.addf("\t$#\tr$#, $# ($#)", ($opc).substr(3), x.regA, + result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA, c.constants[idx].renderTree, $idx) elif opc in {opcMarshalLoad, opcMarshalStore}: let y = c.code[i+1] - result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB, + result.addf("\t$#\tr$#, r$#, $#", opc.toStr, x.regA, x.regB, c.types[y.regBx-wordExcess].typeToString) inc i else: - result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA, x.regBx-wordExcess) + result.addf("\t$#\tr$#, $#", opc.toStr, x.regA, x.regBx-wordExcess) result.add("\t#") result.add(debugInfo(c, c.debug[i])) result.add("\n") inc i + when debugEchoCode: + result = result.alignTable proc echoCode*(c: PCtx; start=0; last = -1) {.deprecated.} = var buf = "" From 795e5e11ef53260a0a15e25eba36dddbd83900c1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 12:51:02 -0800 Subject: [PATCH 126/350] parseopt2.cmdLineRest is now correct too (#10304) --- lib/pure/parseopt2.nim | 8 ++++---- tests/misc/tparseopt.nim | 35 +++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index 51a70b6d1d..a84943cf95 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -112,10 +112,10 @@ proc next(p: var OptParser) = p.key = token p.val = "" -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecated.} = - ## Returns part of command line string that has not been parsed yet. - ## Do not use - does not correctly handle whitespace. - return p.cmd[p.pos..p.cmd.len-1].join(" ") +proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1".} = + ## Returns the part of command line string that has not been parsed yet, + ## properly quoted. + return p.cmd[p.pos..p.cmd.len-1].quoteShellCommand type GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString] diff --git a/tests/misc/tparseopt.nim b/tests/misc/tparseopt.nim index 57253389da..d8824e5220 100644 --- a/tests/misc/tparseopt.nim +++ b/tests/misc/tparseopt.nim @@ -97,24 +97,27 @@ else: from stdtest/specialpaths import buildDir import "../.." / compiler/unittest_light - block: # fix #9951 - var p = parseopt.initOptParser(@["echo \"quoted\""]) - let expected = when defined(windows): - """"echo \"quoted\""""" - else: - """'echo "quoted"'""" - assertEquals parseopt.cmdLineRest(p), expected + block: # fix #9951 (and make it work for parseopt and parseopt2) + template runTest(parseoptCustom) = + var p = parseoptCustom.initOptParser(@["echo \"quoted\""]) + let expected = when defined(windows): + """"echo \"quoted\""""" + else: + """'echo "quoted"'""" + assertEquals parseoptCustom.cmdLineRest(p), expected - doAssert "a5'b" == "a5\'b" + doAssert "a5'b" == "a5\'b" - let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] - var p2 = parseopt.initOptParser(args) - let expected2 = when defined(windows): - """a1b "a2 b" "" a4\"b a5'b a6\b a7'b""" - else: - """a1b 'a2 b' '' 'a4"b' 'a5'"'"'b' 'a6\b' 'a7'"'"'b'""" - doAssert "a5'b" == "a5\'b" - assertEquals parseopt.cmdLineRest(p2), expected2 + let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] + var p2 = parseoptCustom.initOptParser(args) + let expected2 = when defined(windows): + """a1b "a2 b" "" a4\"b a5'b a6\b a7'b""" + else: + """a1b 'a2 b' '' 'a4"b' 'a5'"'"'b' 'a6\b' 'a7'"'"'b'""" + doAssert "a5'b" == "a5\'b" + assertEquals parseoptCustom.cmdLineRest(p2), expected2 + runTest(parseopt) + runTest(parseopt2) block: # fix #9842 let exe = buildDir / "D20190112T145450".addFileExt(ExeExt) From 27e366eaec3389c11889375759e682d70b9cf5b6 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 15 Jan 2019 22:47:02 +0100 Subject: [PATCH 127/350] fixes #8294 [backport] --- lib/pure/terminal.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 35dc2483c3..2b3c08d0dc 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -37,7 +37,7 @@ type var gTerm {.threadvar.}: PTerminal -proc newTerminal(): PTerminal +proc newTerminal(): PTerminal {.gcsafe.} proc getTerminal(): PTerminal {.inline.} = if isNil(gTerm): From ef769e20b3b53de4a6940ae5cbc6751647e402f7 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 15 Jan 2019 22:57:50 +0100 Subject: [PATCH 128/350] attempt to fix https://github.com/nim-lang/nightlies/issues/17 --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 5e95376074..b50b6c0e13 100644 --- a/koch.nim +++ b/koch.nim @@ -74,7 +74,8 @@ Web options: build the official docs, use UA-48159761-1 """ -let kochExe* = os.getAppFilename() +let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed eg: `nim c -o:koch_debug koch.nim` + else: getAppDir() / "koch " # `best effort`, works for ur particular use case in nightly proc kochExec*(cmd: string) = exec kochExe.quoteShell & " " & cmd From 7920dc2898b9518a89c4a588dac2bcdea5658a92 Mon Sep 17 00:00:00 2001 From: nepeckman Date: Tue, 15 Jan 2019 16:58:44 -0500 Subject: [PATCH 129/350] Added support for --genDeps on JS backend (#10320) --- compiler/main.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/main.nim b/compiler/main.nim index b5f7e8364a..49c2666eac 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -92,6 +92,7 @@ proc commandJsonScript(graph: ModuleGraph) = when not defined(leanCompiler): proc commandCompileToJS(graph: ModuleGraph) = + let conf = graph.config #incl(gGlobalOptions, optSafeCode) setTarget(graph.config.target, osJS, cpuJS) #initDefines() @@ -100,6 +101,8 @@ when not defined(leanCompiler): semanticPasses(graph) registerPass(graph, JSgenPass) compileProject(graph) + if optGenScript in graph.config.globalOptions: + writeDepsFile(graph, toGeneratedFile(conf, conf.projectFull, "")) proc interactivePasses(graph: ModuleGraph) = initDefines(graph.config.symbols) From ece5887ab0af2e95f8327da59e43fda1e4b1b753 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 15 Jan 2019 23:07:51 +0100 Subject: [PATCH 130/350] another attempt to fix https://github.com/nim-lang/nightlies/issues/17 --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index b50b6c0e13..7059e87a58 100644 --- a/koch.nim +++ b/koch.nim @@ -75,7 +75,7 @@ Web options: """ let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed eg: `nim c -o:koch_debug koch.nim` - else: getAppDir() / "koch " # `best effort`, works for ur particular use case in nightly + else: getAppDir() / "koch".exe # works for winrelease proc kochExec*(cmd: string) = exec kochExe.quoteShell & " " & cmd From 1d7e3a84fd8ebaf183a231c9f8e0159f243ee25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Tue, 15 Jan 2019 23:48:19 +0100 Subject: [PATCH 131/350] Improve exception usage in parseutils (#10284) --- changelog.md | 2 ++ lib/pure/parseutils.nim | 56 +++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/changelog.md b/changelog.md index 2ac668cc65..d56d225060 100644 --- a/changelog.md +++ b/changelog.md @@ -36,6 +36,8 @@ - `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`. +- The procs `parseutils.parseBiggsetInt`, `parseutils.parseInt`, `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a `ValueError` when the parsed integer is outside of the valid range. Previously they sometimes raised a `OverflowError` and sometimes returned `0`. + - nre's `RegexMatch.{captureBounds,captures}[]` no longer return `Option` or `nil`/`""`, respectivly. Use the newly added `n in p.captures` method to check if a group is captured, otherwise you'll recieve an exception. diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index fb4bc19af0..36fcb19652 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -242,8 +242,9 @@ proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = result = "" discard s.parseUntil(result, if second == '\0': first else: second, i) -{.push overflowChecks: on.} -# this must be compiled with overflow checking turned on: +template integerOutOfRangeError(): ref ValueError = + newException(ValueError, "Parsed integer outside of valid range") + proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = var sign: BiggestInt = -1 @@ -256,38 +257,45 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = if i < s.len and s[i] in {'0'..'9'}: b = 0 while i < s.len and s[i] in {'0'..'9'}: - b = b * 10 - (ord(s[i]) - ord('0')) + let c = ord(s[i]) - ord('0') + if b >= (low(int) + c) div 10: + b = b * 10 - c + else: + raise integerOutOfRangeError() inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored + if sign == -1 and b == low(int): + raise integerOutOfRangeError() b = b * sign result = i - start -{.pop.} # overflowChecks proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. - rtl, extern: "npuParseBiggestInt", noSideEffect.} = + rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = ## parses an integer starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. - ## `OverflowError` is raised if an overflow occurs. + ## `ValueError` is raised if the parsed integer is out of the valid range. var res: BiggestInt # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): result = rawParseInt(s, res, start) - number = res + if result != 0: + number = res proc parseInt*(s: string, number: var int, start = 0): int {. - rtl, extern: "npuParseInt", noSideEffect.} = + rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = ## parses an integer starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. - ## `OverflowError` is raised if an overflow occurs. + ## `ValueError` is raised if the parsed integer is out of the valid range. var res: BiggestInt result = parseBiggestInt(s, res, start) - if (sizeof(int) <= 4) and - ((res < low(int)) or (res > high(int))): - raise newException(OverflowError, "overflow") - elif result != 0: + when sizeof(int) <= 4: + if res < low(int) or res > high(int): + raise integerOutOfRangeError() + if result != 0: number = int(res) -proc parseSaturatedNatural*(s: string, b: var int, start = 0): int = +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].}= ## parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. ## The number of processed character is returned. @@ -312,12 +320,13 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int = while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored result = i - start -# overflowChecks doesn't work with BiggestUInt proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = var res = 0.BiggestUInt prev = 0.BiggestUInt i = start + if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: + raise integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -325,35 +334,34 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = prev = res res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt if prev > res: - return 0 # overflowChecks emulation + raise integerOutOfRangeError() inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res result = i - start proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. - rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = ## parses an unsigned integer starting at `start` and stores the value ## into `number`. - ## Result is the number of processed chars or 0 if there is no integer - ## or overflow detected. + ## `ValueError` is raised if the parsed integer is out of the valid range. var res: BiggestUInt # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): result = rawParseUInt(s, res, start) - number = res + if result != 0: + number = res proc parseUInt*(s: string, number: var uint, start = 0): int {. - rtl, extern: "npuParseUInt", noSideEffect.} = + rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = ## parses an unsigned integer starting at `start` and stores the value ## into `number`. - ## Result is the number of processed chars or 0 if there is no integer or - ## overflow detected. + ## `ValueError` is raised if the parsed integer is out of the valid range. var res: BiggestUInt result = parseBiggestUInt(s, res, start) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: - raise newException(OverflowError, "overflow") + raise integerOutOfRangeError() if result != 0: number = uint(res) From fbd6743ea1db89eef3dd89ceb7c0cdcb648c8055 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 15:17:52 -0800 Subject: [PATCH 132/350] fix sdl_test test that failed in CI cpp mode (#10314) * fix sdl_test test that failed in CI cpp mode * preserve old code for NimInAction in `nim c` mode --- testament/categories.nim | 4 ++-- tests/niminaction/Chapter8/sdl/sdl_test.nim | 24 +++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 54b4272ee8..e2b5f0a789 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -352,13 +352,13 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) = "8b5d28e985c0542163927d253a3e4fc9", "783299b98179cc725f9c46b5e3b5381f", "1a2b3fba1187c68d6a9bfa66854f3318", - "80f9c3e594a798225046e8a42e990daf" + "391ff57b38d9ea6f3eeb3fe69ab539d3" ] for i, test in tests: let filename = testsDir / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) - doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed." + doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i]) # Run the tests. for testfile in tests: test "tests/" & testfile & ".nim" diff --git a/tests/niminaction/Chapter8/sdl/sdl_test.nim b/tests/niminaction/Chapter8/sdl/sdl_test.nim index a49e08911e..1c4d258fb3 100644 --- a/tests/niminaction/Chapter8/sdl/sdl_test.nim +++ b/tests/niminaction/Chapter8/sdl/sdl_test.nim @@ -17,12 +17,24 @@ discard pollEvent(nil) renderer.setDrawColor 29, 64, 153, 255 renderer.clear renderer.setDrawColor 255, 255, 255, 255 -var points = [ - (260'i32, 320'i32), - (260'i32, 110'i32), - (360'i32, 320'i32), - (360'i32, 110'i32) -] + +when defined(c): + # just to ensure code from NimInAction still works, but + # the `else` branch would work as well in C mode + var points = [ + (260'i32, 320'i32), + (260'i32, 110'i32), + (360'i32, 320'i32), + (360'i32, 110'i32) + ] +else: + var points = [ + (260.cint, 320.cint), + (260.cint, 110.cint), + (360.cint, 320.cint), + (360.cint, 110.cint) + ] + renderer.drawLines(addr points[0], points.len.cint) renderer.present From e0afacb9f2c182cf23c2f9dcaa87876b880752d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jan 2019 23:14:55 -0800 Subject: [PATCH 133/350] properly fix #10030 by skipping all external configs (#10324) --- build_all.sh | 5 ++--- doc/advopt.txt | 2 +- koch.nim | 7 ++++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build_all.sh b/build_all.sh index 6828bbbd8f..0706dcaf2b 100644 --- a/build_all.sh +++ b/build_all.sh @@ -27,9 +27,8 @@ build_nim_csources(){ [ -f $nim_csources ] || echo_run build_nim_csources # Note: if fails, may need to `cd csources && git pull` -# Note: --skipUserCfg is to prevent newer flags from -# breaking bootstrap phase -echo_run bin/nim c --skipUserCfg koch +# see D20190115T162028 +echo_run bin/nim c --skipUserCfg --skipParentCfg koch echo_run ./koch boot -d:release echo_run ./koch tools # Compile Nimble and other tools. diff --git a/doc/advopt.txt b/doc/advopt.txt index 2e91deed9a..c2e09a3ffd 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -77,7 +77,7 @@ Advanced options: --nilseqs:on|off allow 'nil' for strings/seqs for backwards compatibility --oldast:on|off use old AST for backwards compatibility - --skipCfg do not read the general configuration file + --skipCfg do not read the nim installation's configuration file --skipUserCfg do not read the user's configuration file --skipParentCfg do not read the parent dirs' configuration files --skipProjCfg do not read the project's configuration file diff --git a/koch.nim b/koch.nim index 7059e87a58..1d636914d3 100644 --- a/koch.nim +++ b/koch.nim @@ -287,9 +287,10 @@ proc boot(args: string) = for i in 0..2: echo "iteration: ", i+1 let extraOption = if i == 0: - "--skipUserCfg" - # forward compatibility: for bootstrap (1st iteration), avoid user flags - # that could break things, see #10030 + "--skipUserCfg --skipParentCfg" + # Note(D20190115T162028:here): the configs are skipped for bootstrap + # (1st iteration) to prevent newer flags from breaking bootstrap phase. + # fixes #10030. else: "" exec i.thVersion & " $# $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, extraOption, args, smartNimcache] From 384e517f09cf63cde136123b163c5c021c992877 Mon Sep 17 00:00:00 2001 From: tweenietomatoes Date: Wed, 16 Jan 2019 11:36:18 +0300 Subject: [PATCH 134/350] Most important date change ever (#10323) --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index c333611e2f..114086eb69 100644 --- a/readme.md +++ b/readme.md @@ -191,7 +191,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2018 Andreas Rumpf, all rights reserved. +Copyright © 2006-2019 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From b097081f1049f17b25a3dbfa5746263854cec581 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 7 Jan 2019 18:56:21 +0100 Subject: [PATCH 135/350] better docs: sequtils --- lib/pure/collections/sequtils.nim | 715 +++++++++++++++++------------- tests/manyloc/nake/nakefile.nim | 2 +- 2 files changed, 414 insertions(+), 303 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 39ba6df493..23ac0902a6 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -7,16 +7,73 @@ # distribution, for details about the copyright. # -## :Author: Alexander Mitchell-Robinson (Amrykid) +## :Author: Alexander Mitchell-Robinson (Amrykid) and Nim contributors ## -## This module implements operations for the built-in `seq`:idx: type which -## were inspired by functional programming languages. +## Although this module has ``seq`` in its name, it implements operations +## not only for `seq`:idx: type, but for three built-in container types under +## the ``openArray`` umbrella: +## * sequences +## * strings +## * array ## -## For functional style programming you may want to pass `anonymous procs -## `_ to procs like ``filter`` to -## reduce typing. Anonymous procs can use `the special do notation -## `_ -## which is more convenient in certain situations. +## The system module defines several common functions, such as: +## * ``newseq[T]`` for creating new sequences of type ``T`` +## * ``@`` for converting arrays and strings to sequences +## * ``add`` for adding new elements to strings and sequences +## * ``&`` for string and seq concatenation +## * ``in`` (alias for ``contains``) and ``notin`` for checking if an item is +## in a container +## +## This module builds upon that, providing additional functionality in form of +## procs, iterators and templates inspired by functional programming +## languages. +## +## For functional style programming you have different options at your disposal: +## * pass `anonymous proc`_ +## * import `sugar module`_ and use +## `=> macro.m,untyped,untyped>`_ +## * use `...It templates<#18>`_ +## (`mapIt<#mapIt.t,typed,untyped>`_, +## `filterIt<#filterIt.t,untyped,untyped>`_, etc.) +## +## The chaining of functions is possible thanks to the +## `method call syntax`_. +## +## .. code-block:: +## import sequtils, sugar +## +## # Creating a sequence from 1 to 10, multiplying each member by 2, +## # keeping only the members which are not divisible by 6. +## let +## foo = toSeq(1..10).map(x => x*2).filter(x => x mod 6 != 0) +## bar = toSeq(1..10).mapIt(it*2).filterIt(it mod 6 != 0) +## +## doAssert foo == bar +## echo foo # @[2, 4, 8, 10, 14, 16, 20] +## +## echo foo.any(x => x > 17) # true +## echo bar.allIt(it < 20) # false +## echo foo.foldl(a + b) # 74; sum of all members +## +## .. code-block:: +## import sequtils +## from strutils import join +## +## let +## vowels = @"aeiou" # creates a sequence @['a', 'e', 'i', 'o', 'u'] +## foo = "sequtils is an awesome module" +## +## echo foo.filterIt(it notin vowels).join # "sqtls s n wsm mdl" +## +## ---- +## +## **See also**: +## * `strutils module`_ for common string functions +## * `sugar module`_ for syntactic sugar macros +## * `algorithm module`_ for common generic algorithms +## * `json module`_ for a structure which allows +## heterogeneous members + include "system/inclrtl" @@ -31,7 +88,7 @@ macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped ## substitution in macro arguments such as ## https://github.com/nim-lang/Nim/issues/7187 ## ``evalOnceAs(myAlias, myExp)`` will behave as ``let myAlias = myExp`` - ## except when ``letAssigneable`` is false (eg to handle openArray) where + ## except when ``letAssigneable`` is false (e.g. to handle openArray) where ## it just forwards ``exp`` unchanged expectKind(expAlias, nnkIdent) var val = exp @@ -49,16 +106,20 @@ macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped proc concat*[T](seqs: varargs[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside a new sequence. + ## All sequences must be of the same type. ## - ## Example: + ## See also: + ## * `distribute proc<#distribute,seq[T],Positive>`_ for a reverse + ## operation ## - ## .. code-block:: - ## let - ## s1 = @[1, 2, 3] - ## s2 = @[4, 5] - ## s3 = @[6, 7] - ## total = concat(s1, s2, s3) - ## assert total == @[1, 2, 3, 4, 5, 6, 7] + runnableExamples: + let + s1 = @[1, 2, 3] + s2 = @[4, 5] + s3 = @[6, 7] + total = concat(s1, s2, s3) + assert total == @[1, 2, 3, 4, 5, 6, 7] + var L = 0 for seqitm in items(seqs): inc(L, len(seqitm)) newSeq(result, L) @@ -71,13 +132,17 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = proc count*[T](s: openArray[T], x: T): int = ## Returns the number of occurrences of the item `x` in the container `s`. ## - ## Example: - ## - ## .. code-block:: - ## let - ## s = @[1, 2, 2, 3, 2, 4, 2] - ## c = count(s, 2) - ## assert c == 4 + runnableExamples: + let + a = @[1, 2, 2, 3, 2, 4, 2] + b = "abracadabra" + c = count(a, 2) + d = count(a, 99) + e = count(b, 'r') + assert c == 4 + assert d == 0 + assert e == 2 + for itm in items(s): if itm == x: inc result @@ -85,15 +150,14 @@ proc count*[T](s: openArray[T], x: T): int = proc cycle*[T](s: openArray[T], n: Natural): seq[T] = ## Returns a new sequence with the items of the container `s` repeated ## `n` times. + ## `n` must be a non-negative number (zero or more). ## - ## Example: - ## - ## .. code-block:: - ## - ## let - ## s = @[1, 2, 3] - ## total = s.cycle(3) - ## assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + runnableExamples: + let + s = @[1, 2, 3] + total = s.cycle(3) + assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + result = newSeq[T](n * s.len) var o = 0 for x in 0 ..< n: @@ -103,14 +167,13 @@ proc cycle*[T](s: openArray[T], n: Natural): seq[T] = proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. + ## `n` must be a non-negative number (zero or more). ## - ## Example: - ## - ## .. code-block:: - ## - ## let - ## total = repeat(5, 3) - ## assert total == @[5, 5, 5] + runnableExamples: + let + total = repeat(5, 3) + assert total == @[5, 5, 5] + result = newSeq[T](n) for i in 0 ..< n: result[i] = x @@ -118,16 +181,18 @@ proc repeat*[T](x: T, n: Natural): seq[T] = proc deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = ## Returns a new sequence without duplicates. ## - ## Example: + ## Setting the optional argument ``isSorted`` to ``true`` (default: false) + ## uses a faster algorithm for deduplication. ## - ## .. code-block:: - ## let - ## dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] - ## dup2 = @["a", "a", "c", "d", "d"] - ## unique1 = deduplicate(dup1) - ## unique2 = deduplicate(dup2) - ## assert unique1 == @[1, 3, 4, 2, 8] - ## assert unique2 == @["a", "c", "d"] + runnableExamples: + let + dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] + dup2 = @["a", "a", "c", "d", "d"] + unique1 = deduplicate(dup1) + unique2 = deduplicate(dup2, isSorted = true) + assert unique1 == @[1, 3, 4, 2, 8] + assert unique2 == @["a", "c", "d"] + result = @[] if s.len > 0: if isSorted: @@ -144,39 +209,44 @@ proc deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = proc zip*[S, T](s1: openArray[S], s2: openArray[T]): seq[tuple[a: S, b: T]] = ## Returns a new sequence with a combination of the two input containers. ## + ## The input containers can be of different types. + ## If one container is shorter, the remaining items in the longer container + ## are discarded. + ## ## For convenience you can access the returned tuples through the named - ## fields `a` and `b`. If one container is shorter, the remaining items in - ## the longer container are discarded. + ## fields `a` and `b`. ## - ## Example: - ## - ## .. code-block:: - ## let - ## short = @[1, 2, 3] - ## long = @[6, 5, 4, 3, 2, 1] - ## words = @["one", "two", "three"] - ## zip1 = zip(short, long) - ## zip2 = zip(short, words) - ## assert zip1 == @[(1, 6), (2, 5), (3, 4)] - ## assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] - ## assert zip1[2].b == 4 - ## assert zip2[2].b == "three" + runnableExamples: + let + short = @[1, 2, 3] + long = @[6, 5, 4, 3, 2, 1] + words = @["one", "two", "three"] + letters = "abcd" + zip1 = zip(short, long) + zip2 = zip(short, words) + zip3 = zip(long, letters) + assert zip1 == @[(1, 6), (2, 5), (3, 4)] + assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] + assert zip3 == @[(a: 6, b: 'a'), (a: 5, b: 'b'), (a: 4, b: 'c'), + (a: 3, b: 'd')] + assert zip1[2].b == 4 + assert zip2[2].b == "three" + var m = min(s1.len, s2.len) newSeq(result, m) for i in 0 ..< m: result[i] = (s1[i], s2[i]) proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = - ## Splits and distributes a sequence `s` into `num` sub sequences. + ## Splits and distributes a sequence `s` into `num` sub-sequences. ## - ## Returns a sequence of `num` sequences. For some input values this is the - ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug - ## builds if `s` is nil or `num` is less than one, and will likely crash on - ## release builds. The input sequence `s` can be empty, which will produce + ## Returns a sequence of `num` sequences. For *some* input values this is the + ## inverse of the `concat <#concat,varargs[seq[T]]>`_ proc. + ## The input sequence `s` can be empty, which will produce ## `num` empty sequences. ## ## If `spread` is false and the length of `s` is not a multiple of `num`, the - ## proc will max out the first sub sequences with ``1 + len(s) div num`` + ## proc will max out the first sub-sequence with ``1 + len(s) div num`` ## entries, leaving the remainder of elements to the last sequence. ## ## On the other hand, if `spread` is true, the proc will distribute evenly @@ -184,18 +254,16 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = ## more suited to multithreading where you are passing equal sized work units ## to a thread pool and want to maximize core usage. ## - ## Example: - ## - ## .. code-block:: - ## let numbers = @[1, 2, 3, 4, 5, 6, 7] - ## assert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - ## assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - ## assert numbers.distribute(6)[0] == @[1, 2] - ## assert numbers.distribute(6)[5] == @[7] + runnableExamples: + let numbers = @[1, 2, 3, 4, 5, 6, 7] + assert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + assert numbers.distribute(6)[0] == @[1, 2] + assert numbers.distribute(6)[1] == @[3] + if num < 2: result = @[s] return - let num = int(num) # XXX probably only needed because of .. bug # Create the result and calculate the stride size and the remainder if any. @@ -209,13 +277,11 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = if extra == 0 or spread == false: # Use an algorithm which overcounts the stride and minimizes reading limits. if extra > 0: inc(stride) - for i in 0 ..< num: result[i] = newSeq[T]() for g in first ..< min(s.len, first + stride): result[i].add(s[g]) first += stride - else: # Use an undercounting algorithm which *adds* the remainder each iteration. for i in 0 ..< num: @@ -223,7 +289,6 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = if extra > 0: extra -= 1 inc(last) - result[i] = newSeq[T]() for g in first ..< last: result[i].add(s[g]) @@ -231,110 +296,103 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): seq[S]{.inline.} = - ## Returns a new sequence with the results of `op` applied to every item in - ## the container `s`. + ## Returns a new sequence with the results of `op` proc applied to every + ## item in the container `s`. ## - ## Since the input is not modified you can use this version of ``map`` to + ## Since the input is not modified you can use it to ## transform the type of the elements in the input container. ## - ## Example: + ## See also: + ## * `mapIt template<#mapIt.t,typed,untyped>`_ + ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ for the in-ace version ## - ## .. code-block:: nim - ## let - ## a = @[1, 2, 3, 4] - ## b = map(a, proc(x: int): string = $x) - ## assert b == @["1", "2", "3", "4"] + runnableExamples: + let + a = @[1, 2, 3, 4] + b = map(a, proc(x: int): string = $x) + assert b == @["1", "2", "3", "4"] + newSeq(result, s.len) for i in 0 ..< s.len: result[i] = op(s[i]) -proc map*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) - {.deprecated.} = - ## Applies `op` to every item in `s` modifying it directly. - ## - ## Note that this version of ``map`` requires your input and output types to - ## be the same, since they are modified in-place. - ## - ## Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: var string) = x &= "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] - ## **Deprecated since version 0.12.0:** Use the ``apply`` proc instead. - for i in 0 ..< s.len: op(s[i]) - proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) {.inline.} = ## Applies `op` to every item in `s` modifying it directly. ## - ## Note that this requires your input and output types to - ## be the same, since they are modified in-place. + ## Note that container `s` must be declared as a ``var`` + ## and it is required for your input and output types to + ## be the same, since `s` is modified in-place. ## The parameter function takes a ``var T`` type parameter. ## - ## Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## apply(a, proc(x: var string) = x &= "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] + ## See also: + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ + ## * `map proc<#map,openArray[T],proc(T)>`_ ## + runnableExamples: + var a = @["1", "2", "3", "4"] + apply(a, proc(x: var string) = x &= "42") + assert a == @["142", "242", "342", "442"] + for i in 0 ..< s.len: op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) {.inline.} = ## Applies `op` to every item in `s` modifying it directly. ## - ## Note that this requires your input and output types to - ## be the same, since they are modified in-place. + ## Note that container `s` must be declared as a ``var`` + ## and it is required for your input and output types to + ## be the same, since `s` is modified in-place. ## The parameter function takes and returns a ``T`` type variable. ## - ## Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## apply(a, proc(x: string): string = x & "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] + ## See also: + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ + ## * `map proc<#map,openArray[T],proc(T)>`_ ## + runnableExamples: + var a = @["1", "2", "3", "4"] + apply(a, proc(x: string): string = x & "42") + assert a == @["142", "242", "342", "442"] + for i in 0 ..< s.len: s[i] = op(s[i]) iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = - ## Iterates through a container and yields every item that fulfills the - ## predicate. + ## Iterates through a container `s` and yields every item that fulfills the + ## predicate `pred` (function that returns a `bool`). ## - ## Example: + ## See also: + ## * `fliter proc<#filter,openArray[T],proc(T)>`_ + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): - ## echo($n) - ## # echoes 4, 8, 4 in separate lines + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + var evens = newSeq[int]() + for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): + evens.add(n) + assert evens == @[4, 8, 4] + for i in 0 ..< s.len: if pred(s[i]): yield s[i] proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] {.inline.} = - ## Returns a new sequence with all the items that fulfilled the predicate. + ## Returns a new sequence with all the items of `s` that fulfilled the + ## predicate `pred` (function that returns a `bool`). ## - ## Example: + ## See also: + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ + ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ + ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ for the in-ace version ## - ## .. code-block:: - ## let - ## colors = @["red", "yellow", "black"] - ## f1 = filter(colors, proc(x: string): bool = x.len < 6) - ## f2 = filter(colors) do (x: string) -> bool : x.len > 5 - ## assert f1 == @["red", "black"] - ## assert f2 == @["yellow"] + runnableExamples: + let + colors = @["red", "yellow", "black"] + f1 = filter(colors, proc(x: string): bool = x.len < 6) + f2 = filter(colors, proc(x: string): bool = x.contains('y')) + assert f1 == @["red", "black"] + assert f2 == @["yellow"] + result = newSeq[T]() for i in 0 ..< s.len: if pred(s[i]): @@ -342,15 +400,23 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) {.inline.} = - ## Keeps the items in the passed sequence if they fulfilled the predicate. - ## Same as the ``filter`` proc, but modifies the sequence directly. + ## Keeps the items in the passed sequence `s` if they fulfilled the + ## predicate `pred` (function that returns a `bool`). ## - ## Example: + ## Note that `s` must be declared as a ``var``. ## - ## .. code-block:: - ## var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] - ## keepIf(floats, proc(x: float): bool = x > 10) - ## assert floats == @[13.0, 12.5, 10.1] + ## Similar to the `filter proc<#filter,openArray[T],proc(T)>`_, + ## but modifies the sequence directly. + ## + ## See also: + ## * `keepItIf template<#keepItIf.t,seq,untyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ + ## + runnableExamples: + var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] + keepIf(floats, proc(x: float): bool = x > 10) + assert floats == @[13.0, 12.5, 10.1] + var pos = 0 for i in 0 ..< len(s): if pred(s[i]): @@ -360,16 +426,15 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) setLen(s, pos) proc delete*[T](s: var seq[T]; first, last: Natural) = - ## Deletes in `s` the items at position `first` .. `last`. This modifies - ## `s` itself, it does not return a copy. + ## Deletes in the items of a sequence `s` at positions ``first..last`` + ## (including both ends of a range). + ## This modifies `s` itself, it does not return a copy. ## - ## Example: - ## - ##.. code-block:: - ## let outcome = @[1,1,1,1,1,1,1,1] - ## var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - ## dest.delete(3, 8) - ## assert outcome == dest + runnableExamples: + let outcome = @[1,1,1,1,1,1,1,1] + var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] + dest.delete(3, 8) + assert outcome == dest var i = first var j = last+1 @@ -384,15 +449,15 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = ## Inserts items from `src` into `dest` at position `pos`. This modifies ## `dest` itself, it does not return a copy. ## - ## Example: + ## Notice that `src` and `dest` must be of the same type. ## - ##.. code-block:: - ## var dest = @[1,1,1,1,1,1,1,1] - ## let - ## src = @[2,2,2,2,2,2] - ## outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - ## dest.insert(src, 3) - ## assert dest == outcome + runnableExamples: + var dest = @[1,1,1,1,1,1,1,1] + let + src = @[2,2,2,2,2,2] + outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] + dest.insert(src, 3) + assert dest == outcome var j = len(dest) - 1 var i = len(dest) + len(src) - 1 @@ -411,37 +476,48 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = template filterIt*(s, pred: untyped): untyped = - ## Returns a new sequence with all the items that fulfilled the predicate. + ## Returns a new sequence with all the items of `s` that fulfilled the + ## predicate `pred`. ## - ## Unlike the `proc` version, the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``. + ## Unlike the `filter proc<#filter,openArray[T],proc(T)>`_ and + ## `filter iterator<#filter.i,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using the ``it`` variable + ## for testing, like: ``filterIt("abcxyz", it == 'x')``. ## - ## Example: + ## See also: + ## * `fliter proc<#filter,openArray[T],proc(T)>`_ + ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ ## - ## .. code-block:: - ## let - ## temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] - ## acceptable = filterIt(temperatures, it < 50 and it > -10) - ## notAcceptable = filterIt(temperatures, it > 50 or it < -10) - ## assert acceptable == @[-2.0, 24.5, 44.31] - ## assert notAcceptable == @[-272.15, 99.9, -113.44] + runnableExamples: + let + temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] + acceptable = temperatures.filterIt(it < 50 and it > -10) + notAcceptable = temperatures.filterIt(it > 50 or it < -10) + assert acceptable == @[-2.0, 24.5, 44.31] + assert notAcceptable == @[-272.15, 99.9, -113.44] + var result = newSeq[type(s[0])]() for it {.inject.} in items(s): if pred: result.add(it) result template keepItIf*(varSeq: seq, pred: untyped) = - ## Convenience template around the ``keepIf`` proc to reduce typing. + ## Keeps the items in the passed sequence (must be declared as a ``var``) + ## if they fulfilled the predicate. ## - ## Unlike the `proc` version, the predicate needs to be an expression using + ## Unlike the `keepIf proc<#keepIf,seq[T],proc(T)>`_, + ## the predicate needs to be an expression using ## the ``it`` variable for testing, like: ``keepItIf("abcxyz", it == 'x')``. ## - ## Example: + ## See also: + ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## - ## .. code-block:: - ## var candidates = @["foo", "bar", "baz", "foobar"] - ## keepItIf(candidates, it.len == 3 and it[0] == 'b') - ## assert candidates == @["bar", "baz"] + runnableExamples: + var candidates = @["foo", "bar", "baz", "foobar"] + candidates.keepItIf(it.len == 3 and it[0] == 'b') + assert candidates == @["bar", "baz"] + var pos = 0 for i in 0 ..< len(varSeq): let it {.inject.} = varSeq[i] @@ -455,26 +531,37 @@ proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = ## Iterates through a container and checks if every item fulfills the ## predicate. ## - ## Example: + ## See also: + ## * `allIt template<#allIt.t,untyped,untyped>`_ + ## * `any proc<#any,openArray[T],proc(T)>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert all(numbers, proc (x: int): bool = return x < 10) == true - ## assert all(numbers, proc (x: int): bool = return x < 9) == false + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert all(numbers, proc (x: int): bool = return x < 10) == true + assert all(numbers, proc (x: int): bool = return x < 9) == false + for i in s: if not pred(i): return false return true template allIt*(s, pred: untyped): bool = - ## Checks if every item fulfills the predicate. + ## Iterates through a container and checks if every item fulfills the + ## predicate. ## - ## Example: + ## Unlike the `all proc<#all,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using + ## the ``it`` variable for testing, like: ``allIt("abba", it == 'a')``. ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert allIt(numbers, it < 10) == true - ## assert allIt(numbers, it < 9) == false + ## See also: + ## * `all proc<#all,openArray[T],proc(T)>`_ + ## * `anyIt template<#anyIt.t,untyped,untyped>`_ + ## + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert numbers.allIt(it < 10) == true + assert numbers.allIt(it < 9) == false + var result = true for it {.inject.} in items(s): if not pred: @@ -486,26 +573,37 @@ proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = ## Iterates through a container and checks if some item fulfills the ## predicate. ## - ## Example: + ## See also: + ## * `anyIt template<#anyIt.t,untyped,untyped>`_ + ## * `all proc<#all,openArray[T],proc(T)>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert any(numbers, proc (x: int): bool = return x > 8) == true - ## assert any(numbers, proc (x: int): bool = return x > 9) == false + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert any(numbers, proc (x: int): bool = return x > 8) == true + assert any(numbers, proc (x: int): bool = return x > 9) == false + for i in s: if pred(i): return true return false template anyIt*(s, pred: untyped): bool = - ## Checks if some item fulfills the predicate. + ## Iterates through a container and checks if some item fulfills the + ## predicate. ## - ## Example: + ## Unlike the `any proc<#any,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using + ## the ``it`` variable for testing, like: ``anyIt("abba", it == 'a')``. ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert anyIt(numbers, it > 8) == true - ## assert anyIt(numbers, it > 9) == false + ## See also: + ## * `any proc<#any,openArray[T],proc(T)>`_ + ## * `allIt template<#allIt.t,untyped,untyped>`_ + ## + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert numbers.anyIt(it > 8) == true + assert numbers.anyIt(it > 9) == false + var result = false for it {.inject.} in items(s): if pred: @@ -555,19 +653,28 @@ template toSeq2(iter: iterator): untyped = result template toSeq*(iter: untyped): untyped = - ## Transforms any iterable into a sequence. + ## Transforms any iterable (anything that can be iterated over, e.g. with + ## a for-loop) into a sequence. + ## runnableExamples: let - numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - odd_numbers = toSeq(filter(numeric, proc(x: int): bool = x mod 2 == 1)) - doAssert odd_numbers == @[1, 3, 5, 7, 9] + myRange = 1..5 + mySet: set[int8] = {5'i8, 3, 1} + assert type(myRange) is HSlice[system.int, system.int] + assert type(mySet) is set[int8] + + let + mySeq1 = toSeq(myRange) + mySeq2 = toSeq(mySet) + assert mySeq1 == @[1, 2, 3, 4, 5] + assert mySeq2 == @[1'i8, 3, 5] when compiles(toSeq1(iter)): toSeq1(iter) elif compiles(toSeq2(iter)): toSeq2(iter) else: - # overload for untyped, eg: `toSeq(myInlineIterator(3))` + # overload for untyped, e.g.: `toSeq(myInlineIterator(3))` when compiles(iter.len): block: evalOnceAs(iter2, iter, true) @@ -597,20 +704,23 @@ template foldl*(sequence, operation: untyped): untyped = ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) - ## 3). ## - ## Example: + ## See also: + ## * `foldl template<#foldl.t,,,>`_ with a starting parameter + ## * `foldr template<#foldr.t,untyped,untyped>`_ ## - ## .. code-block:: - ## let - ## numbers = @[5, 9, 11] - ## addition = foldl(numbers, a + b) - ## subtraction = foldl(numbers, a - b) - ## multiplication = foldl(numbers, a * b) - ## words = @["nim", "is", "cool"] - ## concatenation = foldl(words, a & b) - ## assert addition == 25, "Addition is (((5)+9)+11)" - ## assert subtraction == -15, "Subtraction is (((5)-9)-11)" - ## assert multiplication == 495, "Multiplication is (((5)*9)*11)" - ## assert concatenation == "nimiscool" + runnableExamples: + let + numbers = @[5, 9, 11] + addition = foldl(numbers, a + b) + subtraction = foldl(numbers, a - b) + multiplication = foldl(numbers, a * b) + words = @["nim", "is", "cool"] + concatenation = foldl(words, a & b) + assert addition == 25, "Addition is (((5)+9)+11)" + assert subtraction == -15, "Subtraction is (((5)-9)-11)" + assert multiplication == 495, "Multiplication is (((5)*9)*11)" + assert concatenation == "nimiscool" + let s = sequence assert s.len > 0, "Can't fold empty sequences" var result: type(s[0]) @@ -625,20 +735,22 @@ template foldl*(sequence, operation: untyped): untyped = template foldl*(sequence, operation, first): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## - ## This version of ``foldl`` gets a starting parameter. This makes it possible + ## This version of ``foldl`` gets a **starting parameter**. This makes it possible ## to accumulate the sequence into a different type than the sequence elements. ## ## The ``operation`` parameter should be an expression which uses the variables ## ``a`` and ``b`` for each step of the fold. The ``first`` parameter is the ## start value (the first ``a``) and therefor defines the type of the result. ## - ## Example: + ## See also: + ## * `foldr template<#foldr.t,untyped,untyped>`_ ## - ## .. code-block:: - ## let - ## numbers = @[0, 8, 1, 5] - ## digits = foldl(numbers, a & (chr(b + ord('0'))), "") - ## assert digits == "0815" + runnableExamples: + let + numbers = @[0, 8, 1, 5] + digits = foldl(numbers, a & (chr(b + ord('0'))), "") + assert digits == "0815" + var result: type(first) result = first for x in items(sequence): @@ -662,20 +774,23 @@ template foldr*(sequence, operation: untyped): untyped = ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - ## (3))). ## - ## Example: + ## See also: + ## * `foldl template<#foldl.t,untyped,untyped>`_ + ## * `foldl template<#foldl.t,,,>`_ with a starting parameter ## - ## .. code-block:: - ## let - ## numbers = @[5, 9, 11] - ## addition = foldr(numbers, a + b) - ## subtraction = foldr(numbers, a - b) - ## multiplication = foldr(numbers, a * b) - ## words = @["nim", "is", "cool"] - ## concatenation = foldr(words, a & b) - ## assert addition == 25, "Addition is (5+(9+(11)))" - ## assert subtraction == 7, "Subtraction is (5-(9-(11)))" - ## assert multiplication == 495, "Multiplication is (5*(9*(11)))" - ## assert concatenation == "nimiscool" + runnableExamples: + let + numbers = @[5, 9, 11] + addition = foldr(numbers, a + b) + subtraction = foldr(numbers, a - b) + multiplication = foldr(numbers, a * b) + words = @["nim", "is", "cool"] + concatenation = foldr(words, a & b) + assert addition == 25, "Addition is (5+(9+(11)))" + assert subtraction == 7, "Subtraction is (5-(9-(11)))" + assert multiplication == 495, "Multiplication is (5*(9*(11)))" + assert concatenation == "nimiscool" + let s = sequence assert s.len > 0, "Can't fold empty sequences" var result: type(s[0]) @@ -687,41 +802,26 @@ template foldr*(sequence, operation: untyped): untyped = result = operation result -template mapIt*(s, typ, op: untyped): untyped = - ## Convenience template around the ``map`` proc to reduce typing. - ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. You also need to pass as `typ` the type of the expression, - ## since the new returned sequence can have a different type than the - ## original. - ## - ## Example: - ## - ## .. code-block:: - ## let - ## nums = @[1, 2, 3, 4] - ## strings = nums.mapIt(string, $(4 * it)) - ## assert strings == @["4", "8", "12", "16"] - ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)`` - ## template instead. - var result: seq[typ] = @[] - for it {.inject.} in items(s): - result.add(op) - result - template mapIt*(s: typed, op: untyped): untyped = - ## Convenience template around the ``map`` proc to reduce typing. + ## Returns a new sequence with the results of `op` proc applied to every + ## item in the container `s`. + ## + ## Since the input is not modified you can use it to + ## transform the type of the elements in the input container. ## ## The template injects the ``it`` variable which you can use directly in an ## expression. ## - ## Example: + ## See also: + ## * `map proc<#map,openArray[T],proc(T)>`_ + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ for the in-ace version ## - ## .. code-block:: - ## let - ## nums = @[1, 2, 3, 4] - ## strings = nums.mapIt($(4 * it)) - ## assert strings == @["4", "8", "12", "16"] + runnableExamples: + let + nums = @[1, 2, 3, 4] + strings = nums.mapIt($(4 * it)) + assert strings == @["4", "8", "12", "16"] + when defined(nimHasTypeof): type outType = typeof(( block: @@ -758,31 +858,38 @@ template applyIt*(varSeq, op: untyped) = ## expression. The expression has to return the same type as the sequence you ## are mutating. ## - ## Example: + ## See also: + ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ + ## * `mapIt template<#mapIt.t,typed,untyped>`_ ## - ## .. code-block:: - ## var nums = @[1, 2, 3, 4] - ## nums.applyIt(it * 3) - ## assert nums[0] + nums[3] == 15 + runnableExamples: + var nums = @[1, 2, 3, 4] + nums.applyIt(it * 3) + assert nums[0] + nums[3] == 15 + for i in low(varSeq) .. high(varSeq): let it {.inject.} = varSeq[i] varSeq[i] = op template newSeqWith*(len: int, init: untyped): untyped = - ## creates a new sequence, calling `init` to initialize each value. + ## Creates a new sequence of length `len`, calling `init` to initialize + ## each value of the sequence. ## - ## Example: + ## Useful for creating "2D" sequences - sequences containing other sequences + ## or to populate fields of the created sequence. ## - ## .. code-block:: - ## var seq2D = newSeqWith(20, newSeq[bool](10)) - ## seq2D[0][0] = true - ## seq2D[1][0] = true - ## seq2D[0][1] = true - ## - ## import random - ## var seqRand = newSeqWith(20, random(10)) - ## echo seqRand + runnableExamples: + ## Creates a seqence containing 5 bool sequences, each of length of 3. + var seq2D = newSeqWith(5, newSeq[bool](3)) + assert seq2D.len == 5 + assert seq2D[0].len == 3 + assert seq2D[4][2] == false + + ## Creates a sequence of 20 random numbers from 1 to 10 + import random + var seqRand = newSeqWith(20, random(10)) + var result = newSeq[type(init)](len) for i in 0 ..< len: result[i] = init @@ -804,7 +911,7 @@ proc mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; macro mapLiterals*(constructor, op: untyped; nested = true): untyped = - ## applies ``op`` to each of the **atomic** literals like ``3`` + ## Applies ``op`` to each of the **atomic** literals like ``3`` ## or ``"abc"`` in the specified ``constructor`` AST. This can ## be used to map every array element to some target type: ## @@ -819,16 +926,20 @@ macro mapLiterals*(constructor, op: untyped; ## .. code-block:: ## let x = [int(0.1), int(1.2), int(2.3), int(3.4)] ## - ## If ``nested`` is true, the literals are replaced everywhere - ## in the ``constructor`` AST, otherwise only the first level + ## If ``nested`` is true (which is the default), the literals are replaced + ## everywhere in the ``constructor`` AST, otherwise only the first level ## is considered: ## ## .. code-block:: - ## mapLiterals((1, ("abc"), 2), float, nested=false) + ## let a = mapLiterals((1.2, (2.3, 3.4), 4.8), int) + ## let b = mapLiterals((1.2, (2.3, 3.4), 4.8), int, nested=false) + ## assert a == (1, (2, 3), 4) + ## assert b == (1, (2.3, 3.4), 4) ## - ## Produces:: - ## - ## (float(1), ("abc"), float(2)) + ## let c = mapLiterals((1, (2, 3), 4, (5, 6)), `$`) + ## let d = mapLiterals((1, (2, 3), 4, (5, 6)), `$`, nested=false) + ## assert c == ("1", ("2", "3"), "4", ("5", "6")) + ## assert d == ("1", (2, 3), "4", (5, 6)) ## ## There are no constraints for the ``constructor`` AST, it ## works for nested tuples of arrays of sets etc. diff --git a/tests/manyloc/nake/nakefile.nim b/tests/manyloc/nake/nakefile.nim index 3e8609169b..9c66ad71c9 100644 --- a/tests/manyloc/nake/nakefile.nim +++ b/tests/manyloc/nake/nakefile.nim @@ -76,7 +76,7 @@ task "testskel", "create skeleton test dir for testing": task "clean", "cleanup generated files": var dirs = @["nimcache", "server"/"nimcache"] - dirs.map(proc(x: var string) = + dirs.apply(proc(x: var string) = if existsDir(x): removeDir(x)) task "download", "download game assets": From be6456f0f41a06ff2ed05c76cbaff4b087cd8b31 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 9 Jan 2019 16:26:18 +0100 Subject: [PATCH 136/350] better docs: strutils --- lib/pure/strutils.nim | 2261 ++++++++++++-------- tests/errmsgs/tunknown_named_parameter.nim | 8 +- 2 files changed, 1383 insertions(+), 886 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 00469f9e53..4bb6400943 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -7,11 +7,72 @@ # distribution, for details about the copyright. # -## This module contains various string utility routines. -## See the module `re `_ for regular expression support. -## See the module `pegs `_ for PEG support. +## :Author: Nim contributors +## +## The system module defines several common functions for working with strings, +## such as: +## * ``$`` for converting other data-types to strings +## * ``&`` for string concatenation +## * ``add`` for adding a new character or a string to the existing one +## * ``in`` (alias for ``contains``) and ``notin`` for checking if a character +## is in a string +## +## This module builds upon that, providing additional functionality in form of +## procedures, iterators and templates for strings. +## +## .. code-block:: +## import strutils +## +## let +## numbers = @[867, 5309] +## multiLineString = "first line\nsecond line\nthird line" +## +## let jenny = numbers.join("-") +## assert jenny == "867-5309" +## +## assert splitLines(multiLineString) == +## @["first line", "second line", "third line"] +## assert split(multiLineString) == @["first", "line", "second", +## "line", "third", "line"] +## assert indent(multiLineString, 4) == +## " first line\n second line\n third line" +## assert 'z'.repeat(5) == "zzzzz" +## +## The chaining of functions is possible thanks to the +## `method call syntax`_: +## +## .. code-block:: +## import strutils +## from sequtils import map +## +## let jenny = "867-5309" +## assert jenny.split('-').map(parseInt) == @[867, 5309] +## +## assert "Beetlejuice".indent(1).repeat(3).strip == +## "Beetlejuice Beetlejuice Beetlejuice" +## ## This module is available for the `JavaScript target ## `_. +## +## ---- +## +## **See also:** +## * `strformat module`_ for string interpolation and formatting +## * `unicode module`_ for Unicode UTF-8 handling +## * `sequtils module`_ for operations on container +## types (including strings) +## * `parseutils module`_ for lower-level parsing of tokens, +## numbers, identifiers, etc. +## * `parseopt module`_ for command-line parsing +## * `strtabs module`_ for efficient hash tables +## (dictionaries, in some programming languages) mapping from strings to strings +## * `pegs module`_ for PEG (Parsing Expression Grammar) support +## * `ropes module`_ for rope data type, which can represent very +## long strings efficiently +## * `re module`_ for regular expression (regex) support +## * `strscans`_ for ``scanf`` and ``scanp`` macros, which offer +## easier substring extraction than regular expressions + import parseutils from math import pow, floor, log10 @@ -38,7 +99,8 @@ else: const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} - ## All the characters that count as whitespace. + ## All the characters that count as whitespace (space, tab, vertical tab, + ## carriage return, new line, form feed) Letters* = {'A'..'Z', 'a'..'z'} ## the set of letters @@ -56,14 +118,15 @@ const ## the set of characters an identifier can start with NewLines* = {'\13', '\10'} - ## the set of characters a newline terminator can start with + ## the set of characters a newline terminator can start with (carriage + ## return, line feed) AllChars* = {'\x00'..'\xFF'} ## A set with all the possible characters. ## ## Not very useful by its own, you can use it to create *inverted* sets to - ## make the `find() proc <#find,string,set[char],int>`_ find **invalid** - ## characters in strings. Example: + ## make the `find proc<#find,string,set[char],Natural,int>`_ + ## find **invalid** characters in strings. Example: ## ## .. code-block:: nim ## let invalid = AllChars - Digits @@ -72,9 +135,10 @@ const proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaAsciiChar".}= - ## Checks whether or not `c` is alphabetical. + ## Checks whether or not character `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. + ## Use `Unicode module`_ for UTF-8 support. runnableExamples: doAssert isAlphaAscii('e') == true doAssert isAlphaAscii('E') == true @@ -108,6 +172,7 @@ proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, runnableExamples: doAssert isSpaceAscii('n') == false doAssert isSpaceAscii(' ') == true + doAssert isSpaceAscii('\t') == true return c in Whitespace proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, @@ -115,6 +180,10 @@ proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. + ## Use `Unicode module`_ for UTF-8 support. + ## + ## See also: + ## * `toLowerAscii proc<#toLowerAscii,char>`_ runnableExamples: doAssert isLowerAscii('e') == true doAssert isLowerAscii('E') == false @@ -126,138 +195,28 @@ proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. + ## Use `Unicode module`_ for UTF-8 support. + ## + ## See also: + ## * `toUpperAscii proc<#toUpperAscii,char>`_ runnableExamples: doAssert isUpperAscii('e') == false doAssert isUpperAscii('E') == true doAssert isUpperAscii('7') == false return c in {'A'..'Z'} -template isImpl(call) = - if s.len == 0: return false - result = true - for c in s: - if not call(c): return false - -proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaAsciiStr", - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether or not `s` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## Returns true if all characters in `s` are - ## alphabetic and there is at least one character - ## in `s`. - runnableExamples: - doAssert isAlphaAscii("fooBar") == true - doAssert isAlphaAscii("fooBar1") == false - doAssert isAlphaAscii("foo Bar") == false - isImpl isAlphaAscii - -proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericStr", - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether or not `s` is alphanumeric. - ## - ## This checks a-z, A-Z, 0-9 ASCII characters only. - ## Returns true if all characters in `s` are - ## alpanumeric and there is at least one character - ## in `s`. - runnableExamples: - doAssert isAlphaNumeric("fooBar") == true - doAssert isAlphaNumeric("fooBar") == true - doAssert isAlphaNumeric("foo Bar") == false - isImpl isAlphaNumeric - -proc isDigit*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitStr", - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether or not `s` is a numeric value. - ## - ## This checks 0-9 ASCII characters only. - ## Returns true if all characters in `s` are - ## numeric and there is at least one character - ## in `s`. - runnableExamples: - doAssert isDigit("1908") == true - doAssert isDigit("fooBar1") == false - isImpl isDigit - -proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiStr", - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether or not `s` is completely whitespace. - ## - ## Returns true if all characters in `s` are whitespace - ## characters and there is at least one character in `s`. - runnableExamples: - doAssert isSpaceAscii(" ") == true - doAssert isSpaceAscii("") == false - isImpl isSpaceAscii - -template isCaseImpl(s, charProc, skipNonAlpha) = - var hasAtleastOneAlphaChar = false - if s.len == 0: return false - for c in s: - if skipNonAlpha: - var charIsAlpha = c.isAlphaAscii() - if not hasAtleastOneAlphaChar: - hasAtleastOneAlphaChar = charIsAlpha - if charIsAlpha and (not charProc(c)): - return false - else: - if not charProc(c): - return false - return if skipNonAlpha: hasAtleastOneAlphaChar else: true - -proc isLowerAscii*(s: string, skipNonAlpha: bool): bool {. - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether ``s`` is lower case. - ## - ## This checks ASCII characters only. - ## - ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## characters in ``s`` are lower case. Returns false if none of the - ## characters in ``s`` are alphabetical. - ## - ## If ``skipNonAlpha`` is false, returns true only if all characters - ## in ``s`` are alphabetical and lower case. - ## - ## For either value of ``skipNonAlpha``, returns false if ``s`` is - ## an empty string. - runnableExamples: - doAssert isLowerAscii("1foobar", false) == false - doAssert isLowerAscii("1foobar", true) == true - doAssert isLowerAscii("1fooBar", true) == false - isCaseImpl(s, isLowerAscii, skipNonAlpha) - -proc isUpperAscii*(s: string, skipNonAlpha: bool): bool {. - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## Checks whether ``s`` is upper case. - ## - ## This checks ASCII characters only. - ## - ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## characters in ``s`` are upper case. Returns false if none of the - ## characters in ``s`` are alphabetical. - ## - ## If ``skipNonAlpha`` is false, returns true only if all characters - ## in ``s`` are alphabetical and upper case. - ## - ## For either value of ``skipNonAlpha``, returns false if ``s`` is - ## an empty string. - runnableExamples: - doAssert isUpperAscii("1FOO", false) == false - doAssert isUpperAscii("1FOO", true) == true - doAssert isUpperAscii("1Foo", true) == false - isCaseImpl(s, isUpperAscii, skipNonAlpha) proc toLowerAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiChar".} = - ## Returns the lower case version of ``c``. + ## Returns the lower case version of character ``c``. ## ## This works only for the letters ``A-Z``. See `unicode.toLower ## `_ for a version that works for any Unicode ## character. + ## + ## See also: + ## * `isLowerAscii proc<#isLowerAscii,char>`_ + ## * `toLowerAscii proc<#toLowerAscii,string>`_ for converting a string runnableExamples: doAssert toLowerAscii('A') == 'a' doAssert toLowerAscii('e') == 'e' @@ -273,22 +232,30 @@ template toImpl(call) = proc toLowerAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiStr".} = - ## Converts `s` into lower case. + ## Converts string `s` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower ## `_ for a version that works for any Unicode ## character. + ## + ## See also: + ## * `normalize proc<#normalize,string>`_ runnableExamples: doAssert toLowerAscii("FooBar!") == "foobar!" toImpl toLowerAscii proc toUpperAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToUpperAsciiChar".} = - ## Converts `c` into upper case. + ## Converts character `c` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper ## `_ for a version that works for any Unicode ## character. + ## + ## See also: + ## * `isLowerAscii proc<#isLowerAscii,char>`_ + ## * `toUpperAscii proc<#toUpperAscii,string>`_ for converting a string + ## * `capitalizeAscii proc<#capitalizeAscii,string>`_ runnableExamples: doAssert toUpperAscii('a') == 'A' doAssert toUpperAscii('E') == 'E' @@ -299,20 +266,27 @@ proc toUpperAscii*(c: char): char {.noSideEffect, procvar, proc toUpperAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuToUpperAsciiStr".} = - ## Converts `s` into upper case. + ## Converts string `s` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper ## `_ for a version that works for any Unicode ## character. + ## + ## See also: + ## * `capitalizeAscii proc<#capitalizeAscii,string>`_ runnableExamples: doAssert toUpperAscii("FooBar!") == "FOOBAR!" toImpl toUpperAscii proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuCapitalizeAscii".} = - ## Converts the first character of `s` into upper case. + ## Converts the first character of string `s` into upper case. ## ## This works only for the letters ``A-Z``. + ## Use `Unicode module`_ for UTF-8 support. + ## + ## See also: + ## * `toUpperAscii proc<#toUpperAscii,char>`_ runnableExamples: doAssert capitalizeAscii("foo") == "Foo" doAssert capitalizeAscii("-bar") == "-bar" @@ -325,6 +299,9 @@ proc normalize*(s: string): string {.noSideEffect, procvar, ## ## That means to convert it to lower case and remove any '_'. This ## should NOT be used to normalize Nim identifier names. + ## + ## See also: + ## * `toLowerAscii proc<#toLowerAscii,string>`_ runnableExamples: doAssert normalize("Foo_bar") == "foobar" doAssert normalize("Foo Bar") == "foo bar" @@ -343,9 +320,9 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, rtl, extern: "nsuCmpIgnoreCase", procvar.} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b + ## | 0 if a == b + ## | < 0 if a < b + ## | > 0 if a > b runnableExamples: doAssert cmpIgnoreCase("FooBar", "foobar") == 0 doAssert cmpIgnoreCase("bar", "Foo") < 0 @@ -365,12 +342,14 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, rtl, extern: "nsuCmpIgnoreStyle", procvar.} = ## Semantically the same as ``cmp(normalize(a), normalize(b))``. It ## is just optimized to not allocate temporary strings. This should - ## NOT be used to compare Nim identifier names. use `macros.eqIdent` - ## for that. Returns: + ## NOT be used to compare Nim identifier names. + ## Use `macros.eqIdent`_ for that. ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b + ## Returns: + ## + ## | 0 if a == b + ## | < 0 if a < b + ## | > 0 if a > b runnableExamples: doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 @@ -394,51 +373,8 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, inc i inc j -proc strip*(s: string, leading = true, trailing = true, - chars: set[char] = Whitespace): string - {.noSideEffect, rtl, extern: "nsuStrip".} = - ## Strips leading or trailing `chars` from `s` and returns - ## the resulting string. - ## - ## If `leading` is true, leading `chars` are stripped. - ## If `trailing` is true, trailing `chars` are stripped. - ## If both are false, the string is returned unchanged. - runnableExamples: - doAssert " vhellov ".strip().strip(trailing = false, chars = {'v'}) == "hellov" - var - first = 0 - last = len(s)-1 - if leading: - while first <= last and s[first] in chars: inc(first) - if trailing: - while last >= 0 and s[last] in chars: dec(last) - result = substr(s, first, last) -proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = - ## Converts a character `c` to its octal representation. - ## - ## The resulting string may not have a leading zero. Its length is always - ## exactly 3. - runnableExamples: - doAssert toOctal('!') == "041" - result = newString(3) - var val = ord(c) - for i in countdown(2, 0): - result[i] = chr(val mod 8 + ord('0')) - val = val div 8 - -proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, - extern: "nsuIsNilOrEmpty", - deprecated: "use 'x.len == 0' instead".} = - ## Checks if `s` is nil or empty. - result = len(s) == 0 - -proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = - ## Checks if `s` is nil or consists entirely of whitespace characters. - result = true - for c in s: - if not c.isSpaceAscii(): - return false +# --------- Private templates for different split separators ----------- proc substrEq(s: string, pos: int, substr: string): bool = var i = 0 @@ -447,8 +383,6 @@ proc substrEq(s: string, pos: int, substr: string): bool = inc i return i == length -# --------- Private templates for different split separators ----------- - template stringHasSep(s: string, index: int, seps: set[char]): bool = s[index] in seps @@ -459,7 +393,7 @@ template stringHasSep(s: string, index: int, sep: string): bool = s.substrEq(index, sep) template splitCommon(s, sep, maxsplit, sepLen) = - ## Common code for split procedures + ## Common code for split procs var last = 0 var splits = maxsplit @@ -487,6 +421,42 @@ template oldSplit(s, seps, maxsplit) = if splits == 0: break dec(splits) +template accResult(iter: untyped) = + result = @[] + for x in iter: add(result, x) + + +iterator split*(s: string, sep: char, maxsplit: int = -1): string = + ## Splits the string `s` into substrings using a single separator. + ## + ## Substrings are separated by the character `sep`. + ## The code: + ## + ## .. code-block:: nim + ## for word in split(";;this;is;an;;example;;;", ';'): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "" + ## "" + ## "this" + ## "is" + ## "an" + ## "" + ## "example" + ## "" + ## "" + ## "" + ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,char,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split proc<#split,string,char,int>`_ + splitCommon(s, sep, maxsplit, 1) + iterator split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. @@ -529,8 +499,186 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split proc<#split,string,set[char],int>`_ splitCommon(s, seps, maxsplit, 1) +iterator split*(s: string, sep: string, maxsplit: int = -1): string = + ## Splits the string `s` into substrings using a string separator. + ## + ## Substrings are separated by the string `sep`. + ## The code: + ## + ## .. code-block:: nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "this" + ## "is" + ## "corrupted" + ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split proc<#split,string,string,int>`_ + splitCommon(s, sep, maxsplit, sep.len) + + +template rsplitCommon(s, sep, maxsplit, sepLen) = + ## Common code for rsplit functions + var + last = s.len - 1 + first = last + splits = maxsplit + startPos = 0 + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) + if splits == 0: + # No more splits means set first to the beginning + first = -1 + if first == -1: + startPos = 0 + else: + startPos = first + sepLen + yield substr(s, startPos, last) + if splits == 0: break + dec(splits) + dec(first) + last = first + +iterator rsplit*(s: string, sep: char, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char,int>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo:bar".rsplit(':'): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the char `sep`. + ## + ## See also: + ## * `split iterator<#split.i,string,char,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit proc<#rsplit,string,char,int>`_ + rsplitCommon(s, sep, maxsplit, 1) + +iterator rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char,int>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo bar".rsplit(WhiteSpace): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the set of chars `seps` + ## + ## See also: + ## * `split iterator<#split.i,string,set[char],int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit proc<#rsplit,string,set[char],int>`_ + rsplitCommon(s, seps, maxsplit, 1) + +iterator rsplit*(s: string, sep: string, maxsplit: int = -1, + keepSeparators: bool = false): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,string,int>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foothebar".rsplit("the"): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the string `sep` + ## + ## See also: + ## * `split iterator<#split.i,string,string,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit proc<#rsplit,string,string,int>`_ + rsplitCommon(s, sep, maxsplit, sep.len) + +iterator splitLines*(s: string, keepEol = false): string = + ## Splits the string `s` into its containing lines. + ## + ## Every `character literal `_ newline + ## combination (CR, LF, CR-LF) is supported. The result strings contain no + ## trailing end of line characters unless parameter ``keepEol`` is set to + ## ``true``. + ## + ## Example: + ## + ## .. code-block:: nim + ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): + ## writeLine(stdout, line) + ## + ## Results in: + ## + ## .. code-block:: nim + ## "" + ## "this" + ## "is" + ## "an" + ## "" + ## "example" + ## "" + ## + ## See also: + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + var first = 0 + var last = 0 + var eolpos = 0 + while true: + while last < s.len and s[last] notin {'\c', '\l'}: inc(last) + + eolpos = last + if last < s.len: + if s[last] == '\l': inc(last) + elif s[last] == '\c': + inc(last) + if last < s.len and s[last] == '\l': inc(last) + + yield substr(s, first, if keepEol: last-1 else: eolpos-1) + + # no eol characters consumed means that the string is over + if eolpos == last: + break + + first = last + iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## Splits the string ``s`` at whitespace stripping leading and trailing ## whitespace if necessary. If ``maxsplit`` is specified and is positive, @@ -564,254 +712,92 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## "bar" ## "baz" ## + ## See also: + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ oldSplit(s, Whitespace, maxsplit) -template accResult(iter: untyped) = - result = @[] - for x in iter: add(result, x) -proc splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitWhitespace".} = - ## The same as the `splitWhitespace <#splitWhitespace.i,string,int>`_ - ## iterator, but is a proc that returns a sequence of substrings. - accResult(splitWhitespace(s, maxsplit)) - -iterator split*(s: string, sep: char, maxsplit: int = -1): string = - ## Splits the string `s` into substrings using a single separator. - ## - ## Substrings are separated by the character `sep`. - ## The code: - ## - ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", ';'): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: - ## "" - ## "" - ## "this" - ## "is" - ## "an" - ## "" - ## "example" - ## "" - ## "" - ## "" - ## - splitCommon(s, sep, maxsplit, 1) - -iterator split*(s: string, sep: string, maxsplit: int = -1): string = - ## Splits the string `s` into substrings using a string separator. - ## - ## Substrings are separated by the string `sep`. - ## The code: - ## - ## .. code-block:: nim - ## for word in split("thisDATAisDATAcorrupted", "DATA"): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: - ## "this" - ## "is" - ## "corrupted" - ## - splitCommon(s, sep, maxsplit, sep.len) - -template rsplitCommon(s, sep, maxsplit, sepLen) = - ## Common code for rsplit functions - var - last = s.len - 1 - first = last - splits = maxsplit - startPos = 0 - # go to -1 in order to get separators at the beginning - while first >= -1: - while first >= 0 and not stringHasSep(s, first, sep): - dec(first) - if splits == 0: - # No more splits means set first to the beginning - first = -1 - if first == -1: - startPos = 0 - else: - startPos = first + sepLen - yield substr(s, startPos, last) - if splits == 0: break - dec(splits) - dec(first) - last = first - -iterator rsplit*(s: string, seps: set[char] = Whitespace, - maxsplit: int = -1): string = - ## Splits the string `s` into substrings from the right using a - ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. - ## - ## .. code-block:: nim - ## for piece in "foo bar".rsplit(WhiteSpace): - ## echo piece - ## - ## Results in: - ## - ## .. code-block:: nim - ## "bar" - ## "foo" - ## - ## Substrings are separated from the right by the set of chars `seps` - rsplitCommon(s, seps, maxsplit, 1) - -iterator rsplit*(s: string, sep: char, - maxsplit: int = -1): string = - ## Splits the string `s` into substrings from the right using a - ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. - ## - ## .. code-block:: nim - ## for piece in "foo:bar".rsplit(':'): - ## echo piece - ## - ## Results in: - ## - ## .. code-block:: nim - ## "bar" - ## "foo" - ## - ## Substrings are separated from the right by the char `sep` - rsplitCommon(s, sep, maxsplit, 1) - -iterator rsplit*(s: string, sep: string, maxsplit: int = -1, - keepSeparators: bool = false): string = - ## Splits the string `s` into substrings from the right using a - ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string,int>`_ except in reverse order. - ## - ## .. code-block:: nim - ## for piece in "foothebar".rsplit("the"): - ## echo piece - ## - ## Results in: - ## - ## .. code-block:: nim - ## "bar" - ## "foo" - ## - ## Substrings are separated from the right by the string `sep` - rsplitCommon(s, sep, maxsplit, sep.len) - -iterator splitLines*(s: string, keepEol = false): string = - ## Splits the string `s` into its containing lines. - ## - ## Every `character literal `_ newline - ## combination (CR, LF, CR-LF) is supported. The result strings contain no - ## trailing end of line characters unless parameter ``keepEol`` is set to - ## ``true``. - ## - ## Example: - ## - ## .. code-block:: nim - ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): - ## writeLine(stdout, line) - ## - ## Results in: - ## - ## .. code-block:: nim - ## "" - ## "this" - ## "is" - ## "an" - ## "" - ## "example" - ## "" - var first = 0 - var last = 0 - var eolpos = 0 - while true: - while last < s.len and s[last] notin {'\c', '\l'}: inc(last) - - eolpos = last - if last < s.len: - if s[last] == '\l': inc(last) - elif s[last] == '\c': - inc(last) - if last < s.len and s[last] == '\l': inc(last) - - yield substr(s, first, if keepEol: last-1 else: eolpos-1) - - # no eol characters consumed means that the string is over - if eolpos == last: - break - - first = last - -proc splitLines*(s: string, keepEol = false): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitLines".} = - ## The same as the `splitLines <#splitLines.i,string>`_ iterator, but is a - ## proc that returns a sequence of substrings. - accResult(splitLines(s, keepEol=keepEol)) - -proc countLines*(s: string): int {.noSideEffect, - rtl, extern: "nsuCountLines".} = - ## Returns the number of lines in the string `s`. - ## - ## This is the same as ``len(splitLines(s))``, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every - ## `character literal `_ newline combination - ## (CR, LF, CR-LF) is supported. - ## - ## In this context, a line is any string seperated by a newline combination. - ## A line can be an empty string. - runnableExamples: - doAssert countLines("First line\l and second line.") == 2 - result = 1 - var i = 0 - while i < s.len: - case s[i] - of '\c': - if i+1 < s.len and s[i+1] == '\l': inc i - inc result - of '\l': inc result - else: discard - inc i - -proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. - noSideEffect, rtl, extern: "nsuSplitCharSet".} = - ## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a - ## proc that returns a sequence of substrings. - runnableExamples: - doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] - doAssert "".split({' '}) == @[""] - accResult(split(s, seps, maxsplit)) proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = - ## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc - ## that returns a sequence of substrings. + ## The same as the `split iterator <#split.i,string,char,int>`_ (see its + ## documentation), but is a proc that returns a sequence of substrings. + ## + ## See also: + ## * `split iterator <#split.i,string,char,int>`_ + ## * `rsplit proc<#rsplit,string,char,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a,b,c".split(',') == @["a", "b", "c"] doAssert "".split(' ') == @[""] accResult(split(s, sep, maxsplit)) +proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. + noSideEffect, rtl, extern: "nsuSplitCharSet".} = + ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its + ## documentation), but is a proc that returns a sequence of substrings. + ## + ## See also: + ## * `split iterator <#split.i,string,set[char],int>`_ + ## * `rsplit proc<#rsplit,string,set[char],int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + runnableExamples: + doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] + doAssert "".split({' '}) == @[""] + accResult(split(s, seps, maxsplit)) + proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitString".} = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. + ## + ## See also: + ## * `split iterator <#split.i,string,string,int>`_ + ## * `rsplit proc<#rsplit,string,string,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a,b,c".split(",") == @["a", "b", "c"] doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] doAssert "".split("Elon Musk") == @[""] doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] - doAssert "a largely spaced sentence".split(" ", maxsplit=1) == @["a", " largely spaced sentence"] doAssert(sep.len > 0) accResult(split(s, sep, maxsplit)) +proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitChar".} = + ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a proc + ## that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,char,int>`_ + ## * `split proc<#split,string,char,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + accResult(rsplit(s, sep, maxsplit)) + result.reverse() + proc rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuRSplitCharSet".} = @@ -832,34 +818,17 @@ proc rsplit*(s: string, seps: set[char] = Whitespace, ## .. code-block:: nim ## @["Root#Object#Method", "Index"] ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ + ## * `split proc<#split,string,set[char],int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ accResult(rsplit(s, seps, maxsplit)) result.reverse() -proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] - {.noSideEffect, rtl, extern: "nsuRSplitChar".} = - ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a proc - ## that returns a sequence of substrings. - ## - ## A possible common use case for `rsplit` is path manipulation, - ## particularly on systems that don't use a common delimiter. - ## - ## For example, if a system had `#` as a delimiter, you could - ## do the following to get the tail of the path: - ## - ## .. code-block:: nim - ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) - ## - ## Results in `tailSplit` containing: - ## - ## .. code-block:: nim - ## @["Root#Object#Method", "Index"] - ## - accResult(rsplit(s, sep, maxsplit)) - result.reverse() - proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuRSplitString".} = - ## The same as the `rsplit iterator <#rsplit.i,string,string,int>`_, but is a proc + ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a proc ## that returns a sequence of substrings. ## ## A possible common use case for `rsplit` is path manipulation, @@ -876,9 +845,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] ## .. code-block:: nim ## @["Root#Object#Method", "Index"] ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ + ## * `split proc<#split,string,string,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a largely spaced sentence".rsplit(" ", maxsplit=1) == @["a largely spaced", "sentence"] - doAssert "a,b,c".rsplit(",") == @["a", "b", "c"] doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"] doAssert "".rsplit("Elon Musk") == @[""] @@ -886,6 +859,75 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] accResult(rsplit(s, sep, maxsplit)) result.reverse() +proc splitLines*(s: string, keepEol = false): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitLines".} = + ## The same as the `splitLines iterator<#splitLines.i,string>`_ (see its + ## documentation), but is a proc that returns a sequence of substrings. + ## + ## See also: + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `countLines proc<#countLines,string>`_ + accResult(splitLines(s, keepEol=keepEol)) + +proc splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ + ## (see its documentation), but is a proc that returns a sequence of substrings. + ## + ## See also: + ## * `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ + ## * `splitLines proc<#splitLines,string>`_ + accResult(splitWhitespace(s, maxsplit)) + +proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, + rtl, extern: "nsuToBin".} = + ## Converts `x` into its binary representation. + ## + ## The resulting string is always `len` characters long. No leading ``0b`` + ## prefix is generated. + runnableExamples: + let + a = 29 + b = 257 + doAssert a.toBin(8) == "00011101" + doAssert b.toBin(8) == "00000001" + doAssert b.toBin(9) == "100000001" + var + mask: BiggestInt = 1 + shift: BiggestInt = 0 + assert(len > 0) + result = newString(len) + for j in countdown(len-1, 0): + result[j] = chr(int((x and mask) shr shift) + ord('0')) + shift = shift + 1 + mask = mask shl 1 + +proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, + rtl, extern: "nsuToOct".} = + ## Converts `x` into its octal representation. + ## + ## The resulting string is always `len` characters long. No leading ``0o`` + ## prefix is generated. + ## + ## Do not confuse it with `toOctal proc<#toOctal,char>`_. + runnableExamples: + let + a = 62 + b = 513 + doAssert a.toOct(3) == "076" + doAssert b.toOct(3) == "001" + doAssert b.toOct(5) == "01001" + var + mask: BiggestInt = 7 + shift: BiggestInt = 0 + assert(len > 0) + result = newString(len) + for j in countdown(len-1, 0): + result[j] = chr(int((x and mask) shr shift) + ord('0')) + shift = shift + 3 + mask = mask shl 3 + proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToHex".} = ## Converts `x` to its hexadecimal representation. @@ -893,8 +935,12 @@ proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, ## The resulting string will be exactly `len` characters long. No prefix like ## ``0x`` is generated. `x` is treated as an unsigned value. runnableExamples: - doAssert toHex(1984, 6) == "0007C0" - doAssert toHex(1984, 2) == "C0" + let + a = 62 + b = 4097 + doAssert a.toHex(3) == "03E" + doAssert b.toHex(3) == "001" + doAssert b.toHex(4) == "1001" const HexChars = "0123456789ABCDEF" var @@ -917,6 +963,18 @@ proc toHex*(s: string): string {.noSideEffect, rtl.} = ## ## The output is twice the input long. No prefix like ## ``0x`` is generated. + ## + ## See also: + ## * `parseHexStr proc<#parseHexStr,string>`_ for the reverse operation + runnableExamples: + let + a = "1" + b = "A" + c = "\0\255" + doAssert a.toHex() == "31" + doAssert b.toHex() == "41" + doAssert c.toHex() == "00FF" + const HexChars = "0123456789ABCDEF" result = newString(s.len * 2) for pos, c in s: @@ -925,6 +983,25 @@ proc toHex*(s: string): string {.noSideEffect, rtl.} = n = n shr 4 result[pos * 2] = HexChars[n] +proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = + ## Converts a character `c` to its octal representation. + ## + ## The resulting string may not have a leading zero. Its length is always + ## exactly 3. + ## + ## Do not confuse it with `toOct proc<#toOct,BiggestInt,Positive>`_. + runnableExamples: + doAssert toOctal('1') == "061" + doAssert toOctal('A') == "101" + doAssert toOctal('a') == "141" + doAssert toOctal('!') == "041" + + result = newString(3) + var val = ord(c) + for i in countdown(2, 0): + result[i] = chr(val mod 8 + ord('0')) + val = val div 8 + proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, rtl, extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. @@ -980,9 +1057,10 @@ proc parseBiggestUInt*(s: string): BiggestUInt {.noSideEffect, procvar, proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = - ## Parses a decimal floating point value contained in `s`. If `s` is not - ## a valid floating point number, `ValueError` is raised. ``NAN``, - ## ``INF``, ``-INF`` are also supported (case insensitive comparison). + ## Parses a decimal floating point value contained in `s`. + ## + ## If `s` is not a valid floating point number, `ValueError` is raised. + ##``NAN``, ``INF``, ``-INF`` are also supported (case insensitive comparison). runnableExamples: doAssert parseFloat("3.14") == 3.14 doAssert parseFloat("inf") == 1.0/0 @@ -997,6 +1075,13 @@ proc parseBinInt*(s: string): int {.noSideEffect, procvar, ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have ## one of the following optional prefixes: ``0b``, ``0B``. Underscores within ## `s` are ignored. + runnableExamples: + let + a = "0b11_0101" + b = "111" + doAssert a.parseBinInt() == 53 + doAssert b.parseBinInt() == 7 + let L = parseutils.parseBin(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid binary integer: " & s) @@ -1042,11 +1127,20 @@ proc parseHexStr*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuParseHexStr".} = ## Convert hex-encoded string to byte string, e.g.: ## - ## .. code-block:: nim - ## hexToStr("00ff") == "\0\255" - ## ## Raises ``ValueError`` for an invalid hex values. The comparison is ## case-insensitive. + ## + ## See also: + ## * `toHex proc<#toHex,string>`_ for the reverse operation + runnableExamples: + let + a = "41" + b = "3161" + c = "00ff" + doAssert parseHexStr(a) == "A" + doAssert parseHexStr(b) == "1a" + doAssert parseHexStr(c) == "\0\255" + if s.len mod 2 != 0: raise newException(ValueError, "Incorrect hex string len") result = newString(s.len div 2) @@ -1067,6 +1161,10 @@ proc parseBool*(s: string): bool = ## returns `true`. If ``s`` is one of the following values: ``n, no, false, ## 0, off``, then returns `false`. If ``s`` is something else a ## ``ValueError`` exception is raised. + runnableExamples: + let a = "n" + doAssert parseBool(a) == false + case normalize(s) of "y", "yes", "true", "1", "on": result = true of "n", "no", "false", "0", "off": result = false @@ -1095,39 +1193,41 @@ proc parseEnum*[T: enum](s: string, default: T): T = proc repeat*(c: char, count: Natural): string {.noSideEffect, rtl, extern: "nsuRepeatChar".} = ## Returns a string of length `count` consisting only of - ## the character `c`. You can use this proc to left align strings. Example: - ## - ## .. code-block:: nim - ## proc tabexpand(indent: int, text: string, tabsize: int = 4) = - ## echo '\t'.repeat(indent div tabsize), ' '.repeat(indent mod tabsize), - ## text - ## - ## tabexpand(4, "At four") - ## tabexpand(5, "At five") - ## tabexpand(6, "At six") + ## the character `c`. + runnableExamples: + let a = 'z' + doAssert a.repeat(5) == "zzzzz" result = newString(count) for i in 0..count-1: result[i] = c proc repeat*(s: string, n: Natural): string {.noSideEffect, rtl, extern: "nsuRepeatStr".} = - ## Returns String `s` concatenated `n` times. Example: - ## - ## .. code-block:: nim - ## echo "+++ STOP ".repeat(4), "+++" + ## Returns string `s` concatenated `n` times. + runnableExamples: + doAssert "+ foo +".repeat(3) == "+ foo ++ foo ++ foo +" + result = newStringOfCap(n * s.len) for i in 1..n: result.add(s) -template spaces*(n: Natural): string = repeat(' ', n) - ## Returns a String with `n` space characters. You can use this proc - ## to left align strings. Example: +proc spaces*(n: Natural): string {.inline.} = + ## Returns a string with `n` space characters. You can use this proc + ## to left align strings. ## - ## .. code-block:: nim - ## let - ## width = 15 - ## text1 = "Hello user!" - ## text2 = "This is a very long string" - ## echo text1 & spaces(max(0, width - text1.len)) & "|" - ## echo text2 & spaces(max(0, width - text2.len)) & "|" + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `indent proc<#indent,string,Natural,string>`_ + ## * `center proc<#center,string,int,char>`_ + runnableExamples: + let + width = 15 + text1 = "Hello user!" + text2 = "This is a very long string" + doAssert text1 & spaces(max(0, width - text1.len)) & "|" == + "Hello user! |" + doAssert text2 & spaces(max(0, width - text2.len)) & "|" == + "This is a very long string|" + repeat(' ', n) proc align*(s: string, count: Natural, padding = ' '): string {. noSideEffect, rtl, extern: "nsuAlignString".} = @@ -1136,13 +1236,18 @@ proc align*(s: string, count: Natural, padding = ' '): string {. ## `padding` characters (by default spaces) are added before `s` resulting in ## right alignment. If ``s.len >= count``, no spaces are added and `s` is ## returned unchanged. If you need to left align a string use the `alignLeft - ## proc <#alignLeft>`_. Example: + ## proc <#alignLeft,string,Natural,Char>`_. ## - ## .. code-block:: nim - ## assert align("abc", 4) == " abc" - ## assert align("a", 0) == "a" - ## assert align("1232", 6) == " 1232" - ## assert align("1232", 6, '#') == "##1232" + ## See also: + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `indent proc<#indent,string,Natural,string>`_ + ## * `center proc<#center,string,int,char>`_ + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#') == "##1232" if s.len < count: result = newString(count) let spaces = count - s.len @@ -1157,13 +1262,18 @@ proc alignLeft*(s: string, count: Natural, padding = ' '): string {.noSideEffect ## `padding` characters (by default spaces) are added after `s` resulting in ## left alignment. If ``s.len >= count``, no spaces are added and `s` is ## returned unchanged. If you need to right align a string use the `align - ## proc <#align>`_. Example: + ## proc <#align,string,Natural,Char>`_. ## - ## .. code-block:: nim - ## assert alignLeft("abc", 4) == "abc " - ## assert alignLeft("a", 0) == "a" - ## assert alignLeft("1232", 6) == "1232 " - ## assert alignLeft("1232", 6, '#') == "1232##" + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `indent proc<#indent,string,Natural,string>`_ + ## * `center proc<#center,string,int,char>`_ + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#') == "1232##" if s.len < count: result = newString(count) if s.len > 0: @@ -1173,83 +1283,55 @@ proc alignLeft*(s: string, count: Natural, padding = ' '): string {.noSideEffect else: result = s -iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ - token: string, isSep: bool] = - ## Tokenizes the string `s` into substrings. +proc center*(s: string, width: int, fillChar: char = ' '): string {. + noSideEffect, rtl, extern: "nsuCenterString".} = + ## Return the contents of `s` centered in a string `width` long using + ## `fillChar` (default: space) as padding. ## - ## Substrings are separated by a substring containing only `seps`. - ## Examples: + ## The original string is returned if `width` is less than or equal + ## to `s.len`. ## - ## .. code-block:: nim - ## for word in tokenize(" this is an example "): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nim - ## (" ", true) - ## ("this", false) - ## (" ", true) - ## ("is", false) - ## (" ", true) - ## ("an", false) - ## (" ", true) - ## ("example", false) - ## (" ", true) - var i = 0 - while true: - var j = i - var isSep = j < s.len and s[j] in seps - while j < s.len and (s[j] in seps) == isSep: inc(j) - if j > i: - yield (substr(s, i, j-1), isSep) + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `indent proc<#indent,string,Natural,string>`_ + runnableExamples: + let a = "foo" + doAssert a.center(2) == "foo" + doAssert a.center(5) == " foo " + doAssert a.center(6) == " foo " + if width <= s.len: return s + result = newString(width) + # Left padding will be one fillChar + # smaller if there are an odd number + # of characters + let + charsLeft = (width - s.len) + leftPadding = charsLeft div 2 + for i in 0 ..< width: + if i >= leftPadding and i < leftPadding + s.len: + # we are where the string should be located + result[i] = s[i-leftPadding] else: - break - i = j - -proc wordWrap*(s: string, maxLineWidth = 80, - splitLongWords = true, - seps: set[char] = Whitespace, - newLine = "\n"): string {. - noSideEffect, rtl, extern: "nsuWordWrap", - deprecated: "use wrapWords in std/wordwrap instead".} = - ## Word wraps `s`. - result = newStringOfCap(s.len + s.len shr 6) - var spaceLeft = maxLineWidth - var lastSep = "" - for word, isSep in tokenize(s, seps): - if isSep: - lastSep = word - spaceLeft = spaceLeft - len(word) - continue - if len(word) > spaceLeft: - if splitLongWords and len(word) > maxLineWidth: - result.add(substr(word, 0, spaceLeft-1)) - var w = spaceLeft - var wordLeft = len(word) - spaceLeft - while wordLeft > 0: - result.add(newLine) - var L = min(maxLineWidth, wordLeft) - spaceLeft = maxLineWidth - L - result.add(substr(word, w, w+L-1)) - inc(w, L) - dec(wordLeft, L) - else: - spaceLeft = maxLineWidth - len(word) - result.add(newLine) - result.add(word) - else: - spaceLeft = spaceLeft - len(word) - result.add(lastSep & word) - lastSep.setLen(0) + # we are either before or after where + # the string s should go + result[i] = fillChar proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## ## **Note:** This does not preserve the new line characters used in ``s``. + ## + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `unindent proc<#unindent,string,Natural,string>`_ runnableExamples: - doAssert indent("First line\c\l and second line.", 2) == " First line\l and second line." + doAssert indent("First line\c\l and second line.", 2) == + " First line\l and second line." result = "" var i = 0 for line in s.splitLines(): @@ -1266,8 +1348,15 @@ proc unindent*(s: string, count: Natural, padding: string = " "): string ## Sometimes called `dedent`:idx: ## ## **Note:** This does not preserve the new line characters used in ``s``. + ## + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `indent proc<#indent,string,Natural,string>`_ runnableExamples: - doAssert unindent(" First line\l and second line", 3) == "First line\land second line" + doAssert unindent(" First line\l and second line", 3) == + "First line\land second line" result = "" var i = 0 for line in s.splitLines(): @@ -1286,37 +1375,105 @@ proc unindent*(s: string): string {.noSideEffect, rtl, extern: "nsuUnindentAll".} = ## Removes all indentation composed of whitespace from each line in ``s``. ## - ## For example: - ## - ## .. code-block:: nim - ## const x = """ - ## Hello - ## There - ## """.unindent() - ## - ## doAssert x == "Hello\nThere\n" + ## See also: + ## * `align proc<#align,string,Natural,Char>`_ + ## * `alignLeft proc<#alignLeft,string,Natural,Char>`_ + ## * `spaces proc<#spaces,Natural>`_ + ## * `indent proc<#indent,string,Natural,string>`_ + runnableExamples: + let x = """ + Hello + There + """.unindent() + + doAssert x == "Hello\nThere\n" unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. +proc delete*(s: var string, first, last: int) {.noSideEffect, + rtl, extern: "nsuDelete".} = + ## Deletes in `s` (must be declared as ``var``) the characters at positions + ## ``first ..last`` (both ends included). + ## + ## This modifies `s` itself, it does not return a copy. + runnableExamples: + var a = "abracadabra" + + a.delete(4, 5) + doAssert a == "abradabra" + + a.delete(1, 6) + doAssert a == "ara" + + var i = first + var j = last+1 + var newLen = len(s)-j+i + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) + + +proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` starts with character ``prefix``. + ## + ## See also: + ## * `endsWith proc<#endsWith,string,char>`_ + ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ + ## * `removePrefix proc<#removePrefix,string,char>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.startsWith('a') == true + doAssert a.startsWith('b') == false + result = s.len > 0 and s[0] == prefix + proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = - ## Returns true iff ``s`` starts with ``prefix``. + ## Returns true if ``s`` starts with string ``prefix``. ## ## If ``prefix == ""`` true is returned. + ## + ## See also: + ## * `endsWith proc<#endsWith,string,string>`_ + ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ + ## * `removePrefix proc<#removePrefix,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.startsWith("abra") == true + doAssert a.startsWith("bra") == false var i = 0 while true: if i >= prefix.len: return true if i >= s.len or s[i] != prefix[i]: return false inc(i) -proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = - ## Returns true iff ``s`` starts with ``prefix``. - result = s.len > 0 and s[0] == prefix +proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` ends with ``suffix``. + ## + ## See also: + ## * `startsWith proc<#startsWith,string,char>`_ + ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix proc<#removeSuffix,string,char>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.endsWith('a') == true + doAssert a.endsWith('b') == false + result = s.len > 0 and s[s.high] == suffix proc endsWith*(s, suffix: string): bool {.noSideEffect, rtl, extern: "nsuEndsWith".} = - ## Returns true iff ``s`` ends with ``suffix``. + ## Returns true if ``s`` ends with ``suffix``. ## ## If ``suffix == ""`` true is returned. + ## + ## See also: + ## * `startsWith proc<#startsWith,string,string>`_ + ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix proc<#removeSuffix,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.endsWith("abra") == true + doAssert a.endsWith("dab") == false var i = 0 var j = len(s) - len(suffix) while i+j <% s.len: @@ -1324,21 +1481,136 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, inc(i) if i >= suffix.len: return true -proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = - ## Returns true iff ``s`` ends with ``suffix``. - result = s.len > 0 and s[s.high] == suffix - proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = - ## Returns true iff ``s`` continues with ``substr`` at position ``start``. + ## Returns true if ``s`` continues with ``substr`` at position ``start``. ## ## If ``substr == ""`` true is returned. + ## + ## See also: + ## * `startsWith proc<#startsWith,string,string>`_ + ## * `endsWith proc<#endsWith,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.continuesWith("ca", 4) == true + doAssert a.continuesWith("ca", 5) == false + doAssert a.continuesWith("dab", 6) == true var i = 0 while true: if i >= substr.len: return true if i+start >= s.len or s[i+start] != substr[i]: return false inc(i) + +proc removePrefix*(s: var string, chars: set[char] = Newlines) {. + rtl, extern: "nsuRemovePrefixCharSet".} = + ## Removes all characters from `chars` from the start of the string `s` + ## (in-place). + ## + ## See also: + ## * `removeSuffix proc<#removeSuffix,string,set[char]>`_ + runnableExamples: + var userInput = "\r\n*~Hello World!" + userInput.removePrefix + doAssert userInput == "*~Hello World!" + userInput.removePrefix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "?!?Hello!?!" + otherInput.removePrefix({'!', '?'}) + doAssert otherInput == "Hello!?!" + + var start = 0 + while start < s.len and s[start] in chars: start += 1 + if start > 0: s.delete(0, start - 1) + +proc removePrefix*(s: var string, c: char) {. + rtl, extern: "nsuRemovePrefixChar".} = + ## Removes all occurrences of a single character (in-place) from the start + ## of a string. + ## + ## See also: + ## * `removeSuffix proc<#removeSuffix,string,char>`_ + ## * `startsWith proc<#startsWith,string,char>`_ + runnableExamples: + var ident = "pControl" + ident.removePrefix('p') + doAssert ident == "Control" + removePrefix(s, chars = {c}) + +proc removePrefix*(s: var string, prefix: string) {. + rtl, extern: "nsuRemovePrefixString".} = + ## Remove the first matching prefix (in-place) from a string. + ## + ## See also: + ## * `removeSuffix proc<#removeSuffix,string,string>`_ + ## * `startsWith proc<#startsWith,string,string>`_ + runnableExamples: + var answers = "yesyes" + answers.removePrefix("yes") + doAssert answers == "yes" + if s.startsWith(prefix): + s.delete(0, prefix.len - 1) + +proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. + rtl, extern: "nsuRemoveSuffixCharSet".} = + ## Removes all characters from `chars` from the end of the string `s` + ## (in-place). + ## + ## See also: + ## * `removePrefix proc<#removePrefix,string,set[char]>`_ + runnableExamples: + var userInput = "Hello World!*~\r\n" + userInput.removeSuffix + doAssert userInput == "Hello World!*~" + userInput.removeSuffix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "Hello!?!" + otherInput.removeSuffix({'!', '?'}) + doAssert otherInput == "Hello" + + if s.len == 0: return + var last = s.high + while last > -1 and s[last] in chars: last -= 1 + s.setLen(last + 1) + +proc removeSuffix*(s: var string, c: char) {. + rtl, extern: "nsuRemoveSuffixChar".} = + ## Removes all occurrences of a single character (in-place) from the end + ## of a string. + ## + ## See also: + ## * `removePrefix proc<#removePrefix,string,char>`_ + ## * `endsWith proc<#endsWith,string,char>`_ + runnableExamples: + var table = "users" + table.removeSuffix('s') + doAssert table == "user" + + var dots = "Trailing dots......." + dots.removeSuffix('.') + doAssert dots == "Trailing dots" + + removeSuffix(s, chars = {c}) + +proc removeSuffix*(s: var string, suffix: string) {. + rtl, extern: "nsuRemoveSuffixString".} = + ## Remove the first matching suffix (in-place) from a string. + ## + ## See also: + ## * `removePrefix proc<#removePrefix,string,string>`_ + ## * `endsWith proc<#endsWith,string,string>`_ + runnableExamples: + var answers = "yeses" + answers.removeSuffix("es") + doAssert answers == "yes" + var newLen = s.len + if s.endsWith(suffix): + newLen -= len(suffix) + s.setLen(newLen) + + proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.noSideEffect, inline.} = ## Adds a separator to `dest` only if its length is bigger than `startLen`. @@ -1353,24 +1625,28 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) ## `startLen`. The following example creates a string describing ## an array of integers. runnableExamples: - var arr = "[" - for x in items([2, 3, 5, 7, 11]): - addSep(arr, startLen=len("[")) - add(arr, $x) - add(arr, "]") + var arr = "[" + for x in items([2, 3, 5, 7, 11]): + addSep(arr, startLen=len("[")) + add(arr, $x) + add(arr, "]") + doAssert arr == "[2, 3, 5, 7, 11]" + if dest.len > startLen: add(dest, sep) proc allCharsInSet*(s: string, theSet: set[char]): bool = - ## Returns true iff each character of `s` is in the set `theSet`. + ## Returns true if every character of `s` is in the set `theSet`. runnableExamples: doAssert allCharsInSet("aeea", {'a', 'e'}) == true doAssert allCharsInSet("", {'a', 'e'}) == true + for c in items(s): if c notin theSet: return false return true proc abbrev*(s: string, possibilities: openArray[string]): int = - ## Returns the index of the first item in ``possibilities`` which starts with ``s``, if not ambiguous. + ## Returns the index of the first item in ``possibilities`` which starts + ## with ``s``, if not ambiguous. ## ## Returns -1 if no item has been found and -2 if multiple items match. runnableExamples: @@ -1378,6 +1654,7 @@ proc abbrev*(s: string, possibilities: openArray[string]): int = doAssert abbrev("foo", ["college", "faculty", "industry"]) == -1 # Not found doAssert abbrev("fac", ["college", "faculty", "faculties"]) == -2 # Ambiguous doAssert abbrev("college", ["college", "colleges", "industry"]) == 0 + result = -1 # none found for i in 0..possibilities.len-1: if possibilities[i].startsWith(s): @@ -1391,9 +1668,10 @@ proc abbrev*(s: string, possibilities: openArray[string]): int = proc join*(a: openArray[string], sep: string = ""): string {. noSideEffect, rtl, extern: "nsuJoinSep".} = - ## Concatenates all strings in `a` separating them with `sep`. + ## Concatenates all strings in the container `a`, separating them with `sep`. runnableExamples: doAssert join(["A", "B", "Conclusion"], " -> ") == "A -> B -> Conclusion" + if len(a) > 0: var L = sep.len * (a.len-1) for i in 0..high(a): inc(L, a[i].len) @@ -1407,10 +1685,11 @@ proc join*(a: openArray[string], sep: string = ""): string {. proc join*[T: not string](a: openArray[T], sep: string = ""): string {. noSideEffect, rtl.} = - ## Converts all elements in `a` to strings using `$` and concatenates them - ## with `sep`. + ## Converts all elements in the container `a` to strings using `$`, + ## and concatenates them with `sep`. runnableExamples: doAssert join([1, 2, 3], " -> ") == "1 -> 2 -> 3" + result = "" for i, x in a: if i > 0: @@ -1441,11 +1720,11 @@ proc initSkipTable*(a: var SkipTable, sub: string) proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {.noSideEffect, rtl, extern: "nsuFindStrA".} = - ## Searches for `sub` in `s` inside range `start`..`last` using preprocessed table `a`. - ## If `last` is unspecified, it defaults to `s.high`. + ## Searches for `sub` in `s` inside range `start`..`last` using preprocessed + ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last + ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - let last = if last==0: s.high else: last subLast = sub.len - 1 @@ -1466,7 +1745,6 @@ proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): @@ -1478,10 +1756,14 @@ else: proc find*(s: string, sub: char, start: Natural = 0, last = 0): int {.noSideEffect, rtl, extern: "nsuFindChar".} = - ## Searches for `sub` in `s` inside range `start`..`last`. - ## If `last` is unspecified, it defaults to `s.high`. + ## Searches for `sub` in `s` inside range ``start..last`` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `rfind proc<#rfind,string,char,int>`_ + ## * `replace proc<#replace,string,char,char>`_ let last = if last==0: s.high else: last when nimvm: for i in int(start)..last: @@ -1498,34 +1780,72 @@ proc find*(s: string, sub: char, start: Natural = 0, last = 0): int {.noSideEffe if sub == s[i]: return i return -1 +proc find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {.noSideEffect, + rtl, extern: "nsuFindCharSet".} = + ## Searches for `chars` in `s` inside range ``start..last`` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## If `s` contains none of the characters in `chars`, -1 is returned. + ## + ## See also: + ## * `rfind proc<#rfind,string,set[char],int>`_ + ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ + let last = if last==0: s.high else: last + for i in int(start)..last: + if s[i] in chars: return i + return -1 + proc find*(s, sub: string, start: Natural = 0, last = 0): int {.noSideEffect, rtl, extern: "nsuFindStr".} = - ## Searches for `sub` in `s` inside range `start`..`last`. - ## If `last` is unspecified, it defaults to `s.high`. + ## Searches for `sub` in `s` inside range ``start..last`` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `rfind proc<#rfind,string,string,int>`_ + ## * `replace proc<#replace,string,string,string>`_ if sub.len > s.len: return -1 if sub.len == 1: return find(s, sub[0], start, last) var a {.noinit.}: SkipTable initSkipTable(a, sub) result = find(a, s, sub, start, last) -proc find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {.noSideEffect, - rtl, extern: "nsuFindCharSet".} = - ## Searches for `chars` in `s` inside range `start`..`last`. - ## If `last` is unspecified, it defaults to `s.high`. +proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, + rtl.} = + ## Searches for characer `sub` in `s` in reverse, starting at position `start` + ## (default: the last character) and going backwards to the first character. ## - ## If `s` contains none of the characters in `chars`, -1 is returned. - let last = if last==0: s.high else: last - for i in int(start)..last: + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `find proc<#find,string,char,int,int>`_ + let realStart = if start == -1: s.len-1 else: start + for i in countdown(realStart, 0): + if sub == s[i]: return i + return -1 + +proc rfind*(s: string, chars: set[char], start: int = -1): int {.noSideEffect.} = + ## Searches for `chars` in `s` in reverse, starting at position `start` + ## (default: the last character) and going backwards to the first character. + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `find proc<#find,string,set[char],Natural,int>`_ + let realStart = if start == -1: s.len-1 else: start + for i in countdown(realStart, 0): if s[i] in chars: return i return -1 proc rfind*(s, sub: string, start: int = -1): int {.noSideEffect.} = - ## Searches for `sub` in `s` in reverse, starting at `start` and going - ## backwards to 0. + ## Searches for string `sub` in `s` in reverse, starting at position `start` + ## (default: the last character) and going backwards to the first character. ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `find proc<#find,string,string,Natural,int>`_ if sub.len == 0: return -1 let realStart = if start == -1: s.len else: start @@ -1538,54 +1858,34 @@ proc rfind*(s, sub: string, start: int = -1): int {.noSideEffect.} = if result != -1: return return -1 -proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, - rtl.} = - ## Searches for `sub` in `s` in reverse starting at position `start`. - ## - ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - let realStart = if start == -1: s.len-1 else: start - for i in countdown(realStart, 0): - if sub == s[i]: return i - return -1 -proc rfind*(s: string, chars: set[char], start: int = -1): int {.noSideEffect.} = - ## Searches for `chars` in `s` in reverse starting at position `start`. +proc count*(s: string, sub: char): int {.noSideEffect, + rtl, extern: "nsuCountChar".} = + ## Count the occurrences of the character `sub` in the string `s`. ## - ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - let realStart = if start == -1: s.len-1 else: start - for i in countdown(realStart, 0): - if s[i] in chars: return i - return -1 + ## See also: + ## * `countLines proc<#countLines,string>`_ + for c in s: + if c == sub: inc result -proc center*(s: string, width: int, fillChar: char = ' '): string {. - noSideEffect, rtl, extern: "nsuCenterString".} = - ## Return the contents of `s` centered in a string `width` long using - ## `fillChar` as padding. +proc count*(s: string, subs: set[char]): int {.noSideEffect, + rtl, extern: "nsuCountCharSet".} = + ## Count the occurrences of the group of character `subs` in the string `s`. ## - ## The original string is returned if `width` is less than or equal - ## to `s.len`. - if width <= s.len: return s - result = newString(width) - # Left padding will be one fillChar - # smaller if there are an odd number - # of characters - let - charsLeft = (width - s.len) - leftPadding = charsLeft div 2 - for i in 0 ..< width: - if i >= leftPadding and i < leftPadding + s.len: - # we are where the string should be located - result[i] = s[i-leftPadding] - else: - # we are either before or after where - # the string s should go - result[i] = fillChar + ## See also: + ## * `countLines proc<#countLines,string>`_ + doAssert card(subs) > 0 + for c in s: + if c in subs: inc result proc count*(s: string, sub: string, overlapping: bool = false): int {. noSideEffect, rtl, extern: "nsuCountString".} = ## Count the occurrences of a substring `sub` in the string `s`. ## Overlapping occurrences of `sub` only count when `overlapping` - ## is set to true. + ## is set to true (default: false). + ## + ## See also: + ## * `countLines proc<#countLines,string>`_ doAssert sub.len > 0 var i = 0 while true: @@ -1595,43 +1895,58 @@ proc count*(s: string, sub: string, overlapping: bool = false): int {. else: i += sub.len inc result -proc count*(s: string, sub: char): int {.noSideEffect, - rtl, extern: "nsuCountChar".} = - ## Count the occurrences of the character `sub` in the string `s`. - for c in s: - if c == sub: inc result - -proc count*(s: string, subs: set[char]): int {.noSideEffect, - rtl, extern: "nsuCountCharSet".} = - ## Count the occurrences of the group of character `subs` in the string `s`. - doAssert card(subs) > 0 - for c in s: - if c in subs: inc result - -proc quoteIfContainsWhite*(s: string): string {.deprecated.} = - ## Returns ``'"' & s & '"'`` if `s` contains a space and does not - ## start with a quote, else returns `s`. +proc countLines*(s: string): int {.noSideEffect, + rtl, extern: "nsuCountLines".} = + ## Returns the number of lines in the string `s`. ## - ## **DEPRECATED** as it was confused for shell quoting function. For this - ## application use `osproc.quoteShell `_. - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': result = '"' & s & '"' - else: result = s + ## This is the same as ``len(splitLines(s))``, but much more efficient + ## because it doesn't modify the string creating temporal objects. Every + ## `character literal `_ + ## newline combination (CR, LF, CR-LF) is supported. + ## + ## In this context, a line is any string seperated by a newline combination. + ## A line can be an empty string. + ## + ## See also: + ## * `splitLines proc<#splitLines,string>`_ + runnableExamples: + doAssert countLines("First line\l and second line.") == 2 + result = 1 + var i = 0 + while i < s.len: + case s[i] + of '\c': + if i+1 < s.len and s[i+1] == '\l': inc i + inc result + of '\l': inc result + else: discard + inc i -proc contains*(s: string, c: char): bool {.noSideEffect.} = - ## Same as ``find(s, c) >= 0``. - return find(s, c) >= 0 proc contains*(s, sub: string): bool {.noSideEffect.} = ## Same as ``find(s, sub) >= 0``. + ## + ## See also: + ## * `find proc<#find,string,string,Natural,int>`_ return find(s, sub) >= 0 proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} = ## Same as ``find(s, chars) >= 0``. + ## + ## See also: + ## * `find proc<#find,string,set[char],Natural,int>`_ return find(s, chars) >= 0 proc replace*(s, sub: string, by = ""): string {.noSideEffect, rtl, extern: "nsuReplaceStr".} = ## Replaces `sub` in `s` by the string `by`. + ## + ## See also: + ## * `find proc<#find,string,string,Natural,int>`_ + ## * `replace proc<#replace,string,char,char>`_ for replacing + ## single characters + ## * `replaceWord proc<#replaceWord,string,string,string>`_ + ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ result = "" let subLen = sub.len if subLen == 0: @@ -1669,6 +1984,11 @@ proc replace*(s: string, sub, by: char): string {.noSideEffect, ## Replaces `sub` in `s` by the character `by`. ## ## Optimized version of `replace <#replace,string,string>`_ for characters. + ## + ## See also: + ## * `find proc<#find,string,char,Natural,int>`_ + ## * `replaceWord proc<#replaceWord,string,string,string>`_ + ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ result = newString(s.len) var i = 0 while i < s.len: @@ -1681,7 +2001,7 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, ## Replaces `sub` in `s` by the string `by`. ## ## Each occurrence of `sub` has to be surrounded by word boundaries - ## (comparable to ``\\w`` in regular expressions), otherwise it is not + ## (comparable to ``\b`` in regular expressions), otherwise it is not ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} @@ -1711,14 +2031,14 @@ proc multiReplace*(s: string, replacements: varargs[(string, string)]): string { ## Same as replace, but specialized for doing multiple replacements in a single ## pass through the input string. ## - ## multiReplace performs all replacements in a single pass, this means it can be used - ## to swap the occurences of "a" and "b", for instance. + ## `multiReplace` performs all replacements in a single pass, this means it + ## can be used to swap the occurences of "a" and "b", for instance. ## - ## If the resulting string is not longer than the original input string, only a single - ## memory allocation is required. + ## If the resulting string is not longer than the original input string, + ## only a single memory allocation is required. ## - ## The order of the replacements does matter. Earlier replacements are preferred over later - ## replacements in the argument list. + ## The order of the replacements does matter. Earlier replacements are + ## preferred over later replacements in the argument list. result = newStringOfCap(s.len) var i = 0 var fastChk: set[char] = {} @@ -1740,55 +2060,12 @@ proc multiReplace*(s: string, replacements: varargs[(string, string)]): string { add result, s[i] inc(i) -proc delete*(s: var string, first, last: int) {.noSideEffect, - rtl, extern: "nsuDelete".} = - ## Deletes in `s` the characters at position `first` .. `last`. - ## - ## This modifies `s` itself, it does not return a copy. - var i = first - var j = last+1 - var newLen = len(s)-j+i - while i < newLen: - s[i] = s[j] - inc(i) - inc(j) - setLen(s, newLen) -proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToOct".} = - ## Converts `x` into its octal representation. - ## - ## The resulting string is always `len` characters long. No leading ``0o`` - ## prefix is generated. - var - mask: BiggestInt = 7 - shift: BiggestInt = 0 - assert(len > 0) - result = newString(len) - for j in countdown(len-1, 0): - result[j] = chr(int((x and mask) shr shift) + ord('0')) - shift = shift + 3 - mask = mask shl 3 - -proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToBin".} = - ## Converts `x` into its binary representation. - ## - ## The resulting string is always `len` characters long. No leading ``0b`` - ## prefix is generated. - var - mask: BiggestInt = 1 - shift: BiggestInt = 0 - assert(len > 0) - result = newString(len) - for j in countdown(len-1, 0): - result[j] = chr(int((x and mask) shr shift) + ord('0')) - shift = shift + 1 - mask = mask shl 1 proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, rtl, extern: "nsuInsertSep".} = - ## Inserts the separator `sep` after `digits` digits from right to left. + ## Inserts the separator `sep` after `digits` characters (default: 3) + ## from right to left. ## ## Even though the algorithm works with any string `s`, it is only useful ## if `s` contains a number. @@ -1810,11 +2087,15 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. See `system.addEscapedChar `_ - ## for the escaping scheme. + ## Escapes a string `s`. See `system.addEscapedChar + ## `_ for the escaping scheme. ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. + ## + ## See also: + ## * `unescape proc<#unescape,string,string,string>`_ for the opposite + ## operation result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): @@ -1832,8 +2113,8 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, rtl, extern: "nsuUnescape".} = ## Unescapes a string `s`. ## - ## This complements `escape <#escape>`_ as it performs the opposite - ## operations. + ## This complements `escape proc<#escape,string,string,string>`_ + ## as it performs the opposite operations. ## ## If `s` does not begin with ``prefix`` and end with ``suffix`` a ## ValueError exception will be raised. @@ -1879,101 +2160,12 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## and is followed by any number of characters of the set `IdentChars`. runnableExamples: doAssert "abc_def08".validIdentifier + if s.len > 0 and s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false return true -{.push warning[Deprecated]: off.} -proc editDistance*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuEditDistance", - deprecated: "use editdistance.editDistanceAscii instead".} = - ## Returns the edit distance between `a` and `b`. - ## - ## This uses the `Levenshtein`:idx: distance algorithm with only a linear - ## memory overhead. - var len1 = a.len - var len2 = b.len - if len1 > len2: - # make `b` the longer string - return editDistance(b, a) - - # strip common prefix: - var s = 0 - while s < len1 and a[s] == b[s]: - inc(s) - dec(len1) - dec(len2) - # strip common suffix: - while len1 > 0 and len2 > 0 and a[s+len1-1] == b[s+len2-1]: - dec(len1) - dec(len2) - # trivial cases: - if len1 == 0: return len2 - if len2 == 0: return len1 - - # another special case: - if len1 == 1: - for j in s..s+len2-1: - if a[s] == b[j]: return len2 - 1 - return len2 - - inc(len1) - inc(len2) - var half = len1 shr 1 - # initalize first row: - #var row = cast[ptr array[0..high(int) div 8, int]](alloc(len2*sizeof(int))) - var row: seq[int] - newSeq(row, len2) - var e = s + len2 - 1 # end marker - for i in 1..len2 - half - 1: row[i] = i - row[0] = len1 - half - 1 - for i in 1 .. len1 - 1: - var char1 = a[i + s - 1] - var char2p: int - var D, x: int - var p: int - if i >= len1 - half: - # skip the upper triangle: - var offset = i - len1 + half - char2p = offset - p = offset - var c3 = row[p] + ord(char1 != b[s + char2p]) - inc(p) - inc(char2p) - x = row[p] + 1 - D = x - if x > c3: x = c3 - row[p] = x - inc(p) - else: - p = 1 - char2p = 0 - D = i - x = i - if i <= half + 1: - # skip the lower triangle: - e = len2 + i - half - 2 - # main: - while p <= e: - dec(D) - var c3 = D + ord(char1 != b[char2p + s]) - inc(char2p) - inc(x) - if x > c3: x = c3 - D = row[p] + 1 - if x > D: x = D - row[p] = x - inc(p) - # lower triangle sentinel: - if i <= half: - dec(D) - var c3 = D + ord(char1 != b[char2p + s]) - inc(x) - if x > c3: x = c3 - row[p] = x - result = row[e] -{.pop.} # floating point formating: when not defined(js): @@ -1981,10 +2173,11 @@ when not defined(js): importc: "sprintf", varargs, noSideEffect.} type - FloatFormatMode* = enum ## the different modes of floating point formating - ffDefault, ## use the shorter floating point notation - ffDecimal, ## use decimal floating point notation - ffScientific ## use scientific notation (using ``e`` character) + FloatFormatMode* = enum + ## the different modes of floating point formating + ffDefault, ## use the shorter floating point notation + ffDecimal, ## use decimal floating point notation + ffScientific ## use scientific notation (using ``e`` character) proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; @@ -2000,6 +2193,11 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``biggestFloat`` type. ## ## If ``precision == -1``, it tries to format it nicely. + runnableExamples: + let x = 123.456 + doAssert x.formatBiggestFloat() == "123.4560000000000" + doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" when defined(js): var precision = precision if precision == -1: @@ -2079,11 +2277,18 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, doAssert x.formatFloat() == "123.4560000000000" doAssert x.formatFloat(ffDecimal, 4) == "123.4560" doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" + result = formatBiggestFloat(f, format, precision, decimalSep) proc trimZeros*(x: var string) {.noSideEffect.} = ## Trim trailing zeros from a formatted floating point - ## value (`x`). Modifies the passed value. + ## value `x` (must be declared as ``var``). + ## + ## This modifies `x` itself, it does not return a copy. + runnableExamples: + var x = "123.456000000" + x.trimZeros() + doAssert x == "123.456" var spl: seq[string] if x.contains('.') or x.contains(','): if x.contains('e'): @@ -2113,6 +2318,9 @@ proc formatSize*(bytes: int64, ## ## `includeSpace` can be set to true to include the (SI preferred) space ## between the number and the unit (e.g. 1 KiB). + ## + ## See also: + ## * `strformat module`_ for string interpolation and formatting runnableExamples: doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" @@ -2120,6 +2328,7 @@ proc formatSize*(bytes: int64, doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" doAssert formatSize(4096) == "4KiB" doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] var @@ -2215,6 +2424,9 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true ## ## `decimalSep` is used as the decimal separator. + ## + ## See also: + ## * `strformat module`_ for string interpolation and formatting var absolute: BiggestFloat significand: BiggestFloat @@ -2402,110 +2614,69 @@ proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. + ## + ## See also: + ## * `strformat module`_ for string interpolation and formatting result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) proc `%` *(formatstr, a: string): string {.noSideEffect, rtl, extern: "nsuFormatSingleElem".} = - ## This is the same as ``formatstr % [a]``. + ## This is the same as ``formatstr % [a]`` (see + ## `% proc<#%25,string,openArray[string]>`_). result = newStringOfCap(formatstr.len + a.len) addf(result, formatstr, [a]) proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect, rtl, extern: "nsuFormatVarargs".} = - ## This is the same as ``formatstr % a`` except that it supports + ## This is the same as ``formatstr % a`` (see + ## `% proc<#%25,string,openArray[string]>`_) except that it supports ## auto stringification. + ## + ## See also: + ## * `strformat module`_ for string interpolation and formatting result = newStringOfCap(formatstr.len + a.len) addf(result, formatstr, a) {.pop.} -proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. - rtl, extern: "nsuRemoveSuffixCharSet".} = - ## Removes all characters from `chars` from the end of the string `s` - ## (in-place). - runnableExamples: - var userInput = "Hello World!*~\r\n" - userInput.removeSuffix - doAssert userInput == "Hello World!*~" - userInput.removeSuffix({'~', '*'}) - doAssert userInput == "Hello World!" - var otherInput = "Hello!?!" - otherInput.removeSuffix({'!', '?'}) - doAssert otherInput == "Hello" - if s.len == 0: return - var last = s.high - while last > -1 and s[last] in chars: last -= 1 - s.setLen(last + 1) -proc removeSuffix*(s: var string, c: char) {. - rtl, extern: "nsuRemoveSuffixChar".} = - ## Removes all occurrences of a single character (in-place) from the end - ## of a string. +proc strip*(s: string, leading = true, trailing = true, + chars: set[char] = Whitespace): string + {.noSideEffect, rtl, extern: "nsuStrip".} = + ## Strips leading or trailing `chars` (default: whitespace characters) + ## from `s` and returns the resulting string. ## - runnableExamples: - var table = "users" - table.removeSuffix('s') - doAssert table == "user" - - var dots = "Trailing dots......." - dots.removeSuffix('.') - doAssert dots == "Trailing dots" - removeSuffix(s, chars = {c}) - -proc removeSuffix*(s: var string, suffix: string) {. - rtl, extern: "nsuRemoveSuffixString".} = - ## Remove the first matching suffix (in-place) from a string. - runnableExamples: - var answers = "yeses" - answers.removeSuffix("es") - doAssert answers == "yes" - var newLen = s.len - if s.endsWith(suffix): - newLen -= len(suffix) - s.setLen(newLen) - -proc removePrefix*(s: var string, chars: set[char] = Newlines) {. - rtl, extern: "nsuRemovePrefixCharSet".} = - ## Removes all characters from `chars` from the start of the string `s` - ## (in-place). + ## If `leading` is true (default), leading `chars` are stripped. + ## If `trailing` is true (default), trailing `chars` are stripped. + ## If both are false, the string is returned unchanged. ## + ## See also: + ## * `stripLineEnd proc<#stripLineEnd,string>`_ runnableExamples: - var userInput = "\r\n*~Hello World!" - userInput.removePrefix - doAssert userInput == "*~Hello World!" - userInput.removePrefix({'~', '*'}) - doAssert userInput == "Hello World!" + let a = " vhellov " + let b = strip(a) + doAssert b == "vhellov" - var otherInput = "?!?Hello!?!" - otherInput.removePrefix({'!', '?'}) - doAssert otherInput == "Hello!?!" - var start = 0 - while start < s.len and s[start] in chars: start += 1 - if start > 0: s.delete(0, start - 1) + doAssert a.strip(leading = false) == " vhellov" + doAssert a.strip(trailing = false) == "vhellov " -proc removePrefix*(s: var string, c: char) {. - rtl, extern: "nsuRemovePrefixChar".} = - ## Removes all occurrences of a single character (in-place) from the start - ## of a string. - ## - runnableExamples: - var ident = "pControl" - ident.removePrefix('p') - doAssert ident == "Control" - removePrefix(s, chars = {c}) + doAssert b.strip(chars = {'v'}) == "hello" + doAssert b.strip(leading = false, chars = {'v'}) == "vhello" -proc removePrefix*(s: var string, prefix: string) {. - rtl, extern: "nsuRemovePrefixString".} = - ## Remove the first matching prefix (in-place) from a string. - ## - runnableExamples: - var answers = "yesyes" - answers.removePrefix("yes") - doAssert answers == "yes" - if s.startsWith(prefix): - s.delete(0, prefix.len - 1) + let c = "blaXbla" + doAssert c.strip(chars = {'b', 'a'}) == "laXbl" + doAssert c.strip(chars = {'b', 'a', 'l'}) == "X" + + var + first = 0 + last = len(s)-1 + if leading: + while first <= last and s[first] in chars: inc(first) + if trailing: + while last >= 0 and s[last] in chars: dec(last) + result = substr(s, first, last) proc stripLineEnd*(s: var string) = ## Returns ``s`` stripped from one of these suffixes: @@ -2519,6 +2690,7 @@ proc stripLineEnd*(s: var string) = s = "foo\r\n" s.stripLineEnd doAssert s == "foo" + if s.len > 0: case s[^1] of '\n': @@ -2531,6 +2703,331 @@ proc stripLineEnd*(s: var string) = else: discard + +iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ + token: string, isSep: bool] = + ## Tokenizes the string `s` into substrings. + ## + ## Substrings are separated by a substring containing only `seps`. + ## Example: + ## + ## .. code-block:: nim + ## for word in tokenize(" this is an example "): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: nim + ## (" ", true) + ## ("this", false) + ## (" ", true) + ## ("is", false) + ## (" ", true) + ## ("an", false) + ## (" ", true) + ## ("example", false) + ## (" ", true) + var i = 0 + while true: + var j = i + var isSep = j < s.len and s[j] in seps + while j < s.len and (s[j] in seps) == isSep: inc(j) + if j > i: + yield (substr(s, i, j-1), isSep) + else: + break + i = j + + + + + +# -------------------------------------------------------------------------- +# Deprecated procs + +{.push warning[Deprecated]: off.} +proc editDistance*(a, b: string): int {.noSideEffect, + rtl, extern: "nsuEditDistance", + deprecated: "use editdistance.editDistanceAscii instead".} = + ## **Deprecated**: Use `editdistance module`_ + ## + ## Returns the edit distance between `a` and `b`. + ## + ## This uses the `Levenshtein`:idx: distance algorithm with only a linear + ## memory overhead. + var len1 = a.len + var len2 = b.len + if len1 > len2: + # make `b` the longer string + return editDistance(b, a) + + # strip common prefix: + var s = 0 + while s < len1 and a[s] == b[s]: + inc(s) + dec(len1) + dec(len2) + # strip common suffix: + while len1 > 0 and len2 > 0 and a[s+len1-1] == b[s+len2-1]: + dec(len1) + dec(len2) + # trivial cases: + if len1 == 0: return len2 + if len2 == 0: return len1 + + # another special case: + if len1 == 1: + for j in s..s+len2-1: + if a[s] == b[j]: return len2 - 1 + return len2 + + inc(len1) + inc(len2) + var half = len1 shr 1 + # initalize first row: + #var row = cast[ptr array[0..high(int) div 8, int]](alloc(len2*sizeof(int))) + var row: seq[int] + newSeq(row, len2) + var e = s + len2 - 1 # end marker + for i in 1..len2 - half - 1: row[i] = i + row[0] = len1 - half - 1 + for i in 1 .. len1 - 1: + var char1 = a[i + s - 1] + var char2p: int + var D, x: int + var p: int + if i >= len1 - half: + # skip the upper triangle: + var offset = i - len1 + half + char2p = offset + p = offset + var c3 = row[p] + ord(char1 != b[s + char2p]) + inc(p) + inc(char2p) + x = row[p] + 1 + D = x + if x > c3: x = c3 + row[p] = x + inc(p) + else: + p = 1 + char2p = 0 + D = i + x = i + if i <= half + 1: + # skip the lower triangle: + e = len2 + i - half - 2 + # main: + while p <= e: + dec(D) + var c3 = D + ord(char1 != b[char2p + s]) + inc(char2p) + inc(x) + if x > c3: x = c3 + D = row[p] + 1 + if x > D: x = D + row[p] = x + inc(p) + # lower triangle sentinel: + if i <= half: + dec(D) + var c3 = D + ord(char1 != b[char2p + s]) + inc(x) + if x > c3: x = c3 + row[p] = x + result = row[e] +{.pop.} + +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, + extern: "nsuIsNilOrEmpty", + deprecated: "use 'x.len == 0' instead".} = + ## **Deprecated**: use 'x.len == 0' + ## + ## Checks if `s` is nil or empty. + result = len(s) == 0 + +proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = + ## Checks if `s` is nil or consists entirely of whitespace characters. + result = true + for c in s: + if not c.isSpaceAscii(): + return false + +template isImpl(call) = + if s.len == 0: return false + result = true + for c in s: + if not call(c): return false + +proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiStr", + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether or not `s` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## Returns true if all characters in `s` are + ## alphabetic and there is at least one character + ## in `s`. + ## Use `Unicode module`_ for UTF-8 support. + runnableExamples: + doAssert isAlphaAscii("fooBar") == true + doAssert isAlphaAscii("fooBar1") == false + doAssert isAlphaAscii("foo Bar") == false + isImpl isAlphaAscii + +proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaNumericStr", + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether or not `s` is alphanumeric. + ## + ## This checks a-z, A-Z, 0-9 ASCII characters only. + ## Returns true if all characters in `s` are + ## alpanumeric and there is at least one character + ## in `s`. + ## Use `Unicode module`_ for UTF-8 support. + runnableExamples: + doAssert isAlphaNumeric("fooBar") == true + doAssert isAlphaNumeric("fooBar1") == true + doAssert isAlphaNumeric("foo Bar") == false + isImpl isAlphaNumeric + +proc isDigit*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsDigitStr", + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether or not `s` is a numeric value. + ## + ## This checks 0-9 ASCII characters only. + ## Returns true if all characters in `s` are + ## numeric and there is at least one character + ## in `s`. + runnableExamples: + doAssert isDigit("1908") == true + doAssert isDigit("fooBar1") == false + isImpl isDigit + +proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiStr", + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether or not `s` is completely whitespace. + ## + ## Returns true if all characters in `s` are whitespace + ## characters and there is at least one character in `s`. + runnableExamples: + doAssert isSpaceAscii(" ") == true + doAssert isSpaceAscii("") == false + isImpl isSpaceAscii + +template isCaseImpl(s, charProc, skipNonAlpha) = + var hasAtleastOneAlphaChar = false + if s.len == 0: return false + for c in s: + if skipNonAlpha: + var charIsAlpha = c.isAlphaAscii() + if not hasAtleastOneAlphaChar: + hasAtleastOneAlphaChar = charIsAlpha + if charIsAlpha and (not charProc(c)): + return false + else: + if not charProc(c): + return false + return if skipNonAlpha: hasAtleastOneAlphaChar else: true + +proc isLowerAscii*(s: string, skipNonAlpha: bool): bool {. + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether ``s`` is lower case. + ## + ## This checks ASCII characters only. + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## characters in ``s`` are lower case. Returns false if none of the + ## characters in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all characters + ## in ``s`` are alphabetical and lower case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + ## Use `Unicode module`_ for UTF-8 support. + runnableExamples: + doAssert isLowerAscii("1foobar", false) == false + doAssert isLowerAscii("1foobar", true) == true + doAssert isLowerAscii("1fooBar", true) == false + isCaseImpl(s, isLowerAscii, skipNonAlpha) + +proc isUpperAscii*(s: string, skipNonAlpha: bool): bool {. + deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = + ## **Deprecated**: Deprecated since version 0.20 since its semantics are unclear + ## + ## Checks whether ``s`` is upper case. + ## + ## This checks ASCII characters only. + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## characters in ``s`` are upper case. Returns false if none of the + ## characters in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all characters + ## in ``s`` are alphabetical and upper case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + ## Use `Unicode module`_ for UTF-8 support. + runnableExamples: + doAssert isUpperAscii("1FOO", false) == false + doAssert isUpperAscii("1FOO", true) == true + doAssert isUpperAscii("1Foo", true) == false + isCaseImpl(s, isUpperAscii, skipNonAlpha) + +proc wordWrap*(s: string, maxLineWidth = 80, + splitLongWords = true, + seps: set[char] = Whitespace, + newLine = "\n"): string {. + noSideEffect, rtl, extern: "nsuWordWrap", + deprecated: "use wrapWords in std/wordwrap instead".} = + ## **Deprecated**: use wrapWords in std/wordwrap instead + ## + ## Word wraps `s`. + result = newStringOfCap(s.len + s.len shr 6) + var spaceLeft = maxLineWidth + var lastSep = "" + for word, isSep in tokenize(s, seps): + if isSep: + lastSep = word + spaceLeft = spaceLeft - len(word) + continue + if len(word) > spaceLeft: + if splitLongWords and len(word) > maxLineWidth: + result.add(substr(word, 0, spaceLeft-1)) + var w = spaceLeft + var wordLeft = len(word) - spaceLeft + while wordLeft > 0: + result.add(newLine) + var L = min(maxLineWidth, wordLeft) + spaceLeft = maxLineWidth - L + result.add(substr(word, w, w+L-1)) + inc(w, L) + dec(wordLeft, L) + else: + spaceLeft = maxLineWidth - len(word) + result.add(newLine) + result.add(word) + else: + spaceLeft = spaceLeft - len(word) + result.add(lastSep & word) + lastSep.setLen(0) + + + when isMainModule: proc nonStaticTests = doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" diff --git a/tests/errmsgs/tunknown_named_parameter.nim b/tests/errmsgs/tunknown_named_parameter.nim index b6b855136c..8a3bcaf03b 100644 --- a/tests/errmsgs/tunknown_named_parameter.nim +++ b/tests/errmsgs/tunknown_named_parameter.nim @@ -2,10 +2,6 @@ discard """ cmd: "nim check $file" errormsg: "type mismatch: got " nimout: ''' -proc rsplit(s: string; sep: string; maxsplit: int = -1): seq[string] - first type mismatch at position: 2 - required type: string - but expression '{':'}' is of type: set[char] proc rsplit(s: string; sep: char; maxsplit: int = -1): seq[string] first type mismatch at position: 2 required type: char @@ -13,6 +9,10 @@ proc rsplit(s: string; sep: char; maxsplit: int = -1): seq[string] proc rsplit(s: string; seps: set[char] = Whitespace; maxsplit: int = -1): seq[string] first type mismatch at position: 3 unknown named parameter: maxsplits +proc rsplit(s: string; sep: string; maxsplit: int = -1): seq[string] + first type mismatch at position: 2 + required type: string + but expression '{':'}' is of type: set[char] expression: rsplit("abc:def", {':'}, maxsplits = 1) ''' From b40a637fac1334339c260951d0dd3fa3682de476 Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 11 Jan 2019 13:22:10 +0100 Subject: [PATCH 137/350] better docs: tables --- lib/pure/collections/tableimpl.nim | 2 +- lib/pure/collections/tables.nim | 2870 +++++++++++++++++++++------- 2 files changed, 2140 insertions(+), 732 deletions(-) diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 9a5bffcef4..2cdc62996f 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## An ``include`` file for the different table implementations. +# An ``include`` file for the different table implementations. # hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These # two procs retain clarity of that encoding without the space cost of an enum. diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 1fa2ca0a60..659b4473b3 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -7,18 +7,25 @@ # distribution, for details about the copyright. # +## :Author: Nim contributors +## ## The ``tables`` module implements variants of an efficient `hash table`:idx: ## (also often named `dictionary`:idx: in other programming languages) that is -## a mapping from keys to values. ``Table`` is the usual hash table, -## ``OrderedTable`` is like ``Table`` but remembers insertion order -## and ``CountTable`` is a mapping from a key to its number of occurrences. +## a mapping from keys to values. +## +## There are several different types of hash tables available: +## * `Table<#Table>`_ is the usual hash table, +## * `OrderedTable<#OrderedTable>`_ is like ``Table`` but remembers insertion order, +## * `CountTable<#CountTable>`_ is a mapping from a key to its number of occurrences ## ## For consistency with every other data type in Nim these have **value** ## semantics, this means that ``=`` performs a copy of the hash table. -## For **reference** semantics use the ``Ref`` variant: ``TableRef``, -## ``OrderedTableRef``, ``CountTableRef``. ## -## To give an example, when ``a`` is a Table, then ``var b = a`` gives ``b`` +## For `ref semantics`_ +## use their ``Ref`` variants: `TableRef<#TableRef>`_, +## `OrderedTableRef<#OrderedTableRef>`_, and `CountTableRef<#CountTableRef>`_. +## +## To give an example, when ``a`` is a ``Table``, then ``var b = a`` gives ``b`` ## as a new independent table. ``b`` is initialised with the contents of ``a``. ## Changing ``b`` does not affect ``a`` and vice versa: ## @@ -35,8 +42,8 @@ ## echo a, b # output: {1: one, 2: two}{1: one, 2: two, 3: three} ## echo a == b # output: false ## -## On the other hand, when ``a`` is a TableRef instead, then changes to ``b`` -## also affect ``a``. Both ``a`` and ``b`` reference the same data structure: +## On the other hand, when ``a`` is a ``TableRef`` instead, then changes to ``b`` +## also affect ``a``. Both ``a`` and ``b`` **ref** the same data structure: ## ## .. code-block:: ## import tables @@ -51,27 +58,111 @@ ## echo a, b # output: {1: one, 2: two, 3: three}{1: one, 2: two, 3: three} ## echo a == b # output: true ## +## ---- ## -## Here is an example of ``CountTable`` usage: +## Basic usage +## =========== +## +## Table +## ----- +## +## .. code-block:: +## import tables +## from sequtils import zip +## +## let +## names = ["John", "Paul", "George", "Ringo"] +## years = [1940, 1942, 1943, 1940] +## +## var beatles = initTable[string, int]() +## +## for pairs in zip(names, years): +## let (name, birthYear) = pairs +## beatles[name] = birthYear +## +## echo beatles +## # {"George": 1943, "Ringo": 1940, "Paul": 1942, "John": 1940} +## +## +## var beatlesByYear = initTable[int, seq[string]]() +## +## for pairs in zip(years, names): +## let (birthYear, name) = pairs +## if not beatlesByYear.hasKey(birthYear): +## # if a key doesn't exists, we create one with an empty sequence +## # before we can add elements to it +## beatlesByYear[birthYear] = @[] +## beatlesByYear[birthYear].add(name) +## +## echo beatlesByYear +## # {1940: @["John", "Ringo"], 1942: @["Paul"], 1943: @["George"]} +## +## +## +## OrderedTable +## ------------ +## +## `OrderedTable<#OrderedTable>`_ is used when it is important to preserve +## the insertion order of keys. +## +## .. code-block:: +## import tables +## +## let +## a = [('z', 1), ('y', 2), ('x', 3)] +## t = a.toTable # regular table +## ot = a.toOrderedTable # ordered tables +## +## echo t # {'x': 3, 'y': 2, 'z': 1} +## echo ot # {'z': 1, 'y': 2, 'x': 3} +## +## +## +## CountTable +## ---------- +## +## `CountTable<#CountTable>`_ is useful for counting number of items of some +## container (e.g. string, sequence or array), as it is a mapping where the +## items are the keys, and their number of occurrences are the values. +## For that purpose `toCountTable proc<#toCountTable,openArray[A]>`_ +## comes handy: +## +## .. code-block:: +## import tables ## -## .. code-block:: nim ## let myString = "abracadabra" -## var myTable = initCountTable[char]() +## let letterFrequencies = toCountTable(myString) +## echo letterFrequencies +## # 'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r': 2} ## +## The same could have been achieved by manually iterating over a container +## and increasing each key's value with `inc proc<#inc,CountTable[A],A,int>`_: +## +## .. code-block:: +## import tables +## +## let myString = "abracadabra" +## var letterFrequencies = initCountTable[char]() ## for c in myString: -## myTable.inc(c) +## letterFrequencies.inc(c) +## echo letterFrequencies +## # output: {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r': 2} ## -## echo myTable # output: {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r': 2} +## ---- ## ## +## +## Hashing +## ------- +## ## If you are using simple standard types like ``int`` or ``string`` for the ## keys of the table you won't have any problems, but as soon as you try to use ## a more complex object as a key you will be greeted by a strange compiler -## error:: +## error: ## ## Error: type mismatch: got (Person) ## but expected one of: -## hashes.hash(x: openarray[A]): Hash +## hashes.hash(x: openArray[A]): Hash ## hashes.hash(x: int): Hash ## hashes.hash(x: float): Hash ## … @@ -89,6 +180,8 @@ ## example implementing only ``hash`` suffices: ## ## .. code-block:: +## import tables, hashes +## ## type ## Person = object ## firstName, lastName: string @@ -111,45 +204,50 @@ ## p2.firstName = "소진" ## p2.lastName = "박" ## salaries[p2] = 45_000 +## +## ---- +## +## See also +## ======== +## +## * `json module`_ for table-like structure which allows +## heterogeneous members +## * `sharedtables module`_ for shared hash table support +## * `strtabs module`_ for efficient hash tables +## mapping from strings to strings +## * `hashes module`_ for helper functions for hashing -import - hashes, math + +import hashes, math include "system/inclrtl" type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] - Table*[A, B] = object ## generic hash table + Table*[A, B] = object + ## Generic hash table, consisting of a key-value pair. + ## + ## `data` and `counter` are internal implementation details which + ## can't be accessed. + ## + ## For creating an empty Table, use `initTable proc<#initTable,int>`_. data: KeyValuePairSeq[A, B] counter: int - TableRef*[A,B] = ref Table[A, B] + TableRef*[A,B] = ref Table[A, B] ## Ref version of `Table<#Table>`_. + ## + ## For creating a new empty TableRef, use `newTable proc + ## <#newTable,int>`_. + + +# ------------------------------ helpers --------------------------------- template maxHash(t): untyped = high(t.data) template dataLen(t): untyped = len(t.data) include tableimpl -proc clear*[A, B](t: var Table[A, B]) = - ## resets the table so that it is empty. - clearImpl() - -proc clear*[A, B](t: TableRef[A, B]) = - ## resets the ref table so that it is empty. - clearImpl() - -proc rightSize*(count: Natural): int {.inline.} = - ## return the value of ``initialSize`` to support ``count`` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - ## - ## Internally, we want mustRehash(rightSize(x), x) == false. - result = nextPowerOfTwo(count * 3 div 2 + 4) - -proc len*[A, B](t: Table[A, B]): int = - ## returns the number of keys in ``t``. - result = t.counter +proc rightSize*(count: Natural): int {.inline.} template get(t, key): untyped = ## retrieves the value at ``t[key]``. The value can be modified. @@ -176,36 +274,340 @@ template getOrDefaultImpl(t, key, default: untyped): untyped = var index = rawGet(t, key, hc) result = if index >= 0: t.data[index].val else: default -proc `[]`*[A, B](t: Table[A, B], key: A): B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If ``key`` is not in ``t``, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether +template dollarImpl(): untyped {.dirty.} = + if t.len == 0: + result = "{:}" + else: + result = "{" + for key, val in pairs(t): + if result.len > 1: result.add(", ") + result.addQuoted(key) + result.add(": ") + result.addQuoted(val) + result.add("}") + +proc enlarge[A, B](t: var Table[A, B]) = + var n: KeyValuePairSeq[A, B] + newSeq(n, len(t.data) * growthFactor) + swap(t.data, n) + for i in countup(0, high(n)): + let eh = n[i].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + rawInsert(t, t.data, n[i].key, n[i].val, eh, j) + +template equalsImpl(s, t: typed): typed = + if s.counter == t.counter: + # different insertion orders mean different 'data' seqs, so we have + # to use the slow route here: + for key, val in s: + if not t.hasKey(key): return false + if t.getOrDefault(key) != val: return false + return true + + + +# ------------------------------------------------------------------- +# ------------------------------ Table ------------------------------ +# ------------------------------------------------------------------- + +proc initTable*[A, B](initialSize=64): Table[A, B] = + ## Creates a new hash table that is empty. + ## + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `toTable proc<#toTable,openArray[]>`_ + ## * `newTable proc<#newTable,int>`_ for creating a `TableRef` + runnableExamples: + let + a = initTable[int, string]() + b = initTable[char, seq[int]]() + assert isPowerOfTwo(initialSize) + result.counter = 0 + newSeq(result.data, initialSize) + +proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = + ## Creates a new hash table that contains the given ``pairs``. + ## + ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## + ## See also: + ## * `initTable proc<#initTable,int>`_ + ## * `newTable proc<#newTable,openArray[]>`_ for a `TableRef` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = toTable(a) + assert b == {'a': 5, 'b': 9}.toTable + result = initTable[A, B](rightSize(pairs.len)) + for key, val in items(pairs): result[key] = val + +proc `[]`*[A, B](t: Table[A, B], key: A): B = + ## Retrieves the value at ``t[key]``. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## One can check with `hasKey proc<#hasKey,Table[A,B],A>`_ whether ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in + ## the table + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] get(t, key) -proc `[]`*[A, B](t: var Table[A, B], key: A): var B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. +proc `[]`*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at ``t[key]``. The value can be modified. + ## ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in + ## the table get(t, key) -proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - get(t, key) +proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `del proc<#del,Table[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = initTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.toTable + putImpl(enlarge) + +proc hasKey*[A, B](t: Table[A, B], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,Table[A,B],A>`_ for use with the `in` operator + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + var hc: Hash + result = rawGet(t, key, hc) >= 0 + +proc contains*[A, B](t: Table[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,Table[A,B],A>`_ for use with + ## the ``in`` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = + ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## + ## See also: + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.toTable + hasKeyOrPutImpl(enlarge) proc getOrDefault*[A, B](t: Table[A, B], key: A): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the ## default initialization value for type ``B`` is returned (e.g. 0 for any ## integer type). + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. ## Otherwise, ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 getOrDefaultImpl(t, key, default) +proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = + ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.toTable + mgetOrPutImpl(enlarge) + +proc len*[A, B](t: Table[A, B]): int = + ## Returns the number of keys in ``t``. + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert len(a) == 2 + result = t.counter + +proc add*[A, B](t: var Table[A, B], key: A, val: B) = + ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. + addImpl(enlarge) + +proc del*[A, B](t: var Table[A, B], key: A) = + ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## + ## See also: + ## * `take proc<#take,Table[A,B],A,B>`_ + ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.toTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.toTable + delImpl() + +proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool = + ## Deletes the ``key`` from the table. + ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the + ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## unchanged. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'a': 5, 'b': 9, 'c': 13}.toTable + i: int + doAssert a.take('b', i) == true + doAssert a == {'a': 5, 'c': 13}.toTable + doAssert i == 9 + i = 0 + doAssert a.take('z', i) == false + doAssert a == {'a': 5, 'c': 13}.toTable + doAssert i == 0 + + var hc: Hash + var index = rawGet(t, key, hc) + result = index >= 0 + if result: + shallowCopy(val, t.data[index].val) + delImplIdx(t, index) + +proc clear*[A, B](t: var Table[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `take proc<#take,Table[A,B],A,B>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + clearImpl() + +proc `$`*[A, B](t: Table[A, B]): string = + ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## on a table. + dollarImpl() + +proc `==`*[A, B](s, t: Table[A, B]): bool = + ## The ``==`` operator for hash tables. Returns ``true`` if the content of both + ## tables contains the same key-value pairs. Insert order does not matter. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.toTable + b = {'b': 9, 'c': 13, 'a': 5}.toTable + doAssert a == b + equalsImpl(s, t) + +proc rightSize*(count: Natural): int {.inline.} = + ## Return the value of ``initialSize`` to support ``count`` items. + ## + ## If more items are expected to be added, simply add that + ## expected extra amount to the parameter before calling this. + ## + ## Internally, we want mustRehash(rightSize(x), x) == false. + result = nextPowerOfTwo(count * 3 div 2 + 4) + +proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = + ## Index the collection with the proc provided. + # TODO: As soon as supported, change collection: A to collection: A[B] + result = initTable[C, B]() + for item in collection: + result[index(item)] = item + + + template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = - ## retrieves the value at ``t[key]``. + ## Retrieves the value at ``t[key]``. + ## ## ``value`` can be modified in the scope of the ``withValue`` call. ## ## .. code-block:: nim @@ -225,7 +627,8 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = template withValue*[A, B](t: var Table[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at ``t[key]``. + ## Retrieves the value at ``t[key]``. + ## ## ``value`` can be modified in the scope of the ``withValue`` call. ## ## .. code-block:: nim @@ -248,258 +651,171 @@ template withValue*[A, B](t: var Table[A, B], key: A, else: body2 + +iterator pairs*[A, B](t: Table[A, B]): (A, B) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,Table[A,B]>`_ + ## * `keys iterator<#keys.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.toTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: e + ## # value: [2, 4, 6, 8] + ## # key: o + ## # value: [1, 5, 7, 9] + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.toTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: Table[A, B]): A = + ## Iterates over any key in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for k in a.keys: + a[k].add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].key + +iterator values*[A, B](t: Table[A, B]): B = + ## Iterates over any value in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `keys iterator<#keys.i,Table[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,Table[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for v in a.values: + doAssert v.len == 4 + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + +iterator mvalues*[A, B](t: var Table[A, B]): var B = + ## Iterates over any value in the table ``t`` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for v in a.mvalues: + v.add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + iterator allValues*[A, B](t: Table[A, B]; key: A): B = - ## iterates over any value in the table ``t`` that belongs to the given ``key``. + ## Iterates over any value in the table ``t`` that belongs to the given ``key``. + ## + ## Used if you have a table with duplicate keys (as a result of using + ## `add proc<#add,Table[A,B],A,B>`_). + ## + ## **Examples:** + ## + ## .. code-block:: + ## var a = {'a': 3, 'b': 5}.toTable + ## for i in 1..3: + ## a.add('z', 10*i) + ## echo a # {'a': 3, 'b': 5, 'z': 10, 'z': 20, 'z': 30} + ## + ## for v in a.allValues('z'): + ## echo v + ## # 10 + ## # 20 + ## # 30 var h: Hash = genHash(key) and high(t.data) while isFilled(t.data[h].hcode): if t.data[h].key == key: yield t.data[h].val h = nextTry(h, high(t.data)) -proc hasKey*[A, B](t: Table[A, B], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - var hc: Hash - result = rawGet(t, key, hc) >= 0 -proc contains*[A, B](t: Table[A, B], key: A): bool = - ## alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A, B](t, key) -iterator pairs*[A, B](t: Table[A, B]): (A, B) = - ## iterates over any ``(key, value)`` pair in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) +# ------------------------------------------------------------------- +# ---------------------------- TableRef ----------------------------- +# ------------------------------------------------------------------- -iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = - ## iterates over any ``(key, value)`` pair in the table ``t``. The values - ## can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) - -iterator keys*[A, B](t: Table[A, B]): A = - ## iterates over any key in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].key - -iterator values*[A, B](t: Table[A, B]): B = - ## iterates over any value in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -iterator mvalues*[A, B](t: var Table[A, B]): var B = - ## iterates over any value in the table ``t``. The values can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -proc del*[A, B](t: var Table[A, B], key: A) = - ## deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. - delImpl() - -proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool = - ## deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is - ## unchanged. - var hc: Hash - var index = rawGet(t, key, hc) - result = index >= 0 - if result: - shallowCopy(val, t.data[index].val) - delImplIdx(t, index) - -proc enlarge[A, B](t: var Table[A, B]) = - var n: KeyValuePairSeq[A, B] - newSeq(n, len(t.data) * growthFactor) - swap(t.data, n) - for i in countup(0, high(n)): - let eh = n[i].hcode - if isFilled(eh): - var j: Hash = eh and maxHash(t) - while isFilled(t.data[j].hcode): - j = nextTry(j, maxHash(t)) - rawInsert(t, t.data, n[i].key, n[i].val, eh, j) - -proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way - ## returning a value which can be modified. - mgetOrPutImpl(enlarge) - -proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = - ## returns true iff ``key`` is in the table, otherwise inserts ``value``. - hasKeyOrPutImpl(enlarge) - -proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = - ## puts a ``(key, value)`` pair into ``t``. - putImpl(enlarge) - -proc add*[A, B](t: var Table[A, B], key: A, val: B) = - ## puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. - ## This can introduce duplicate keys into the table! - addImpl(enlarge) - -proc len*[A, B](t: TableRef[A, B]): int = - ## returns the number of keys in ``t``. - result = t.counter - -proc initTable*[A, B](initialSize=64): Table[A, B] = - ## creates a new hash table that is empty. - ## - ## ``initialSize`` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - newSeq(result.data, initialSize) - -proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = - ## creates a new hash table that contains the given ``pairs``. - result = initTable[A, B](rightSize(pairs.len)) - for key, val in items(pairs): result[key] = val - -template dollarImpl(): untyped {.dirty.} = - if t.len == 0: - result = "{:}" - else: - result = "{" - for key, val in pairs(t): - if result.len > 1: result.add(", ") - result.addQuoted(key) - result.add(": ") - result.addQuoted(val) - result.add("}") - -proc `$`*[A, B](t: Table[A, B]): string = - ## the ``$`` operator for hash tables. - dollarImpl() - -proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - result = t[].hasKey(key) - -template equalsImpl(s, t: typed): typed = - if s.counter == t.counter: - # different insertion orders mean different 'data' seqs, so we have - # to use the slow route here: - for key, val in s: - if not t.hasKey(key): return false - if t.getOrDefault(key) != val: return false - return true - -proc `==`*[A, B](s, t: Table[A, B]): bool = - ## The ``==`` operator for hash tables. Returns ``true`` iff the content of both - ## tables contains the same key-value pairs. Insert order does not matter. - equalsImpl(s, t) - -proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = - ## Index the collection with the proc provided. - # TODO: As soon as supported, change collection: A to collection: A[B] - result = initTable[C, B]() - for item in collection: - result[index(item)] = item - -iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = - ## iterates over any ``(key, value)`` pair in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) - -iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = - ## iterates over any ``(key, value)`` pair in the table ``t``. The values - ## can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) - -iterator keys*[A, B](t: TableRef[A, B]): A = - ## iterates over any key in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].key - -iterator values*[A, B](t: TableRef[A, B]): B = - ## iterates over any value in the table ``t``. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -iterator mvalues*[A, B](t: TableRef[A, B]): var B = - ## iterates over any value in the table ``t``. The values can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val - -proc `[]`*[A, B](t: TableRef[A, B], key: A): var B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If ``key`` is not in ``t``, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - result = t[][key] - -proc mget*[A, B](t: TableRef[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - t[][key] - -proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any - ## integer type). - getOrDefault(t[], key) - -proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. - ## Otherwise, ``default`` is returned. - getOrDefault(t[], key, default) - -proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way - ## returning a value which can be modified. - t[].mgetOrPut(key, val) - -proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = - ## returns true iff ``key`` is in the table, otherwise inserts ``value``. - t[].hasKeyOrPut(key, val) - -proc contains*[A, B](t: TableRef[A, B], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A, B](t, key) - -proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = - ## puts a ``(key, value)`` pair into ``t``. - t[][key] = val - -proc add*[A, B](t: TableRef[A, B], key: A, val: B) = - ## puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. - ## This can introduce duplicate keys into the table! - t[].add(key, val) - -proc del*[A, B](t: TableRef[A, B], key: A) = - ## deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. - t[].del(key) - -proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool = - ## deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is - ## unchanged. - result = t[].take(key, val) proc newTable*[A, B](initialSize=64): TableRef[A, B] = + ## Creates a new ref hash table that is empty. + ## + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `newTable proc<#newTable,openArray[]>`_ for creating a `TableRef` + ## from a collection of `(key, value)` pairs + ## * `initTable proc<#initTable,int>`_ for creating a `Table` + runnableExamples: + let + a = newTable[int, string]() + b = newTable[char, seq[int]]() new(result) result[] = initTable[A, B](initialSize) proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = - ## creates a new hash table that contains the given ``pairs``. + ## Creates a new ref hash table that contains the given ``pairs``. + ## + ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## + ## See also: + ## * `newTable proc<#newTable,int>`_ + ## * `toTable proc<#toTable,openArray[]>`_ for a `Table` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = newTable(a) + assert b == {'a': 5, 'b': 9}.newTable new(result) result[] = toTable[A, B](pairs) -proc `$`*[A, B](t: TableRef[A, B]): string = - ## The ``$`` operator for hash tables. - dollarImpl() - -proc `==`*[A, B](s, t: TableRef[A, B]): bool = - ## The ``==`` operator for hash tables. Returns ``true`` iff either both tables - ## are ``nil`` or none is ``nil`` and the content of both tables contains the - ## same key-value pairs. Insert order does not matter. - if isNil(s): result = isNil(t) - elif isNil(t): result = false - else: equalsImpl(s[], t[]) - proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] @@ -507,65 +823,354 @@ proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] for item in collection: result[index(item)] = item -# ------------------------------ ordered table ------------------------------ +proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at ``t[key]``. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## One can check with `hasKey proc<#hasKey,TableRef[A,B],A>`_ whether + ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,TableRef[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ for checking if a key is in + ## the table + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + result = t[][key] + +proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `del proc<#del,TableRef[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = newTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.newTable + t[][key] = val + +proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,TableRef[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + result = t[].hasKey(key) + +proc contains*[A, B](t: TableRef[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,TableRef[A,B],A>`_ for use with + ## the ``in`` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = + ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## + ## See also: + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.newTable + t[].hasKeyOrPut(key, val) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the + ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + getOrDefault(t[], key) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise, ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + getOrDefault(t[], key, default) + +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = + ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.newTable + t[].mgetOrPut(key, val) + +proc len*[A, B](t: TableRef[A, B]): int = + ## Returns the number of keys in ``t``. + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert len(a) == 2 + result = t.counter + +proc add*[A, B](t: TableRef[A, B], key: A, val: B) = + ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,TableRef[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. + t[].add(key, val) + +proc del*[A, B](t: TableRef[A, B], key: A) = + ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## + ## See also: + ## * `take proc<#take,TableRef[A,B],A,B>`_ + ## * `clear proc<#clear,TableRef[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.newTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.newTable + t[].del(key) + +proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool = + ## Deletes the ``key`` from the table. + ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the + ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## unchanged. + ## + ## See also: + ## * `del proc<#del,TableRef[A,B],A>`_ + ## * `clear proc<#clear,TableRef[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'a': 5, 'b': 9, 'c': 13}.newTable + i: int + doAssert a.take('b', i) == true + doAssert a == {'a': 5, 'c': 13}.newTable + doAssert i == 9 + i = 0 + doAssert a.take('z', i) == false + doAssert a == {'a': 5, 'c': 13}.newTable + doAssert i == 0 + result = t[].take(key, val) + +proc clear*[A, B](t: TableRef[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `take proc<#take,Table[A,B],A,B>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + clearImpl() + +proc `$`*[A, B](t: TableRef[A, B]): string = + ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## on a table. + dollarImpl() + +proc `==`*[A, B](s, t: TableRef[A, B]): bool = + ## The ``==`` operator for hash tables. Returns ``true`` if either both tables + ## are ``nil``, or neither is ``nil`` and the content of both tables contains the + ## same key-value pairs. Insert order does not matter. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.newTable + b = {'b': 9, 'c': 13, 'a': 5}.newTable + doAssert a == b + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: equalsImpl(s[], t[]) + + + +iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ + ## * `keys iterator<#keys.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.newTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: e + ## # value: [2, 4, 6, 8] + ## # key: o + ## # value: [1, 5, 7, 9] + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. The values + ## can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.newTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: TableRef[A, B]): A = + ## Iterates over any key in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for k in a.keys: + a[k].add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].key + +iterator values*[A, B](t: TableRef[A, B]): B = + ## Iterates over any value in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `keys iterator<#keys.i,TableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for v in a.values: + doAssert v.len == 4 + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + +iterator mvalues*[A, B](t: TableRef[A, B]): var B = + ## Iterates over any value in the table ``t``. The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for v in a.mvalues: + v.add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable + for h in 0..high(t.data): + if isFilled(t.data[h].hcode): yield t.data[h].val + + + + + + + + +# --------------------------------------------------------------------------- +# ------------------------------ OrderedTable ------------------------------- +# --------------------------------------------------------------------------- type OrderedKeyValuePair[A, B] = tuple[ hcode: Hash, next: int, key: A, val: B] OrderedKeyValuePairSeq[A, B] = seq[OrderedKeyValuePair[A, B]] - OrderedTable* [A, B] = object ## table that remembers insertion order + OrderedTable* [A, B] = object + ## Hash table that remembers insertion order. + ## + ## For creating an empty OrderedTable, use `initOrderedTable proc + ## <#initOrderedTable,int>`_. data: OrderedKeyValuePairSeq[A, B] counter, first, last: int - OrderedTableRef*[A, B] = ref OrderedTable[A, B] + OrderedTableRef*[A, B] = ref OrderedTable[A, B] ## Ref version of + ## `OrderedTable<#OrderedTable>`_. + ## + ## For creating a new empty OrderedTableRef, use `newOrderedTable proc + ## <#newOrderedTable,int>`_. -proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = - ## returns the number of keys in ``t``. - result = t.counter -proc clear*[A, B](t: var OrderedTable[A, B]) = - ## resets the table so that it is empty. - clearImpl() - t.first = -1 - t.last = -1 - -proc clear*[A, B](t: var OrderedTableRef[A, B]) = - ## resets the table so that is is empty. - clear(t[]) - -template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} = - var h = t.first - while h >= 0: - var nxt = t.data[h].next - if isFilled(t.data[h].hcode): yieldStmt - h = nxt - -iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = - ## iterates over any ``(key, value)`` pair in the table ``t`` in insertion - ## order. - forAllOrderedPairs: - yield (t.data[h].key, t.data[h].val) - -iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = - ## iterates over any ``(key, value)`` pair in the table ``t`` in insertion - ## order. The values can be modified. - forAllOrderedPairs: - yield (t.data[h].key, t.data[h].val) - -iterator keys*[A, B](t: OrderedTable[A, B]): A = - ## iterates over any key in the table ``t`` in insertion order. - forAllOrderedPairs: - yield t.data[h].key - -iterator values*[A, B](t: OrderedTable[A, B]): B = - ## iterates over any value in the table ``t`` in insertion order. - forAllOrderedPairs: - yield t.data[h].val - -iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = - ## iterates over any value in the table ``t`` in insertion order. The values - ## can be modified. - forAllOrderedPairs: - yield t.data[h].val +# ------------------------------ helpers --------------------------------- proc rawGetKnownHC[A, B](t: OrderedTable[A, B], key: A, hc: Hash): int = rawGetKnownHCImpl() @@ -576,43 +1181,6 @@ proc rawGetDeep[A, B](t: OrderedTable[A, B], key: A, hc: var Hash): int {.inline proc rawGet[A, B](t: OrderedTable[A, B], key: A, hc: var Hash): int = rawGetImpl() -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If ``key`` is not in ``t``, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - get(t, key) - -proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B{.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - get(t, key) - -proc mget*[A, B](t: var OrderedTable[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - get(t, key) - -proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any - ## integer type). - getOrDefaultImpl(t, key) - -proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, - ## ``default`` is returned. - getOrDefaultImpl(t, key, default) - -proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - var hc: Hash - result = rawGet(t, key, hc) >= 0 - -proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A, B](t, key) - proc rawInsert[A, B](t: var OrderedTable[A, B], data: var OrderedKeyValuePairSeq[A, B], key: A, val: B, hc: Hash, h: Hash) = @@ -639,30 +1207,32 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) = rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) h = nxt -proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## puts a ``(key, value)`` pair into ``t``. - putImpl(enlarge) +template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} = + var h = t.first + while h >= 0: + var nxt = t.data[h].next + if isFilled(t.data[h].hcode): yieldStmt + h = nxt -proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. - ## This can introduce duplicate keys into the table! - addImpl(enlarge) - -proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``value`` if not present, either way - ## returning a value which can be modified. - mgetOrPutImpl(enlarge) - -proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = - ## returns true iff ``key`` is in the table, otherwise inserts ``value``. - hasKeyOrPutImpl(enlarge) +# ---------------------------------------------------------------------- proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = - ## creates a new ordered hash table that is empty. + ## Creates a new ordered hash table that is empty. ## - ## ``initialSize`` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module or the ``rightSize`` proc from this module. + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ + ## * `newOrderedTable proc<#newOrderedTable,int>`_ for creating an + ## `OrderedTableRef` + runnableExamples: + let + a = initOrderedTable[int, string]() + b = initOrderedTable[char, seq[int]]() assert isPowerOfTwo(initialSize) result.counter = 0 result.first = -1 @@ -670,36 +1240,250 @@ proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = newSeq(result.data, initialSize) proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = - ## creates a new ordered hash table that contains the given ``pairs``. + ## Creates a new ordered hash table that contains the given ``pairs``. + ## + ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## + ## See also: + ## * `initOrderedTable proc<#initOrderedTable,int>`_ + ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for an + ## `OrderedTableRef` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = toOrderedTable(a) + assert b == {'a': 5, 'b': 9}.toOrderedTable result = initOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val -proc `$`*[A, B](t: OrderedTable[A, B]): string = - ## The ``$`` operator for ordered hash tables. - dollarImpl() +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = + ## Retrieves the value at ``t[key]``. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## One can check with `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ whether + ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a + ## key is in the table + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + get(t, key) -proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = - ## The ``==`` operator for ordered hash tables. Returns true iff both the - ## content and the order are equal. - if s.counter != t.counter: - return false - var ht = t.first - var hs = s.first - while ht >= 0 and hs >= 0: - var nxtt = t.data[ht].next - var nxts = s.data[hs].next - if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode): - if (s.data[hs].key != t.data[ht].key) or (s.data[hs].val != t.data[ht].val): - return false - ht = nxtt - hs = nxts - return true +proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B= + ## Retrieves the value at ``t[key]``. The value can be modified. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a + ## key is in the table + get(t, key) + +proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `del proc<#del,OrderedTable[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = initOrderedTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.toOrderedTable + putImpl(enlarge) + +proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,OrderedTable[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + var hc: Hash + result = rawGet(t, key, hc) >= 0 + +proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for use with + ## the ``in`` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = + ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## + ## See also: + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toOrderedTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.toOrderedTable + hasKeyOrPutImpl(enlarge) + +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the + ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + getOrDefaultImpl(t, key) + +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise, ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + getOrDefaultImpl(t, key, default) + +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = + ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.toOrderedTable + mgetOrPutImpl(enlarge) + +proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = + ## Returns the number of keys in ``t``. + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert len(a) == 2 + result = t.counter + +proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = + ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. + addImpl(enlarge) + +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## + ## O(n) complexity. + ## + ## See also: + ## * `clear proc<#clear,OrderedTable[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.toOrderedTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.toOrderedTable + var n: OrderedKeyValuePairSeq[A, B] + newSeq(n, len(t.data)) + var h = t.first + t.first = -1 + t.last = -1 + swap(t.data, n) + let hc = genHash(key) + while h >= 0: + var nxt = n[h].next + if isFilled(n[h].hcode): + if n[h].hcode == hc and n[h].key == key: + dec t.counter + else: + var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) + rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) + h = nxt + +proc clear*[A, B](t: var OrderedTable[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,OrderedTable[A,B],A>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + clearImpl() + t.first = -1 + t.last = -1 proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x,y: (A, B)): int) = - ## sorts ``t`` according to ``cmp``. This modifies the internal list + ## Sorts ``t`` according to the function ``cmp``. + ## + ## This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after ``sort`` (in - ## contrast to the ``sort`` for count tables). + ## contrast to the `sort proc<#sort,CountTable[A]>`_ for count tables). + runnableExamples: + var a = initOrderedTable[char, int]() + for i, c in "cab": + a[c] = 10*i + doAssert a == {'c': 0, 'a': 10, 'b': 20}.toOrderedTable + a.sort(system.cmp) + doAssert a == {'a': 10, 'b': 20, 'c': 0}.toOrderedTable + var list = t.first var p, q, e, tail, oldhead: int @@ -740,198 +1524,519 @@ proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x,y: (A, B)): int) = t.first = list t.last = tail -proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = - ## returns the number of keys in ``t``. - result = t.counter +proc `$`*[A, B](t: OrderedTable[A, B]): string = + ## The ``$`` operator for ordered hash tables. Used internally when calling + ## `echo` on a table. + dollarImpl() -iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = - ## iterates over any ``(key, value)`` pair in the table ``t`` in insertion +proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = + ## The ``==`` operator for ordered hash tables. Returns ``true`` if both the + ## content and the order are equal. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + b = {'b': 9, 'c': 13, 'a': 5}.toOrderedTable + doAssert a != b + + if s.counter != t.counter: + return false + var ht = t.first + var hs = s.first + while ht >= 0 and hs >= 0: + var nxtt = t.data[ht].next + var nxts = s.data[hs].next + if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode): + if (s.data[hs].key != t.data[ht].key) or (s.data[hs].val != t.data[ht].val): + return false + ht = nxtt + hs = nxts + return true + + + +iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion ## order. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTable[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.toOrderedTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: o + ## # value: [1, 5, 7, 9] + ## # key: e + ## # value: [2, 4, 6, 8] forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = - ## iterates over any ``(key, value)`` pair in the table ``t`` in insertion - ## order. The values can be modified. +iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## declared as `var`) in insertion order. The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'o': @[1, 5, 7, 9, 11], 'e': @[2, 4, 6, 8, 12]}.toOrderedTable forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = - ## iterates over any key in the table ``t`` in insertion order. +iterator keys*[A, B](t: OrderedTable[A, B]): A = + ## Iterates over any key in the table ``t`` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for k in a.keys: + a[k].add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.toOrderedTable forAllOrderedPairs: yield t.data[h].key -iterator values*[A, B](t: OrderedTableRef[A, B]): B = - ## iterates over any value in the table ``t`` in insertion order. +iterator values*[A, B](t: OrderedTable[A, B]): B = + ## Iterates over any value in the table ``t`` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTable[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTable[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for v in a.values: + doAssert v.len == 4 forAllOrderedPairs: yield t.data[h].val -iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B = - ## iterates over any value in the table ``t`` in insertion order. The values +iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = + ## Iterates over any value in the table ``t`` (must be + ## declared as `var`) in insertion order. The values ## can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for v in a.mvalues: + v.add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.toOrderedTable forAllOrderedPairs: yield t.data[h].val -proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = - ## retrieves the value at ``t[key]``. If ``key`` is not in ``t``, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - result = t[][key] -proc mget*[A, B](t: OrderedTableRef[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - result = t[][key] -proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any - ## integer type). - getOrDefault(t[], key) -proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, - ## ``default`` is returned. - getOrDefault(t[], key, default) -proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way - ## returning a value which can be modified. - result = t[].mgetOrPut(key, val) - -proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = - ## returns true iff ``key`` is in the table, otherwise inserts ``val``. - result = t[].hasKeyOrPut(key, val) - -proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - result = t[].hasKey(key) - -proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A, B](t, key) - -proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## puts a ``(key, value)`` pair into ``t``. - t[][key] = val - -proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. - ## This can introduce duplicate keys into the table! - t[].add(key, val) +# --------------------------------------------------------------------------- +# --------------------------- OrderedTableRef ------------------------------- +# --------------------------------------------------------------------------- proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = - ## creates a new ordered hash table that is empty. + ## Creates a new ordered ref hash table that is empty. ## - ## ``initialSize`` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module or the ``rightSize`` proc from this module. + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for creating + ## an `OrderedTableRef` from a collection of `(key, value)` pairs + ## * `initOrderedTable proc<#initOrderedTable,int>`_ for creating an + ## `OrderedTable` + runnableExamples: + let + a = newOrderedTable[int, string]() + b = newOrderedTable[char, seq[int]]() new(result) result[] = initOrderedTable[A, B](initialSize) proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = - ## creates a new ordered hash table that contains the given ``pairs``. + ## Creates a new ordered ref hash table that contains the given ``pairs``. + ## + ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## + ## See also: + ## * `newOrderedTable proc<#newOrderedTable,int>`_ + ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ for an + ## `OrderedTable` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = newOrderedTable(a) + assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result.add(key, val) + +proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at ``t[key]``. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## One can check with `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ whether + ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTableRef[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for checking if + ## a key is in the table + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + result = t[][key] + +proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `del proc<#del,OrderedTableRef[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = newOrderedTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.newOrderedTable + t[][key] = val + +proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,OrderedTableRef[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + result = t[].hasKey(key) + +proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for use with + ## the ``in`` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = + ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## + ## See also: + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newOrderedTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.newOrderedTable + result = t[].hasKeyOrPut(key, val) + +proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the + ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + getOrDefault(t[], key) + +proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise, ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + getOrDefault(t[], key, default) + +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = + ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.newOrderedTable + result = t[].mgetOrPut(key, val) + +proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = + ## Returns the number of keys in ``t``. + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert len(a) == 2 + result = t.counter + +proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = + ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,OrderedTableRef[A,B],A,B>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. + t[].add(key, val) + +proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = + ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## + ## See also: + ## * `clear proc<#clear,OrderedTableRef[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.newOrderedTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.newOrderedTable + t[].del(key) + +proc clear*[A, B](t: var OrderedTableRef[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,OrderedTable[A,B],A>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + clear(t[]) + +proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int) = + ## Sorts ``t`` according to the function ``cmp``. + ## + ## This modifies the internal list + ## that kept the insertion order, so insertion order is lost after this + ## call but key lookup and insertions remain possible after ``sort`` (in + ## contrast to the `sort proc<#sort,CountTableRef[A]>`_ for count tables). + runnableExamples: + var a = newOrderedTable[char, int]() + for i, c in "cab": + a[c] = 10*i + doAssert a == {'c': 0, 'a': 10, 'b': 20}.newOrderedTable + a.sort(system.cmp) + doAssert a == {'a': 10, 'b': 20, 'c': 0}.newOrderedTable + t[].sort(cmp) + proc `$`*[A, B](t: OrderedTableRef[A, B]): string = - ## The ``$`` operator for ordered hash tables. + ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## on a table. dollarImpl() proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = - ## The ``==`` operator for ordered hash tables. Returns true iff either both - ## tables are ``nil`` or none is ``nil`` and the content and the order of + ## The ``==`` operator for ordered hash tables. Returns true if either both + ## tables are ``nil``, or neither is ``nil`` and the content and the order of ## both are equal. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + b = {'b': 9, 'c': 13, 'a': 5}.newOrderedTable + doAssert a != b if isNil(s): result = isNil(t) elif isNil(t): result = false else: result = s[] == t[] -proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int) = - ## sorts ``t`` according to ``cmp``. This modifies the internal list - ## that kept the insertion order, so insertion order is lost after this - ## call but key lookup and insertions remain possible after ``sort`` (in - ## contrast to the ``sort`` for count tables). - t[].sort(cmp) -proc del*[A, B](t: var OrderedTable[A, B], key: A) = - ## deletes ``key`` from ordered hash table ``t``. O(n) complexity. Does nothing - ## if the key does not exist. - var n: OrderedKeyValuePairSeq[A, B] - newSeq(n, len(t.data)) - var h = t.first - t.first = -1 - t.last = -1 - swap(t.data, n) - let hc = genHash(key) - while h >= 0: - var nxt = n[h].next - if isFilled(n[h].hcode): - if n[h].hcode == hc and n[h].key == key: - dec t.counter - else: - var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) - rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) - h = nxt -proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = - ## deletes ``key`` from ordered hash table ``t``. O(n) complexity. Does nothing - ## if the key does not exist. - t[].del(key) +iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion + ## order. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTableRef[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.newOrderedTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: o + ## # value: [1, 5, 7, 9] + ## # key: e + ## # value: [2, 4, 6, 8] + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) -# ------------------------------ count tables ------------------------------- +iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion + ## order. The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'o': @[1, 5, 7, 9, 11], 'e': @[2, 4, 6, 8, 12]}.newOrderedTable + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: OrderedTableRef[A, B]): A = + ## Iterates over any key in the table ``t`` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for k in a.keys: + a[k].add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.newOrderedTable + forAllOrderedPairs: + yield t.data[h].key + +iterator values*[A, B](t: OrderedTableRef[A, B]): B = + ## Iterates over any value in the table ``t`` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for v in a.values: + doAssert v.len == 4 + forAllOrderedPairs: + yield t.data[h].val + +iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B = + ## Iterates over any value in the table ``t`` in insertion order. The values + ## can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for v in a.mvalues: + v.add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, 99]}.newOrderedTable + forAllOrderedPairs: + yield t.data[h].val + + + + + + + +# ------------------------------------------------------------------------- +# ------------------------------ CountTable ------------------------------- +# ------------------------------------------------------------------------- type - CountTable* [ - A] = object ## table that counts the number of each key + CountTable* [A] = object + ## Hash table that counts the number of each key. + ## + ## For creating an empty CountTable, use `initCountTable proc + ## <#initCountTable,int>`_. data: seq[tuple[key: A, val: int]] counter: int - CountTableRef*[A] = ref CountTable[A] + CountTableRef*[A] = ref CountTable[A] ## Ref version of + ## `CountTable<#CountTable>`_. + ## + ## For creating a new empty CountTableRef, use `newCountTable proc + ## <#newCountTable,int>`_. -proc len*[A](t: CountTable[A]): int = - ## returns the number of keys in ``t``. - result = t.counter -proc clear*[A](t: CountTableRef[A]) = - ## resets the table so that it is empty. - clearImpl() - -proc clear*[A](t: var CountTable[A]) = - ## resets the table so that it is empty. - clearImpl() - -iterator pairs*[A](t: CountTable[A]): (A, int) = - ## iterates over any ``(key, value)`` pair in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) - -iterator mpairs*[A](t: var CountTable[A]): (A, var int) = - ## iterates over any ``(key, value)`` pair in the table ``t``. The values can - ## be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) - -iterator keys*[A](t: CountTable[A]): A = - ## iterates over any key in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].key - -iterator values*[A](t: CountTable[A]): int = - ## iterates over any value in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val - -iterator mvalues*[A](t: CountTable[A]): var int = - ## iterates over any value in the table ``t``. The values can be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val - -proc rawGet[A](t: CountTable[A], key: A): int = - var h: Hash = hash(key) and high(t.data) # start with real hash value - while t.data[h].val != 0: - if t.data[h].key == key: return h - h = nextTry(h, high(t.data)) - result = -1 - h # < 0 => MISSING; insert idx = -1 - result +# ------------------------------ helpers --------------------------------- proc rawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]], key: A, val: int) = @@ -947,8 +2052,71 @@ proc enlarge[A](t: var CountTable[A]) = if t.data[i].val != 0: rawInsert(t, n, t.data[i].key, t.data[i].val) swap(t.data, n) +proc rawGet[A](t: CountTable[A], key: A): int = + var h: Hash = hash(key) and high(t.data) # start with real hash value + while t.data[h].val != 0: + if t.data[h].key == key: return h + h = nextTry(h, high(t.data)) + result = -1 - h # < 0 => MISSING; insert idx = -1 - result + +template ctget(t, key, default: untyped): untyped = + var index = rawGet(t, key) + result = if index >= 0: t.data[index].val else: default + +proc inc*[A](t: var CountTable[A], key: A, val = 1) + +# ---------------------------------------------------------------------- + +proc initCountTable*[A](initialSize=64): CountTable[A] = + ## Creates a new count table that is empty. + ## + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `toCountTable proc<#toCountTable,openArray[A]>`_ + ## * `newCountTable proc<#newCountTable,int>`_ for creating a + ## `CountTableRef` + assert isPowerOfTwo(initialSize) + result.counter = 0 + newSeq(result.data, initialSize) + +proc toCountTable*[A](keys: openArray[A]): CountTable[A] = + ## Creates a new count table with every member of a container ``keys`` + ## having a count of how many times it occurs in that container. + result = initCountTable[A](rightSize(keys.len)) + for key in items(keys): result.inc(key) + +proc `[]`*[A](t: CountTable[A], key: A): int = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise ``0`` is returned. + ## + ## See also: + ## * `getOrDefault<#getOrDefault,CountTable[A],A,int>`_ to return + ## a custom value if the key doesn't exist + ## * `mget proc<#mget,CountTable[A],A>`_ + ## * `[]= proc<#[]%3D,CountTable[A],A,int>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,CountTable[A],A>`_ for checking if a key + ## is in the table + ctget(t, key, 0) + +proc mget*[A](t: var CountTable[A], key: A): var int = + ## Retrieves the value at ``t[key]``. The value can be modified. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + get(t, key) + proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = - ## puts a ``(key, value)`` pair into ``t``. + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `inc proc<#inc,CountTable[A],A,int>`_ for incrementing a + ## value of a key assert val >= 0 let h = rawGet(t, key) if h >= 0: @@ -958,35 +2126,13 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = rawInsert(t, t.data, key, val) inc(t.counter) -template ctget(t, key, default: untyped): untyped = - var index = rawGet(t, key) - result = if index >= 0: t.data[index].val else: default - -proc `[]`*[A](t: CountTable[A], key: A): int = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise ``0`` is returned. - ctget(t, key, 0) - -proc mget*[A](t: var CountTable[A], key: A): var int = - ## Retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - get(t, key) - -proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the - ## integer value of ``default`` is returned. - ctget(t, key, default) - -proc hasKey*[A](t: CountTable[A], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - result = rawGet(t, key) >= 0 - -proc contains*[A](t: CountTable[A], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A](t, key) - proc inc*[A](t: var CountTable[A], key: A, val = 1) = - ## increments ``t[key]`` by ``val``. + ## Increments ``t[key]`` by ``val`` (default: 1). + runnableExamples: + var a = toCountTable("aab") + a.inc('a') + a.inc('b', 10) + doAssert a == toCountTable("aaabbbbbbbbbbb") var index = rawGet(t, key) if index >= 0: inc(t.data[index].val, val) @@ -996,33 +2142,11 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) = rawInsert(t, t.data, key, val) inc(t.counter) -proc initCountTable*[A](initialSize=64): CountTable[A] = - ## creates a new count table that is empty. - ## - ## ``initialSize`` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module or the ``rightSize`` proc in this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - newSeq(result.data, initialSize) - -proc toCountTable*[A](keys: openArray[A]): CountTable[A] = - ## creates a new count table with every key in ``keys`` having a count - ## of how many times it occurs in ``keys``. - result = initCountTable[A](rightSize(keys.len)) - for key in items(keys): result.inc(key) - -proc `$`*[A](t: CountTable[A]): string = - ## The ``$`` operator for count tables. - dollarImpl() - -proc `==`*[A](s, t: CountTable[A]): bool = - ## The ``==`` operator for count tables. Returns ``true`` iff both tables - ## contain the same keys with the same count. Insert order does not matter. - equalsImpl(s, t) - proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) + ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) + ## + ## See also: + ## * `largest proc<#largest,CountTable[A]>`_ assert t.len > 0 var minIdx = -1 for h in 0..high(t.data): @@ -1032,7 +2156,10 @@ proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = result.val = t.data[minIdx].val proc largest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) + ## Returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) + ## + ## See also: + ## * `smallest proc<#smallest,CountTable[A]>`_ assert t.len > 0 var maxIdx = 0 for h in 1..high(t.data): @@ -1040,11 +2167,49 @@ proc largest*[A](t: CountTable[A]): tuple[key: A, val: int] = result.key = t.data[maxIdx].key result.val = t.data[maxIdx].val +proc hasKey*[A](t: CountTable[A], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,CountTable[A],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,CountTable[A],A,int>`_ to return + ## a custom value if the key doesn't exist + result = rawGet(t, key) >= 0 + +proc contains*[A](t: CountTable[A], key: A): bool = + ## Alias of `hasKey proc<#hasKey,CountTable[A],A>`_ for use with + ## the ``in`` operator. + return hasKey[A](t, key) + +proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = + ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the + ## integer value of ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,CountTable[A],A>`_ for checking if a key + ## is in the table + ctget(t, key, default) + +proc len*[A](t: CountTable[A]): int = + ## Returns the number of keys in ``t``. + result = t.counter + +proc clear*[A](t: var CountTable[A]) = + ## Resets the table so that it is empty. + clearImpl() + proc sort*[A](t: var CountTable[A]) = - ## sorts the count table so that the entry with the highest counter comes - ## first. This is destructive! You must not modify ``t`` afterwards! - ## You can use the iterators ``pairs``, ``keys``, and ``values`` to iterate over - ## ``t`` in the sorted order. + ## Sorts the count table so that the entry with the highest counter comes + ## first. + ## + ## **This is destructive! You must not modify ``t`` afterwards!** + ## + ## You can use the iterators `pairs<#pairs.i,CountTable[A]>`_, + ## `keys<#keys.i,CountTable[A]>`_, and `values<#values.i,CountTable[A]>`_ + ## to iterate over ``t`` in the sorted order. # we use shellsort here; fast enough and simple var h = 1 @@ -1061,131 +2226,374 @@ proc sort*[A](t: var CountTable[A]) = if j < h: break if h == 1: break -proc len*[A](t: CountTableRef[A]): int = - ## returns the number of keys in ``t``. - result = t.counter - -iterator pairs*[A](t: CountTableRef[A]): (A, int) = - ## iterates over any ``(key, value)`` pair in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) - -iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = - ## iterates over any ``(key, value)`` pair in the table ``t``. The values can - ## be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) - -iterator keys*[A](t: CountTableRef[A]): A = - ## iterates over any key in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].key - -iterator values*[A](t: CountTableRef[A]): int = - ## iterates over any value in the table ``t``. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val - -iterator mvalues*[A](t: CountTableRef[A]): var int = - ## iterates over any value in the table ``t``. The values can be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val - -proc `[]`*[A](t: CountTableRef[A], key: A): int = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - result = t[][key] - -proc mget*[A](t: CountTableRef[A], key: A): var int = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - mget(t[], key) - -proc getOrDefault*[A](t: CountTableRef[A], key: A): int = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, 0 (the - ## default initialization value of ``int``), is returned. - result = t[].getOrDefault(key) - -proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## retrieves the value at ``t[key]`` iff ``key`` is in ``t``. Otherwise, the - ## integer value of ``default`` is returned. - result = t[].getOrDefault(key, default) - -proc hasKey*[A](t: CountTableRef[A], key: A): bool = - ## returns true iff ``key`` is in the table ``t``. - result = t[].hasKey(key) - -proc contains*[A](t: CountTableRef[A], key: A): bool = - ## Alias of ``hasKey`` for use with the ``in`` operator. - return hasKey[A](t, key) - -proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = - ## puts a ``(key, value)`` pair into ``t``. ``val`` has to be positive. - assert val > 0 - t[][key] = val - -proc inc*[A](t: CountTableRef[A], key: A, val = 1) = - ## increments ``t[key]`` by ``val``. - t[].inc(key, val) - -proc newCountTable*[A](initialSize=64): CountTableRef[A] = - ## creates a new count table that is empty. - ## - ## ``initialSize`` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module or the ``rightSize`` method in this module. - new(result) - result[] = initCountTable[A](initialSize) - -proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = - ## creates a new count table with every key in ``keys`` having a count - ## of how many times it occurs in ``keys``. - result = newCountTable[A](rightSize(keys.len)) - for key in items(keys): result.inc(key) - -proc `$`*[A](t: CountTableRef[A]): string = - ## The ``$`` operator for count tables. - dollarImpl() - -proc `==`*[A](s, t: CountTableRef[A]): bool = - ## The ``==`` operator for count tables. Returns ``true`` iff either both tables - ## are ``nil`` or none is ``nil`` and both contain the same keys with the same - ## count. Insert order does not matter. - if isNil(s): result = isNil(t) - elif isNil(t): result = false - else: result = s[] == t[] - -proc smallest*[A](t: CountTableRef[A]): (A, int) = - ## returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) - t[].smallest - -proc largest*[A](t: CountTableRef[A]): (A, int) = - ## returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) - t[].largest - -proc sort*[A](t: CountTableRef[A]) = - ## sorts the count table so that the entry with the highest counter comes - ## first. This is destructive! You must not modify ``t`` afterwards! - ## You can use the iterators ``pairs``, ``keys``, and ``values`` to iterate over - ## ``t`` in the sorted order. - t[].sort - proc merge*[A](s: var CountTable[A], t: CountTable[A]) = - ## merges the second table into the first one. + ## Merges the second table into the first one (must be declared as `var`). + runnableExamples: + var a = toCountTable("aaabbc") + let b = toCountTable("bcc") + a.merge(b) + doAssert a == toCountTable("aaabbbccc") for key, value in t: s.inc(key, value) proc merge*[A](s, t: CountTable[A]): CountTable[A] = - ## merges the two tables into a new one. + ## Merges the two tables into a new one. + runnableExamples: + let + a = toCountTable("aaabbc") + b = toCountTable("bcc") + doAssert merge(a, b) == toCountTable("aaabbbccc") result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) for table in @[s, t]: for key, value in table: result.inc(key, value) +proc `$`*[A](t: CountTable[A]): string = + ## The ``$`` operator for count tables. Used internally when calling `echo` + ## on a table. + dollarImpl() + +proc `==`*[A](s, t: CountTable[A]): bool = + ## The ``==`` operator for count tables. Returns ``true`` if both tables + ## contain the same keys with the same count. Insert order does not matter. + equalsImpl(s, t) + + +iterator pairs*[A](t: CountTable[A]): (A, int) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTable[A]>`_ + ## * `keys iterator<#keys.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = toCountTable("abracadabra") + ## + ## for k, v in pairs(a): + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: a + ## # value: 5 + ## # key: b + ## # value: 2 + ## # key: c + ## # value: 1 + ## # key: d + ## # value: 1 + ## # key: r + ## # value: 2 + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A](t: var CountTable[A]): (A, var int) = + ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for k, v in mpairs(a): + v = 2 + doAssert a == toCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator keys*[A](t: CountTable[A]): A = + ## Iterates over any key in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for k in keys(a): + a[k] = 2 + doAssert a == toCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].key + +iterator values*[A](t: CountTable[A]): int = + ## Iterates over any value in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `keys iterator<#keys.i,CountTable[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTable[A]>`_ + runnableExamples: + let a = toCountTable("abracadabra") + for v in values(a): + assert v < 10 + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + +iterator mvalues*[A](t: var CountTable[A]): var int = + ## Iterates over any value in the table ``t`` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for v in mvalues(a): + v = 2 + doAssert a == toCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + + + + + + + +# --------------------------------------------------------------------------- +# ---------------------------- CountTableRef -------------------------------- +# --------------------------------------------------------------------------- + +proc inc*[A](t: CountTableRef[A], key: A, val = 1) + +proc newCountTable*[A](initialSize=64): CountTableRef[A] = + ## Creates a new ref count table that is empty. + ## + ## ``initialSize`` must be a power of two (default: 64). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_ or the `rightSize proc<#rightSize,Natural>`_ + ## from this module. + ## + ## See also: + ## * `newCountTable proc<#newCountTable,openArray[A]>`_ for creating + ## a `CountTableRef` from a collection + ## * `initCountTable proc<#initCountTable,int>`_ for creating a + ## `CountTable` + new(result) + result[] = initCountTable[A](initialSize) + +proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = + ## Creates a new ref count table with every member of a container ``keys`` + ## having a count of how many times it occurs in that container. + result = newCountTable[A](rightSize(keys.len)) + for key in items(keys): result.inc(key) + +proc `[]`*[A](t: CountTableRef[A], key: A): int = + ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. + ## Otherwise ``0`` is returned. + ## + ## See also: + ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return + ## a custom value if the key doesn't exist + ## * `mget proc<#mget,CountTableRef[A],A>`_ + ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key + ## is in the table + result = t[][key] + +proc mget*[A](t: CountTableRef[A], key: A): var int = + ## Retrieves the value at ``t[key]``. The value can be modified. + ## + ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + mget(t[], key) + +proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = + ## Inserts a ``(key, value)`` pair into ``t``. + ## + ## See also: + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a + ## value of a key + assert val > 0 + t[][key] = val + +proc inc*[A](t: CountTableRef[A], key: A, val = 1) = + ## Increments ``t[key]`` by ``val`` (default: 1). + runnableExamples: + var a = newCountTable("aab") + a.inc('a') + a.inc('b', 10) + doAssert a == newCountTable("aaabbbbbbbbbbb") + t[].inc(key, val) + +proc smallest*[A](t: CountTableRef[A]): (A, int) = + ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) + ## + ## See also: + ## * `largest proc<#largest,CountTableRef[A]>`_ + t[].smallest + +proc largest*[A](t: CountTableRef[A]): (A, int) = + ## Returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) + ## + ## See also: + ## * `smallest proc<#smallest,CountTable[A]>`_ + t[].largest + +proc hasKey*[A](t: CountTableRef[A], key: A): bool = + ## Returns true if ``key`` is in the table ``t``. + ## + ## See also: + ## * `contains proc<#contains,CountTableRef[A],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,CountTableRef[A],A,int>`_ to return + ## a custom value if the key doesn't exist + result = t[].hasKey(key) + +proc contains*[A](t: CountTableRef[A], key: A): bool = + ## Alias of `hasKey proc<#hasKey,CountTableRef[A],A>`_ for use with + ## the ``in`` operator. + return hasKey[A](t, key) + +proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = + ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the + ## integer value of ``default`` is returned. + ## + ## See also: + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key + ## is in the table + result = t[].getOrDefault(key, default) + +proc len*[A](t: CountTableRef[A]): int = + ## Returns the number of keys in ``t``. + result = t.counter + +proc clear*[A](t: CountTableRef[A]) = + ## Resets the table so that it is empty. + clearImpl() + +proc sort*[A](t: CountTableRef[A]) = + ## Sorts the count table so that the entry with the highest counter comes + ## first. + ## + ## **This is destructive! You must not modify `t` afterwards!** + ## + ## You can use the iterators `pairs<#pairs.i,CountTableRef[A]>`_, + ## `keys<#keys.i,CountTableRef[A]>`_, and `values<#values.i,CountTableRef[A]>`_ + ## to iterate over ``t`` in the sorted order. + t[].sort + proc merge*[A](s, t: CountTableRef[A]) = - ## merges the second table into the first one. + ## Merges the second table into the first one. + runnableExamples: + let + a = newCountTable("aaabbc") + b = newCountTable("bcc") + a.merge(b) + doAssert a == newCountTable("aaabbbccc") s[].merge(t[]) +proc `$`*[A](t: CountTableRef[A]): string = + ## The ``$`` operator for count tables. Used internally when calling `echo` + ## on a table. + dollarImpl() + +proc `==`*[A](s, t: CountTableRef[A]): bool = + ## The ``==`` operator for count tables. Returns ``true`` if either both tables + ## are ``nil``, or neither is ``nil`` and both contain the same keys with the same + ## count. Insert order does not matter. + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: result = s[] == t[] + + +iterator pairs*[A](t: CountTableRef[A]): (A, int) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ + ## * `keys iterator<#keys.i,CountTableRef[A]>`_ + ## * `values iterator<#values.i,CountTableRef[A]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let a = newCountTable("abracadabra") + ## + ## for k, v in pairs(a): + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: a + ## # value: 5 + ## # key: b + ## # value: 2 + ## # key: c + ## # value: 1 + ## # key: d + ## # value: 1 + ## # key: r + ## # value: 2 + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = + ## Iterates over any ``(key, value)`` pair in the table ``t``. The values can + ## be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTableRef[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTableRef[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for k, v in mpairs(a): + v = 2 + doAssert a == newCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + +iterator keys*[A](t: CountTableRef[A]): A = + ## Iterates over any key in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for k in keys(a): + a[k] = 2 + doAssert a == newCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].key + +iterator values*[A](t: CountTableRef[A]): int = + ## Iterates over any value in the table ``t``. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTableRef[A]>`_ + ## * `keys iterator<#keys.i,CountTableRef[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTableRef[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for v in values(a): + assert v < 10 + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + +iterator mvalues*[A](t: CountTableRef[A]): var int = + ## Iterates over any value in the table ``t``. The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ + ## * `values iterator<#values.i,CountTableRef[A]>`_ + runnableExamples: + var a = newCountTable("abracadabra") + for v in mvalues(a): + v = 2 + doAssert a == newCountTable("aabbccddrr") + for h in 0..high(t.data): + if t.data[h].val != 0: yield t.data[h].val + + + + when isMainModule: type Person = object @@ -1302,9 +2710,9 @@ when isMainModule: #test_counttable.nim(7, 43) template/generic instantiation from here #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode - doAssert 0 == t.getOrDefault(testKey) + doAssert 0 == t[testKey] t.inc(testKey, 3) - doAssert 3 == t.getOrDefault(testKey) + doAssert 3 == t[testKey] block: # Clear tests From 387f481e863996c6791171f34cb05e59e620ce1b Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 14 Jan 2019 11:29:31 +0100 Subject: [PATCH 138/350] better docs: math --- lib/pure/math.nim | 523 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 423 insertions(+), 100 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 72abae2653..97d1e7987a 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -7,15 +7,50 @@ # distribution, for details about the copyright. # -## Constructive mathematics is naturally typed. -- Simon Thompson +## :Author: Nim contributors +## +## *Constructive mathematics is naturally typed.* -- Simon Thompson ## ## Basic math routines for Nim. +## +## Note that the trigonometric functions naturally operate on radians. +## The helper functions `degToRad<#degToRad,T>`_ and `radToDeg<#radToDeg,T>`_ +## provide conversion between radians and degrees. +## +## .. code-block:: +## +## import math +## from sequtils import map +## +## let a = [0.0, PI/6, PI/4, PI/3, PI/2] +## +## echo a.map(sin) +## # @[0.0, 0.499…, 0.707…, 0.866…, 1.0] +## +## echo a.map(tan) +## # @[0.0, 0.577…, 0.999…, 1.732…, 1.633…e+16] +## +## echo cos(degToRad(180.0)) +## # -1.0 +## +## echo sqrt(-1.0) +## # nan (use `complex` module) +## ## This module is available for the `JavaScript target ## `_. ## -## Note that the trigonometric functions naturally operate on radians. -## The helper functions `degToRad` and `radToDeg` provide conversion -## between radians and degrees. +## **See also:** +## * `complex module`_ for complex numbers and their +## mathematical operations +## * `rationals module`_ for rational numbers and their +## mathematical operations +## * `fenv module`_ for handling of floating-point rounding +## and exceptions (overflow, zero-devide, etc.) +## * `random module`_ for fast and tiny random number generator +## * `mersenne module`_ for Mersenne twister random number generator +## * `stats module`_ for statistical analysis +## * `strformat module`_ for formatting floats for print + include "system/inclrtl" {.push debugger:off .} # the user does not want to trace a part @@ -25,9 +60,11 @@ import bitops proc binom*(n, k: int): int {.noSideEffect.} = ## Computes the `binomial coefficient `_. - ## - ## .. code-block:: nim - ## echo binom(6, 2) ## 15 + runnableExamples: + doAssert binom(6, 2) == binom(6, 4) + doAssert binom(6, 2) == 15 + doAssert binom(-6, 2) == 1 + doAssert binom(6, 0) == 1 if k <= 0: return 1 if 2*k > n: return binom(n, n-k) result = n @@ -40,10 +77,15 @@ proc createFactTable[N: static[int]]: array[N, int] = result[i] = result[i - 1] * i proc fac*(n: int): int = - ## Computes the `factorial `_ of a non-negative integer ``n`` + ## Computes the `factorial `_ of + ## a non-negative integer ``n``. ## - ## .. code-block:: nim - ## echo fac(4) ## 24 + ## See also: + ## * `prod proc <#prod,openArray[T]>`_ + runnableExamples: + doAssert fac(3) == 6 + doAssert fac(4) == 24 + doAssert fac(10) == 3628800 const factTable = when sizeof(int) == 4: createFactTable[13]() @@ -59,25 +101,26 @@ when defined(Posix): {.passl: "-lm".} const - PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number) - TAU* = 2.0 * PI ## the circle constant TAU (= 2 * PI) + PI* = 3.1415926535897932384626433 ## The circle constant PI (Ludolph's number) + TAU* = 2.0 * PI ## The circle constant TAU (= 2 * PI) E* = 2.71828182845904523536028747 ## Euler's number - MaxFloat64Precision* = 16 ## maximum number of meaningful digits + MaxFloat64Precision* = 16 ## Maximum number of meaningful digits ## after the decimal point for Nim's ## ``float64`` type. - MaxFloat32Precision* = 8 ## maximum number of meaningful digits + MaxFloat32Precision* = 8 ## Maximum number of meaningful digits ## after the decimal point for Nim's ## ``float32`` type. - MaxFloatPrecision* = MaxFloat64Precision ## maximum number of + MaxFloatPrecision* = MaxFloat64Precision ## Maximum number of ## meaningful digits ## after the decimal point ## for Nim's ``float`` type. - RadPerDeg = PI / 180.0 ## number of radians per degree + RadPerDeg = PI / 180.0 ## Number of radians per degree type - FloatClass* = enum ## describes the class a floating point value belongs to. - ## This is the type that is returned by `classify`. + FloatClass* = enum ## Describes the class a floating point value belongs to. + ## This is the type that is returned by + ## `classify proc <#classify,float>`_. fcNormal, ## value is an ordinary nonzero floating point value fcSubnormal, ## value is a subnormal (a very small) floating point value fcZero, ## value is zero @@ -87,13 +130,14 @@ type fcNegInf ## value is negative infinity proc classify*(x: float): FloatClass = - ## Classifies a floating point value. Returns ``x``'s class as specified by - ## `FloatClass`. + ## Classifies a floating point value. ## - ## .. code-block:: nim - ## echo classify(0.3) ## fcNormal - ## echo classify(0.0) ## fcZero - ## echo classify(0.3/0.0) ## fcInf + ## Returns ``x``'s class as specified by `FloatClass enum<#FloatClass>`_. + runnableExamples: + doAssert classify(0.3) == fcNormal + doAssert classify(0.0) == fcZero + doAssert classify(0.3/0.0) == fcInf + doAssert classify(-0.3/0.0) == fcNegInf # JavaScript and most C compilers have no classify: if x == 0.0: @@ -110,20 +154,30 @@ proc classify*(x: float): FloatClass = proc isPowerOfTwo*(x: int): bool {.noSideEffect.} = ## Returns ``true``, if ``x`` is a power of two, ``false`` otherwise. + ## ## Zero and negative numbers are not a power of two. ## - ## .. code-block:: nim - ## echo isPowerOfTwo(5) ## false - ## echo isPowerOfTwo(8) ## true + ## See also: + ## * `nextPowerOfTwo proc<#nextPowerOfTwo,int>`_ + runnableExamples: + doAssert isPowerOfTwo(16) == true + doAssert isPowerOfTwo(5) == false + doAssert isPowerOfTwo(0) == false + doAssert isPowerOfTwo(-16) == false return (x > 0) and ((x and (x - 1)) == 0) proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = ## Returns ``x`` rounded up to the nearest power of two. + ## ## Zero and negative numbers get rounded up to 1. ## - ## .. code-block:: nim - ## echo nextPowerOfTwo(8) ## 8 - ## echo nextPowerOfTwo(9) ## 16 + ## See also: + ## * `isPowerOfTwo proc<#isPowerOfTwo,int>`_ + runnableExamples: + doAssert nextPowerOfTwo(16) == 16 + doAssert nextPowerOfTwo(5) == 8 + doAssert nextPowerOfTwo(0) == 1 + doAssert nextPowerOfTwo(-16) == 1 result = x - 1 when defined(cpu64): result = result or (result shr 32) @@ -138,9 +192,12 @@ proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = proc countBits32*(n: int32): int {.noSideEffect.} = ## Counts the set bits in ``n``. - ## - ## .. code-block:: nim - ## echo countBits32(13'i32) ## 3 + runnableExamples: + doAssert countBits32(7) == 3 + doAssert countBits32(8) == 1 + doAssert countBits32(15) == 4 + doAssert countBits32(16) == 1 + doAssert countBits32(17) == 2 var v = n v = v -% ((v shr 1'i32) and 0x55555555'i32) v = (v and 0x33333333'i32) +% ((v shr 2'i32) and 0x33333333'i32) @@ -148,37 +205,58 @@ proc countBits32*(n: int32): int {.noSideEffect.} = proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## Computes the sum of the elements in ``x``. + ## ## If ``x`` is empty, 0 is returned. ## - ## .. code-block:: nim - ## echo sum([1.0, 2.5, -3.0, 4.3]) ## 4.8 + ## See also: + ## * `prod proc <#prod,openArray[T]>`_ + ## * `cumsum proc <#cumsum,openArray[T]>`_ + ## * `cumsummed proc <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-1.5, 2.7, -0.1]) == 1.1 for i in items(x): result = result + i proc prod*[T](x: openArray[T]): T {.noSideEffect.} = ## Computes the product of the elements in ``x``. + ## ## If ``x`` is empty, 1 is returned. ## - ## .. code-block:: nim - ## echo prod([1.0, 3.0, -0.2]) ## -0.6 + ## See also: + ## * `sum proc <#sum,openArray[T]>`_ + ## * `fac proc <#fac,int>`_ + runnableExamples: + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 result = 1.T for i in items(x): result = result * i proc cumsummed*[T](x: openArray[T]): seq[T] = - ## Return cumulative aka prefix summation of ``x``. + ## Return cumulative (aka prefix) summation of ``x``. ## - ## .. code-block:: nim - ## var x = [1, 2, 3, 4] - ## echo x.cumsummed # [1, 3, 6, 10] + ## See also: + ## * `sum proc <#sum,openArray[T]>`_ + ## * `cumsum proc <#cumsum,openArray[T]>`_ for the in-place version + ## * `cumsummed proc <#cumsummed,openArray[T]>`_ + runnableExamples: + let a = [1, 2, 3, 4] + doAssert cumsummed(a) == @[1, 3, 6, 10] result.setLen(x.len) result[0] = x[0] for i in 1 ..< x.len: result[i] = result[i-1] + x[i] proc cumsum*[T](x: var openArray[T]) = - ## Transforms ``x`` in-place into its cumulative aka prefix summation. + ## Transforms ``x`` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. ## - ## .. code-block:: nim - ## var x = [1, 2, 3, 4] - ## x.cumsum; echo x # [1, 3, 6, 10] + ## See also: + ## * `sum proc <#sum,openArray[T]>`_ + ## * `cumsummed proc <#cumsummed,openArray[T]>`_ for a version which + ## returns cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] for i in 1 ..< x.len: x[i] = x[i-1] + x[i] {.push noSideEffect.} @@ -187,20 +265,40 @@ when not defined(JS): # C proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "".} ## Computes the square root of ``x``. ## + ## See also: + ## * `cbrt proc <#cbrt,float64>`_ for cubic root + ## ## .. code-block:: nim + ## echo sqrt(4.0) ## 2.0 ## echo sqrt(1.44) ## 1.2 + ## echo sqrt(-4.0) ## nan proc cbrt*(x: float32): float32 {.importc: "cbrtf", header: "".} proc cbrt*(x: float64): float64 {.importc: "cbrt", header: "".} ## Computes the cubic root of ``x``. ## + ## See also: + ## * `sqrt proc <#sqrt,float64>`_ for square root + ## ## .. code-block:: nim + ## echo cbrt(8.0) ## 2.0 ## echo cbrt(2.197) ## 1.3 + ## echo cbrt(-27.0) ## -3.0 proc ln*(x: float32): float32 {.importc: "logf", header: "".} proc ln*(x: float64): float64 {.importc: "log", header: "".} - ## Computes the `natural logarithm `_ of ``x``. + ## Computes the `natural logarithm `_ + ## of ``x``. + ## + ## See also: + ## * `log proc <#log,T,T>`_ + ## * `log10 proc <#log10,float64>`_ + ## * `log2 proc <#log2,float64>`_ + ## * `exp proc <#exp,float64>`_ ## ## .. code-block:: nim ## echo ln(exp(4.0)) ## 4.0 + ## echo ln(1.0)) ## 0.0 + ## echo ln(0.0) ## -inf + ## echo ln(-7.0) ## nan else: # JS proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} @@ -211,8 +309,18 @@ else: # JS proc log*[T: SomeFloat](x, base: T): T = ## Computes the logarithm of ``x`` to base ``base``. ## + ## See also: + ## * `ln proc <#ln,float64>`_ + ## * `log10 proc <#log10,float64>`_ + ## * `log2 proc <#log2,float64>`_ + ## * `exp proc <#exp,float64>`_ + ## ## .. code-block:: nim - ## echo log(9.0, 3.0) ## 2.0 + ## echo log(9.0, 3.0) ## 2.0 + ## echo log(32.0, 2.0) ## 5.0 + ## echo log(0.0, 2.0) ## -inf + ## echo log(-7.0, 4.0) ## nan + ## echo log(8.0, -2.0) ## nan ln(x) / ln(base) when not defined(JS): # C @@ -220,77 +328,164 @@ when not defined(JS): # C proc log10*(x: float64): float64 {.importc: "log10", header: "".} ## Computes the common logarithm (base 10) of ``x``. ## - ## .. code-block:: nim - ## echo log10(100.0) ## 2.0 - proc exp*(x: float32): float32 {.importc: "expf", header: "".} - proc exp*(x: float64): float64 {.importc: "exp", header: "".} - ## Computes the exponential function of ``x`` (pow(E, x)). + ## See also: + ## * `ln proc <#ln,float64>`_ + ## * `log proc <#log,T,T>`_ + ## * `log2 proc <#log2,float64>`_ + ## * `exp proc <#exp,float64>`_ ## ## .. code-block:: nim - ## echo exp(1.0) ## 2.718281828459045 + ## echo log10(100.0) ## 2.0 + ## echo log10(0.0) ## nan + ## echo log10(-100.0) ## -inf + proc exp*(x: float32): float32 {.importc: "expf", header: "".} + proc exp*(x: float64): float64 {.importc: "exp", header: "".} + ## Computes the exponential function of ``x`` (e^x). + ## + ## See also: + ## * `ln proc <#ln,float64>`_ + ## * `log proc <#log,T,T>`_ + ## * `log10 proc <#log10,float64>`_ + ## * `log2 proc <#log2,float64>`_ + ## + ## .. code-block:: nim + ## echo exp(1.0) ## 2.718281828459045 ## echo ln(exp(4.0)) ## 4.0 + ## echo exp(0.0) ## 1.0 + ## echo exp(-1.0) ## 0.3678794411714423 proc sin*(x: float32): float32 {.importc: "sinf", header: "".} proc sin*(x: float64): float64 {.importc: "sin", header: "".} ## Computes the sine of ``x``. ## + ## See also: + ## * `cos proc <#cos,float64>`_ + ## * `tan proc <#tan,float64>`_ + ## * `arcsin proc <#arcsin,float64>`_ + ## * `sinh proc <#sinh,float64>`_ + ## ## .. code-block:: nim - ## echo sin(PI / 6) ## 0.4999999999999999 + ## echo sin(PI / 6) ## 0.4999999999999999 ## echo sin(degToRad(90.0)) ## 1.0 proc cos*(x: float32): float32 {.importc: "cosf", header: "".} proc cos*(x: float64): float64 {.importc: "cos", header: "".} ## Computes the cosine of ``x``. ## + ## See also: + ## * `sin proc <#sin,float64>`_ + ## * `tan proc <#tan,float64>`_ + ## * `arccos proc <#arccos,float64>`_ + ## * `cosh proc <#cosh,float64>`_ + ## ## .. code-block:: nim - ## echo cos(2 * PI) ## 1.0 + ## echo cos(2 * PI) ## 1.0 ## echo cos(degToRad(60.0)) ## 0.5000000000000001 proc tan*(x: float32): float32 {.importc: "tanf", header: "".} proc tan*(x: float64): float64 {.importc: "tan", header: "".} ## Computes the tangent of ``x``. ## + ## See also: + ## * `sin proc <#sin,float64>`_ + ## * `cos proc <#cos,float64>`_ + ## * `arctan proc <#arctan,float64>`_ + ## * `tanh proc <#tanh,float64>`_ + ## ## .. code-block:: nim ## echo tan(degToRad(45.0)) ## 0.9999999999999999 - ## echo tan(PI / 4) ## 0.9999999999999999 + ## echo tan(PI / 4) ## 0.9999999999999999 proc sinh*(x: float32): float32 {.importc: "sinhf", header: "".} proc sinh*(x: float64): float64 {.importc: "sinh", header: "".} ## Computes the `hyperbolic sine `_ of ``x``. ## + ## See also: + ## * `cosh proc <#cosh,float64>`_ + ## * `tanh proc <#tanh,float64>`_ + ## * `arcsinh proc <#arcsinh,float64>`_ + ## * `sin proc <#sin,float64>`_ + ## ## .. code-block:: nim - ## echo sinh(1.0) ## 1.175201193643801 + ## echo sinh(0.0) ## 0.0 + ## echo sinh(1.0) ## 1.175201193643801 + ## echo sinh(degToRad(90.0)) ## 2.301298902307295 proc cosh*(x: float32): float32 {.importc: "coshf", header: "".} proc cosh*(x: float64): float64 {.importc: "cosh", header: "".} ## Computes the `hyperbolic cosine `_ of ``x``. ## + ## See also: + ## * `sinh proc <#sinh,float64>`_ + ## * `tanh proc <#tanh,float64>`_ + ## * `arccosh proc <#arccosh,float64>`_ + ## * `cos proc <#cos,float64>`_ + ## ## .. code-block:: nim - ## echo cosh(1.0) ## 1.543080634815244 + ## echo cosh(0.0) ## 1.0 + ## echo cosh(1.0) ## 1.543080634815244 + ## echo cosh(degToRad(90.0)) ## 2.509178478658057 proc tanh*(x: float32): float32 {.importc: "tanhf", header: "".} proc tanh*(x: float64): float64 {.importc: "tanh", header: "".} ## Computes the `hyperbolic tangent `_ of ``x``. ## + ## See also: + ## * `sinh proc <#sinh,float64>`_ + ## * `cosh proc <#cosh,float64>`_ + ## * `arctanh proc <#arctanh,float64>`_ + ## * `tan proc <#tan,float64>`_ + ## ## .. code-block:: nim - ## echo tanh(1.0) ## 0.7615941559557649 + ## echo tanh(0.0) ## 0.0 + ## echo tanh(1.0) ## 0.7615941559557649 + ## echo tanh(degToRad(90.0)) ## 0.9171523356672744 proc arccos*(x: float32): float32 {.importc: "acosf", header: "".} proc arccos*(x: float64): float64 {.importc: "acos", header: "".} ## Computes the arc cosine of ``x``. ## + ## See also: + ## * `arcsin proc <#arcsin,float64>`_ + ## * `arctan proc <#arctan,float64>`_ + ## * `arctan2 proc <#arctan2,float64,float64>`_ + ## * `cos proc <#cos,float64>`_ + ## ## .. code-block:: nim - ## echo arccos(1.0) ## 0.0 + ## echo radToDeg(arccos(0.0)) ## 90.0 + ## echo radToDeg(arccos(1.0)) ## 0.0 proc arcsin*(x: float32): float32 {.importc: "asinf", header: "".} proc arcsin*(x: float64): float64 {.importc: "asin", header: "".} ## Computes the arc sine of ``x``. + ## + ## See also: + ## * `arccos proc <#arccos,float64>`_ + ## * `arctan proc <#arctan,float64>`_ + ## * `arctan2 proc <#arctan2,float64,float64>`_ + ## * `sin proc <#sin,float64>`_ + ## + ## .. code-block:: nim + ## echo radToDeg(arcsin(0.0)) ## 0.0 + ## echo radToDeg(arcsin(1.0)) ## 90.0 proc arctan*(x: float32): float32 {.importc: "atanf", header: "".} proc arctan*(x: float64): float64 {.importc: "atan", header: "".} ## Calculate the arc tangent of ``x``. ## + ## See also: + ## * `arcsin proc <#arcsin,float64>`_ + ## * `arccos proc <#arccos,float64>`_ + ## * `arctan2 proc <#arctan2,float64,float64>`_ + ## * `tan proc <#tan,float64>`_ + ## ## .. code-block:: nim ## echo arctan(1.0) ## 0.7853981633974483 ## echo radToDeg(arctan(1.0)) ## 45.0 proc arctan2*(y, x: float32): float32 {.importc: "atan2f", header: "".} proc arctan2*(y, x: float64): float64 {.importc: "atan2", header: "".} ## Calculate the arc tangent of ``y`` / ``x``. - ## `arctan2` returns the arc tangent of ``y`` / ``x``; it produces correct - ## results even when the resulting angle is near pi/2 or -pi/2 - ## (``x`` near 0). + ## + ## It produces correct results even when the resulting angle is near + ## pi/2 or -pi/2 (``x`` near 0). + ## + ## See also: + ## * `arcsin proc <#arcsin,float64>`_ + ## * `arccos proc <#arccos,float64>`_ + ## * `arctan proc <#arctan,float64>`_ + ## * `tan proc <#tan,float64>`_ ## ## .. code-block:: nim ## echo arctan2(1.0, 0.0) ## 1.570796326794897 @@ -331,18 +526,18 @@ else: # JS proc arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} proc cot*[T: float32|float64](x: T): T = 1.0 / tan(x) - ## Computes the cotangent of ``x``. + ## Computes the cotangent of ``x`` (1 / tan(x)). proc sec*[T: float32|float64](x: T): T = 1.0 / cos(x) - ## Computes the secant of ``x``. + ## Computes the secant of ``x`` (1 / cos(x)). proc csc*[T: float32|float64](x: T): T = 1.0 / sin(x) - ## Computes the cosecant of ``x``. + ## Computes the cosecant of ``x`` (1 / sin(x)). proc coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) - ## Computes the hyperbolic cotangent of ``x``. + ## Computes the hyperbolic cotangent of ``x`` (1 / tanh(x)). proc sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) - ## Computes the hyperbolic secant of ``x``. + ## Computes the hyperbolic secant of ``x`` (1 / cosh(x)). proc csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) - ## Computes the hyperbolic cosecant of ``x``. + ## Computes the hyperbolic cosecant of ``x`` (1 / sinh(x)). proc arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) ## Computes the inverse cotangent of ``x``. @@ -370,11 +565,17 @@ when not defined(JS): # C ## echo hypot(4.0, 3.0) ## 5.0 proc pow*(x, y: float32): float32 {.importc: "powf", header: "".} proc pow*(x, y: float64): float64 {.importc: "pow", header: "".} - ## computes x to power raised of y. + ## Computes x to power raised of y. ## - ## To compute power between integers, use ``^`` e.g. 2 ^ 6 + ## To compute power between integers (e.g. 2^6), use `^ proc<#^,T,Natural>`_. + ## + ## See also: + ## * `^ proc<#^,T,Natural>`_ + ## * `sqrt proc <#sqrt,float64>`_ + ## * `cbrt proc <#cbrt,float64>`_ ## ## .. code-block:: nim + ## echo pow(100, 1.5) ## 1000.0 ## echo pow(16.0, 0.5) ## 4.0 # TODO: add C89 version on windows @@ -388,6 +589,15 @@ when not defined(JS): # C proc gamma*(x: float32): float32 {.importc: "tgammaf", header: "".} proc gamma*(x: float64): float64 {.importc: "tgamma", header: "".} ## Computes the the `gamma function `_ for ``x``. + ## + ## See also: + ## * `lgamma proc <#lgamma,float64>`_ for a natural log of gamma function + ## + ## .. code-block:: Nim + ## echo gamma(1.0) # 1.0 + ## echo gamma(4.0) # 6.0 + ## echo gamma(11.0) # 3628800.0 + ## echo gamma(-1.0) # nan proc tgamma*(x: float32): float32 {.deprecated: "use gamma instead", importc: "tgammaf", header: "".} proc tgamma*(x: float64): float64 @@ -397,19 +607,43 @@ when not defined(JS): # C proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "".} proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "".} ## Computes the natural log of the gamma function for ``x``. + ## + ## See also: + ## * `gamma proc <#gamma,float64>`_ for gamma function + ## + ## .. code-block:: Nim + ## echo lgamma(1.0) # 1.0 + ## echo lgamma(4.0) # 1.791759469228055 + ## echo lgamma(11.0) # 15.10441257307552 + ## echo lgamma(-1.0) # inf proc floor*(x: float32): float32 {.importc: "floorf", header: "".} proc floor*(x: float64): float64 {.importc: "floor", header: "".} ## Computes the floor function (i.e., the largest integer not greater than ``x``). ## + ## See also: + ## * `ceil proc <#ceil,float64>`_ + ## * `round proc <#round,float64>`_ + ## * `trunc proc <#trunc,float64>`_ + ## ## .. code-block:: nim + ## echo floor(2.1) ## 2.0 + ## echo floor(2.9) ## 2.0 ## echo floor(-3.5) ## -4.0 proc ceil*(x: float32): float32 {.importc: "ceilf", header: "".} proc ceil*(x: float64): float64 {.importc: "ceil", header: "".} - ## Computes the ceiling function (i.e., the smallest integer not less than ``x``). + ## Computes the ceiling function (i.e., the smallest integer not smaller + ## than ``x``). + ## + ## See also: + ## * `floor proc <#floor,float64>`_ + ## * `round proc <#round,float64>`_ + ## * `trunc proc <#trunc,float64>`_ ## ## .. code-block:: nim + ## echo ceil(2.1) ## 3.0 + ## echo ceil(2.9) ## 3.0 ## echo ceil(-2.1) ## -2.0 when windowsCC89: @@ -470,26 +704,50 @@ when not defined(JS): # C else: proc round*(x: float32): float32 {.importc: "roundf", header: "".} proc round*(x: float64): float64 {.importc: "round", header: "".} - ## Rounds a float to zero decimal places. Used internally by the round - ## function when the specified number of places is 0. + ## Rounds a float to zero decimal places. + ## + ## Used internally by the `round proc <#round,T,int>`_ + ## when the specified number of places is 0. + ## + ## See also: + ## * `round proc <#round,T,int>`_ for rounding to the specific + ## number of decimal places + ## * `floor proc <#floor,float64>`_ + ## * `ceil proc <#ceil,float64>`_ + ## * `trunc proc <#trunc,float64>`_ + ## + ## .. code-block:: nim + ## echo round(3.4) ## 3.0 + ## echo round(3.5) ## 4.0 + ## echo round(4.5) ## 5.0 proc trunc*(x: float32): float32 {.importc: "truncf", header: "".} proc trunc*(x: float64): float64 {.importc: "trunc", header: "".} ## Truncates ``x`` to the decimal point. ## + ## See also: + ## * `floor proc <#floor,float64>`_ + ## * `ceil proc <#ceil,float64>`_ + ## * `round proc <#round,float64>`_ + ## ## .. code-block:: nim ## echo trunc(PI) # 3.0 ## echo trunc(-1.85) # -1.0 proc fmod*(x, y: float32): float32 {.deprecated: "use mod instead", importc: "fmodf", header: "".} proc fmod*(x, y: float64): float64 {.deprecated: "use mod instead", importc: "fmod", header: "".} + ## **Deprecated since version 0.19.0**: Use the `mod proc + ## <#mod,float64,float64>`_ instead. + ## ## Computes the remainder of ``x`` divided by ``y``. - ## **Deprecated since version 0.19.0**: Use the ``mod`` operator instead. proc `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "".} proc `mod`*(x, y: float64): float64 {.importc: "fmod", header: "".} ## Computes the modulo operation for float values (the remainder of ``x`` divided by ``y``). ## + ## See also: + ## * `floorMod proc <#floorMod,T,T>`_ for Python-like (% operator) behavior + ## ## .. code-block:: nim ## ( 6.5 mod 2.5) == 1.5 ## (-6.5 mod 2.5) == -1.5 @@ -520,16 +778,22 @@ else: # JS ## (-6.5 mod -2.5) == -1.5 proc round*[T: float32|float64](x: T, places: int): T {.deprecated: "use format instead".} = + ## **Deprecated:** use `strformat module `_ + ## ## Decimal rounding on a binary floating point number. ## ## This function is NOT reliable. Floating point numbers cannot hold - ## non integer decimals precisely. If ``places`` is 0 (or omitted), + ## non integer decimals precisely. If ``places`` is 0 (or omitted), ## round to the nearest integral value following normal mathematical - ## rounding rules (e.g. ``round(54.5) -> 55.0``). If ``places`` is + ## rounding rules (e.g. ``round(54.5) -> 55.0``). If ``places`` is ## greater than 0, round to the given number of decimal places, - ## e.g. ``round(54.346, 2) -> 54.350000000000001421...``. If ``places`` is negative, round - ## to the left of the decimal place, e.g. ``round(537.345, -1) -> + ## e.g. ``round(54.346, 2) -> 54.350000000000001421…``. If ``places`` is negative, round + ## to the left of the decimal place, e.g. ``round(537.345, -1) -> ## 540.0`` + ## + ## .. code-block:: Nim + ## echo round(PI, 2) ## 3.14 + ## echo round(PI, 4) ## 3.1416 if places == 0: result = round(x) else: @@ -538,9 +802,14 @@ proc round*[T: float32|float64](x: T, places: int): T {.deprecated: "use format proc floorDiv*[T: SomeInteger](x, y: T): T = ## Floor division is conceptually defined as ``floor(x / y)``. - ## This is different from the ``div`` operator, which is defined - ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` - ## rounds down. + ## + ## This is different from the `system.div `_ + ## operator, which is defined as ``trunc(x / y)``. + ## That is, ``div`` rounds towards ``0`` and ``floorDiv`` rounds down. + ## + ## See also: + ## * `system.div proc `_ for integer division + ## * `floorMod proc <#floorMod,T,T>`_ for Python-like (% operator) behavior ## ## .. code-block:: nim ## echo floorDiv( 13, 3) # 4 @@ -553,8 +822,13 @@ proc floorDiv*[T: SomeInteger](x, y: T): T = proc floorMod*[T: SomeNumber](x, y: T): T = ## Floor modulus is conceptually defined as ``x - (floorDiv(x, y) * y)``. + ## ## This proc behaves the same as the ``%`` operator in Python. ## + ## See also: + ## * `mod proc <#mod,float64,float64>`_ + ## * `floorDiv proc <#floorDiv,T,T>`_ + ## ## .. code-block:: nim ## echo floorMod( 13, 3) # 1 ## echo floorMod(-13, 3) # 2 @@ -570,6 +844,7 @@ when not defined(JS): importc: "frexp", header: "".} proc frexp*[T, U](x: T, exponent: var U): T = ## Split a number into mantissa and exponent. + ## ## ``frexp`` calculates the mantissa m (a float greater than or equal to 0.5 ## and less than 1) and the integer value n such that ``x`` (the original ## float value) equals ``m * 2**n``. frexp stores n in `exponent` and returns @@ -602,7 +877,19 @@ when not defined(JS): else: proc log2*(x: float32): float32 {.importc: "log2f", header: "".} proc log2*(x: float64): float64 {.importc: "log2", header: "".} - ## Computes the binary logarithm (base 2) of ``x`` + ## Computes the binary logarithm (base 2) of ``x``. + ## + ## See also: + ## * `log proc <#log,T,T>`_ + ## * `log10 proc <#log10,float64>`_ + ## * `ln proc <#ln,float64>`_ + ## * `exp proc <#exp,float64>`_ + ## + ## .. code-block:: Nim + ## echo log2(8.0) # 3.0 + ## echo log2(1.0) # 0.0 + ## echo log2(0.0) # -inf + ## echo log2(-2.0) # nan else: proc frexp*[T: float32|float64](x: T, exponent: var int): T = @@ -631,7 +918,8 @@ proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = ## function in C. ## ## .. code-block:: nim - ## echo splitDecimal(5.25) # (intpart: 5.0, floatpart: 0.25) + ## echo splitDecimal(5.25) # (intpart: 5.0, floatpart: 0.25) + ## echo splitDecimal(-2.73) # (intpart: -2.0, floatpart: -0.73) var absolute: T absolute = abs(x) @@ -644,26 +932,36 @@ proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = {.pop.} proc degToRad*[T: float32|float64](d: T): T {.inline.} = - ## Convert from degrees to radians + ## Convert from degrees to radians. + ## + ## See also: + ## * `radToDeg proc <#radToDeg,T>`_ ## ## .. code-block:: nim ## echo degToRad(180.0) # 3.141592653589793 result = T(d) * RadPerDeg proc radToDeg*[T: float32|float64](d: T): T {.inline.} = - ## Convert from radians to degrees - + ## Convert from radians to degrees. + ## + ## See also: + ## * `degToRad proc <#degToRad,T>`_ + ## ## .. code-block:: nim ## echo degToRad(2 * PI) # 360.0 result = T(d) / RadPerDeg proc sgn*[T: SomeNumber](x: T): int {.inline.} = - ## Sign function. Returns -1 for negative numbers and ``NegInf``, 1 for - ## positive numbers and ``Inf``, and 0 for positive zero, negative zero and - ## ``NaN``. + ## Sign function. + ## + ## Returns: + ## * `-1` for negative numbers and ``NegInf``, + ## * `1` for positive numbers and ``Inf``, + ## * `0` for positive zero, negative zero and ``NaN`` ## ## .. code-block:: nim - ## echo sgn(-5) # 1 + ## echo sgn(5) # 1 + ## echo sgn(0) # 0 ## echo sgn(-4.1) # -1 ord(T(0) < x) - ord(x < T(0)) @@ -671,11 +969,20 @@ proc sgn*[T: SomeNumber](x: T): int {.inline.} = {.pop.} proc `^`*[T](x: T, y: Natural): T = - ## Computes ``x`` to the power ``y``. ``x`` must be non-negative, use - ## `pow <#pow,float,float>`_ for negative exponents. + ## Computes ``x`` to the power ``y``. + ## + ## Exponent ``y`` must be non-negative, use + ## `pow proc <#pow,float64,float64>`_ for negative exponents. + ## + ## See also: + ## * `pow proc <#pow,float64,float64>`_ for negative exponent or + ## floats + ## * `sqrt proc <#sqrt,float64>`_ + ## * `cbrt proc <#cbrt,float64>`_ ## ## .. code-block:: nim - ## echo 2 ^ 3 # 8 + ## echo 2^3 # 8 + ## echo -2^3 # -8 when compiles(y >= T(0)): assert y >= T(0) else: @@ -693,9 +1000,16 @@ proc `^`*[T](x: T, y: Natural): T = proc gcd*[T](x, y: T): T = ## Computes the greatest common (positive) divisor of ``x`` and ``y``. + ## ## Note that for floats, the result cannot always be interpreted as ## "greatest decimal `z` such that ``z*N == x and z*M == y`` ## where N and M are positive integers." + ## + ## See also: + ## * `gcd proc <#gcd,SomeInteger,SomeInteger>`_ for integer version + ## * `lcm proc <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(13.5, 9.0) == 4.5 var (x, y) = (x, y) while y != 0: x = x mod y @@ -703,11 +1017,15 @@ proc gcd*[T](x, y: T): T = abs x proc gcd*(x, y: SomeInteger): SomeInteger = - ## Computes the greatest common (positive) divisor of ``x`` and ``y``. - ## Using binary GCD (aka Stein's) algorithm. + ## Computes the greatest common (positive) divisor of ``x`` and ``y``, + ## using binary GCD (aka Stein's) algorithm. ## - ## .. code-block:: nim - ## echo gcd(24, 30) # 6 + ## See also: + ## * `gcd proc <#gcd,T,T>`_ for floats version + ## * `lcm proc <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(12, 8) == 4 + doAssert gcd(17, 63) == 1 when x is SomeSignedInt: var x = abs(x) else: @@ -734,10 +1052,15 @@ proc gcd*(x, y: SomeInteger): SomeInteger = proc lcm*[T](x, y: T): T = ## Computes the least common multiple of ``x`` and ``y``. ## - ## .. code-block:: nim - ## echo lcm(24, 30) # 120 + ## See also: + ## * `gcd proc <#gcd,T,T>`_ + runnableExamples: + doAssert lcm(24, 30) == 120 + doAssert lcm(13, 39) == 39 x div gcd(x, y) * y + + when isMainModule and not defined(JS) and not windowsCC89: # Check for no side effect annotation proc mySqrt(num: float): float {.noSideEffect.} = From b8454327c52faa82632cc90dd8fa326efbb38565 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jan 2019 01:16:14 -0800 Subject: [PATCH 139/350] json: support tuple (#10010) --- lib/pure/json.nim | 17 ++++++++++++++++- tests/stdlib/tjsonmacro.nim | 4 ++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 010dd8f706..02fa6dc678 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -142,7 +142,8 @@ runnableExamples: doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" import - hashes, tables, strutils, lexbase, streams, unicode, macros, parsejson + hashes, tables, strutils, lexbase, streams, unicode, macros, parsejson, + typetraits export tables.`$` @@ -356,6 +357,20 @@ when false: assert false notin elements, "usage error: only empty sets allowed" assert true notin elements, "usage error: only empty sets allowed" +#[ +Note: could use simply: +proc `%`*(o: object|tuple): JsonNode +but blocked by https://github.com/nim-lang/Nim/issues/10019 +]# +proc `%`*(o: tuple): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + when isNamedTuple(type(o)): + result = newJObject() + for k, v in o.fieldPairs: result[k] = %v + else: + result = newJArray() + for a in o.fields: result.add(%a) + proc `%`*(o: object): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` result = newJObject() diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 33332447b5..2e95b48336 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -516,3 +516,7 @@ when true: var w = u.to(MyDistRef) doAssert v.name == "smith" doAssert MyRef(w).name == "smith" + + block test_tuple: + doAssert $(%* (a1: 10, a2: "foo")) == """{"a1":10,"a2":"foo"}""" + doAssert $(%* (10, "foo")) == """[10,"foo"]""" From 3abe6e9e14e1da2890ae7b20f0f2bee247b3920b Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 16 Jan 2019 11:58:51 +0100 Subject: [PATCH 140/350] quickfix for sequtils docs [ci skip] --- lib/pure/collections/sequtils.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 23ac0902a6..b562e2ac6e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -136,12 +136,9 @@ proc count*[T](s: openArray[T], x: T): int = let a = @[1, 2, 2, 3, 2, 4, 2] b = "abracadabra" - c = count(a, 2) - d = count(a, 99) - e = count(b, 'r') - assert c == 4 - assert d == 0 - assert e == 2 + assert count(a, 2) == 4 + assert count(a, 99) == 0 + assert count(b, 'r') == 2 for itm in items(s): if itm == x: From 2e8bf8861417d742a1f1fe7e695a2c10a039d5d6 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 16 Jan 2019 12:32:00 +0100 Subject: [PATCH 141/350] sequtils doc: another quickfix [ci skip] --- lib/pure/collections/sequtils.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index b562e2ac6e..f73d37f47f 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -301,7 +301,7 @@ proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): ## ## See also: ## * `mapIt template<#mapIt.t,typed,untyped>`_ - ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ for the in-ace version + ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ for the in-place version ## runnableExamples: let @@ -380,7 +380,7 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] ## See also: ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ - ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ for the in-ace version + ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ for the in-place version ## runnableExamples: let @@ -811,7 +811,7 @@ template mapIt*(s: typed, op: untyped): untyped = ## ## See also: ## * `map proc<#map,openArray[T],proc(T)>`_ - ## * `applyIt template<#applyIt.t,untyped,untyped>`_ for the in-ace version + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ for the in-place version ## runnableExamples: let From 1cb5e20620690044d158e419855582bc116003ba Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 16 Jan 2019 10:05:33 +0100 Subject: [PATCH 142/350] parseutils.nim: help the codegen produce better code --- lib/pure/parseutils.nim | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 36fcb19652..ee8453b216 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -95,7 +95,7 @@ proc parseOct*(s: string, number: var int, start = 0, maxLen = 0): int {. inc(i) if foundDigit: result = i-start -proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {. +proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {. rtl, extern: "npuParseBin", noSideEffect.} = ## Parses an binary number and stores its value in ``number``. Returns ## the number of the parsed characters or 0 in case of an error. @@ -242,8 +242,8 @@ proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = result = "" discard s.parseUntil(result, if second == '\0': first else: second, i) -template integerOutOfRangeError(): ref ValueError = - newException(ValueError, "Parsed integer outside of valid range") +proc integerOutOfRangeError() {.noinline.} = + raise newException(ValueError, "Parsed integer outside of valid range") proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = var @@ -261,13 +261,14 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = if b >= (low(int) + c) div 10: b = b * 10 - c else: - raise integerOutOfRangeError() + integerOutOfRangeError() inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored if sign == -1 and b == low(int): - raise integerOutOfRangeError() - b = b * sign - result = i - start + integerOutOfRangeError() + else: + b = b * sign + result = i - start proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = @@ -290,7 +291,7 @@ proc parseInt*(s: string, number: var int, start = 0): int {. result = parseBiggestInt(s, res, start) when sizeof(int) <= 4: if res < low(int) or res > high(int): - raise integerOutOfRangeError() + integerOutOfRangeError() if result != 0: number = int(res) @@ -326,7 +327,7 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = prev = 0.BiggestUInt i = start if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: - raise integerOutOfRangeError() + integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -334,7 +335,7 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = prev = res res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt if prev > res: - raise integerOutOfRangeError() + integerOutOfRangeError() inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res @@ -361,7 +362,7 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. result = parseBiggestUInt(s, res, start) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: - raise integerOutOfRangeError() + integerOutOfRangeError() if result != 0: number = uint(res) From fc30cf02bca70341846d07e02b9af0841ba9f985 Mon Sep 17 00:00:00 2001 From: alaviss Date: Thu, 17 Jan 2019 03:06:32 +0700 Subject: [PATCH 143/350] nimsuggest: add an option to bind to a free port (#10328) --- nimsuggest/nimsuggest.nim | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c24a8bafaf..1b5e8326d7 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -35,6 +35,7 @@ Usage: nimsuggest [options] projectfile.nim Options: + --autobind automatically binds into a free port --port:PORT port, by default 6000 --address:HOST binds to that address, by default "" --stdin read commands from stdin and write results to @@ -50,6 +51,8 @@ Options: The server then listens to the connection and takes line-based commands. +If --autobind is used, the binded port number will be printed to stdout. + In addition, all command line options of Nim that do not affect code generation are supported. """ @@ -68,6 +71,7 @@ var gEmitEof: bool # whether we write '!EOF!' dummy lines gLogging = defined(logging) gRefresh: bool + gAutoBind = false requests: Channel[string] results: Channel[Suggest] @@ -306,9 +310,15 @@ proc replCmdline(x: ThreadParams) {.thread.} = proc replTcp(x: ThreadParams) {.thread.} = var server = newSocket() - server.bindAddr(x.port, x.address) + if gAutoBind: + let port = server.connectToNextFreePort(x.address) + server.listen() + echo port + stdout.flushFile() + else: + server.bindAddr(x.port, x.address) + server.listen() var inp = "".TaintedString - server.listen() while true: var stdoutSocket = newSocket() accept(server, stdoutSocket) @@ -544,6 +554,9 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = of "help", "h": stdout.writeline(Usage) quit() + of "autobind": + gMode = mtcp + gAutoBind = true of "port": gPort = parseInt(p.val).Port gMode = mtcp From 5df411bd6e51e801dd06fb6b8b6f1804f8dcf9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Wed, 16 Jan 2019 21:08:32 +0100 Subject: [PATCH 144/350] Revert export of times.CTime; add std/time_t instead. (#10319) * Revert export of times.CTime * Add std/time_t --- lib/pure/times.nim | 11 ++++------- lib/std/time_t.nim | 23 +++++++++++++++++++++++ tools/kochdocs.nim | 1 + 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 lib/std/time_t.nim diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 166dfd446c..0104a97c16 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -217,7 +217,7 @@ when defined(JS): elif defined(posix): import posix - type CTime* = posix.Time + type CTime = posix.Time var realTimeClockId {.importc: "CLOCK_REALTIME", header: "".}: Clockid @@ -233,13 +233,10 @@ elif defined(posix): tzset() elif defined(windows): - import winlean + import winlean, std/time_t + + type CTime = time_t.Time - when defined(i386) and defined(gcc): - type CTime* {.importc: "time_t", header: "".} = distinct int32 - else: - # newest version of Visual C++ defines time_t to be of 64 bits - type CTime* {.importc: "time_t", header: "".} = distinct int64 # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "".}: int diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim new file mode 100644 index 0000000000..37918ee6ca --- /dev/null +++ b/lib/std/time_t.nim @@ -0,0 +1,23 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimdoc): + type + impl = distinct int64 + Time* = impl ## \ + ## Wrapper for ``time_t``. On posix, this is an alias to ``posix.Time``. +elif defined(windows): + when defined(i386) and defined(gcc): + type Time* {.importc: "time_t", header: "".} = distinct int32 + else: + # newest version of Visual C++ defines time_t to be of 64 bits + type Time* {.importc: "time_t", header: "".} = distinct int64 +elif defined(posix): + import posix + export posix.Time \ No newline at end of file diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 68662bab24..f75994f510 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -219,6 +219,7 @@ lib/pure/collections/heapqueue.nim lib/pure/fenv.nim lib/std/sha1.nim lib/std/varints.nim +lib/std/time_t.nim lib/impure/rdstdin.nim lib/wrappers/linenoise/linenoise.nim lib/pure/strformat.nim From 8947779dd033966202be58f105dd307c44f143c8 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 16 Jan 2019 21:11:21 +0100 Subject: [PATCH 145/350] disable one more test for C++ --- tests/objects/tobjcov.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/objects/tobjcov.nim b/tests/objects/tobjcov.nim index 817c1fcda0..6c587e04dd 100644 --- a/tests/objects/tobjcov.nim +++ b/tests/objects/tobjcov.nim @@ -1,8 +1,12 @@ discard """ action: compile +target: "c" """ # Covariance is not type safe: +# Note: `nim cpp` makes it a compile error (after codegen), even with: +# `var f = cast[proc (x: var TA) {.nimcall.}](cast[pointer](bp))`, which +# currently removes all the `cast` in cgen'd code, hence the compile error. type TA = object of RootObj From 2039dad2736909a9d2091b137eeab4293b508e12 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 16 Jan 2019 21:19:38 +0100 Subject: [PATCH 146/350] koch.nim: Make bootstrapping in C++ mode robust --- koch.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 1d636914d3..be8f759869 100644 --- a/koch.nim +++ b/koch.nim @@ -277,14 +277,16 @@ proc boot(args: string) = var output = "compiler" / "nim".exe var finalDest = "bin" / "nim".exe # default to use the 'c' command: - let defaultCommand = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": "cpp" else: "c" - let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" - echo "boot: defaultCommand: ", defaultCommand, " bootOptions: ", bootOptions + let useCpp = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & hostOs & "_" & hostCpu copyExe(findStartNim(), 0.thVersion) - for i in 0..2: + for i in 0..2+ord(useCpp): + # do the first iteration in C mode in order to avoid problem #10315: + let defaultCommand = if useCpp and i > 0: "cpp" else: "c" + let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" + echo "iteration: ", i+1 let extraOption = if i == 0: "--skipUserCfg --skipParentCfg" From 52a54f5f042f2849b50c244332e3169fdc03e194 Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 17 Jan 2019 07:15:40 +0100 Subject: [PATCH 147/350] documentation: remove author field [ci skip] --- lib/pure/collections/sequtils.nim | 2 -- lib/pure/collections/tables.nim | 2 -- lib/pure/math.nim | 3 --- lib/pure/strutils.nim | 2 -- 4 files changed, 9 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index f73d37f47f..b0d50bce26 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -## :Author: Alexander Mitchell-Robinson (Amrykid) and Nim contributors -## ## Although this module has ``seq`` in its name, it implements operations ## not only for `seq`:idx: type, but for three built-in container types under ## the ``openArray`` umbrella: diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 659b4473b3..84ec422d45 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -## :Author: Nim contributors -## ## The ``tables`` module implements variants of an efficient `hash table`:idx: ## (also often named `dictionary`:idx: in other programming languages) that is ## a mapping from keys to values. diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 97d1e7987a..460be1cd0e 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -## :Author: Nim contributors -## ## *Constructive mathematics is naturally typed.* -- Simon Thompson ## ## Basic math routines for Nim. @@ -237,7 +235,6 @@ proc cumsummed*[T](x: openArray[T]): seq[T] = ## See also: ## * `sum proc <#sum,openArray[T]>`_ ## * `cumsum proc <#cumsum,openArray[T]>`_ for the in-place version - ## * `cumsummed proc <#cumsummed,openArray[T]>`_ runnableExamples: let a = [1, 2, 3, 4] doAssert cumsummed(a) == @[1, 3, 6, 10] diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 4bb6400943..8385eb24ef 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -## :Author: Nim contributors -## ## The system module defines several common functions for working with strings, ## such as: ## * ``$`` for converting other data-types to strings From 15584879b91e14565156ca140eef1dc100cf34c4 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Thu, 17 Jan 2019 07:55:29 +0100 Subject: [PATCH 148/350] Properly wrap discarded statements (#10322) Failing to do so lead the codegen to emit invalid code sometimes, especially when C++ references were involved. Fixes #10241 --- compiler/semexprs.nim | 4 ++-- compiler/semstmts.nim | 24 ++++++++++++++---------- tests/cpp/t10241.nim | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 tests/cpp/t10241.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 657df36ddc..e74d56a863 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -917,7 +917,7 @@ proc semExprNoType(c: PContext, n: PNode): PNode = let isPush = hintExtendedContext in c.config.notes if isPush: pushInfoContext(c.config, n.info) result = semExpr(c, n, {efWantStmt}) - discardCheck(c, result, {}) + result = discardCheck(c, result, {}) if isPush: popInfoContext(c.config) proc isTypeExpr(n: PNode): bool = @@ -1639,7 +1639,7 @@ proc semProcBody(c: PContext, n: PNode): PNode = a.sons[1] = result result = semAsgn(c, a) else: - discardCheck(c, result, {}) + result = discardCheck(c, result, {}) if c.p.owner.kind notin {skMacro, skTemplate} and c.p.resultSym != nil and c.p.resultSym.typ.isMetaType: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3fdbb85dbc..70a16b290c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -130,13 +130,13 @@ proc fixNilType(c: PContext; n: PNode) = for it in n: fixNilType(c, it) n.typ = nil -proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) = +proc discardCheck(c: PContext, expr: PNode, flags: TExprFlags): PNode = + result = expr if c.matchedConcept != nil or efInTypeof in flags: return if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}: if implicitlyDiscardable(result): - var n = newNodeI(nkDiscardStmt, result.info, 1) - n[0] = result + result = newNode(nkDiscardStmt, result.info, @[result]) elif result.typ.kind != tyError and c.config.cmd != cmdInteractive: var n = result while n.kind in skipForDiscardable: n = n.lastSon @@ -168,7 +168,8 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode = else: illFormedAst(it, c.config) if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or (not hasElse and efInTypeof notin flags): - for it in n: discardCheck(c, it.lastSon, flags) + for it in n: + it.sons[^1] = discardCheck(c, it.sons[^1], flags) result.kind = nkIfStmt # propagate any enforced VoidContext: if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext @@ -266,12 +267,14 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode = dec c.p.inTryStmt if isEmptyType(typ) or typ.kind in {tyNil, tyExpr}: - discardCheck(c, n.sons[0], flags) - for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags) + n.sons[0] = discardCheck(c, n.sons[0], flags) + for i in 1..n.len-1: + n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags) if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext else: - if n.lastSon.kind == nkFinally: discardCheck(c, n.lastSon.lastSon, flags) + if n.lastSon.kind == nkFinally: + n.sons[^1].sons[^1] = discardCheck(c, n.sons[^1].sons[^1], flags) n.sons[0] = fitNode(c, typ, n.sons[0], n.sons[0].info) for i in 1..last: var it = n.sons[i] @@ -679,7 +682,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode = openScope(c) n.sons[length-1] = semExprBranch(c, n.sons[length-1], flags) if efInTypeof notin flags: - discardCheck(c, n.sons[length-1], flags) + n.sons[^1] = discardCheck(c, n.sons[^1], flags) closeScope(c) dec(c.p.nestedLoopCounter) @@ -866,7 +869,8 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode = closeScope(c) if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or (not hasElse and efInTypeof notin flags): - for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags) + for i in 1..n.len-1: + n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags) # propagate any enforced VoidContext: if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext @@ -2029,7 +2033,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = n.typ = n.sons[i].typ if not isEmptyType(n.typ): n.kind = nkStmtListExpr elif i != last or voidContext: - discardCheck(c, n.sons[i], flags) + n.sons[i] = discardCheck(c, n.sons[i], flags) else: n.typ = n.sons[i].typ if not isEmptyType(n.typ): n.kind = nkStmtListExpr diff --git a/tests/cpp/t10241.nim b/tests/cpp/t10241.nim new file mode 100644 index 0000000000..21d6a0f4e4 --- /dev/null +++ b/tests/cpp/t10241.nim @@ -0,0 +1,19 @@ +discard """ + targets: "cpp" + action: "compile" +""" + +type + String* {.importcpp: "std::string", header: "string".} = object + +proc initString*(): String + {.importcpp: "std::string()", header: "string".} + +proc append*(this: var String, str: String): var String + {.importcpp: "append", header: "string", discardable.} + +var + s1 = initString() + s2 = initString() + +s1.append s2 From 42bac52426bf392fcfa3cfeeee716bec3166b709 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jan 2019 23:00:44 -0800 Subject: [PATCH 149/350] [CI] now enables `NIM_COMPILE_TO_CPP=true` to run without allow_failures (#10315) * better fix for `nim cpp` bootstrap error: error: no member named raise_id * [CI] now enables runs NIM_COMPILE_TO_CPP=true without allow_failures * workaround refs #10343 --- .travis.yml | 5 +++-- koch.nim | 21 +++++++++++++-------- lib/system.nim | 7 ++++++- tests/exception/t9657.nim | 2 ++ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 32a40bcaf0..ea5d54eadc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,9 @@ matrix: - os: osx env: NIM_COMPILE_TO_CPP=true - allow_failures: - - env: NIM_COMPILE_TO_CPP=true +# To allow failures for a failing configuration, use something like: +# allow_failures: +# - env: NIM_COMPILE_TO_CPP=true # - os: osx addons: diff --git a/koch.nim b/koch.nim index be8f759869..f70cf21425 100644 --- a/koch.nim +++ b/koch.nim @@ -281,19 +281,24 @@ proc boot(args: string) = let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & hostOs & "_" & hostCpu - copyExe(findStartNim(), 0.thVersion) - for i in 0..2+ord(useCpp): - # do the first iteration in C mode in order to avoid problem #10315: - let defaultCommand = if useCpp and i > 0: "cpp" else: "c" + let nimStart = findStartNim() + copyExe(nimStart, 0.thVersion) + for i in 0..2: + let defaultCommand = if useCpp: "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" - echo "iteration: ", i+1 - let extraOption = if i == 0: - "--skipUserCfg --skipParentCfg" + var extraOption = "" + if i == 0: + extraOption.add " --skipUserCfg --skipParentCfg" # Note(D20190115T162028:here): the configs are skipped for bootstrap # (1st iteration) to prevent newer flags from breaking bootstrap phase. # fixes #10030. - else: "" + let ret = execCmdEx(nimStart & " --version") + doAssert ret.exitCode == 0 + let version = ret.output.splitLines[0] + if version.startsWith "Nim Compiler Version 0.19.0": + extraOption.add " -d:nimBoostrapCsources0_19_0" + # remove this when csources get updated exec i.thVersion & " $# $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, extraOption, args, smartNimcache] if sameFileContent(output, i.thVersion): diff --git a/lib/system.nim b/lib/system.nim index fb52ee9eb2..b479b920a3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -569,7 +569,12 @@ type trace: string else: trace: seq[StackTraceEntry] - raiseId: uint # set when exception is raised + when defined(nimBoostrapCsources0_19_0): + # see #10315, bootstrap with `nim cpp` from csources gave error: + # error: no member named 'raise_id' in 'Exception' + raise_id: uint # set when exception is raised + else: + raiseId: uint # set when exception is raised up: ref Exception # used for stacking exceptions. Not exported! Defect* = object of Exception ## \ diff --git a/tests/exception/t9657.nim b/tests/exception/t9657.nim index 5d5164f4f0..c96a0a5976 100644 --- a/tests/exception/t9657.nim +++ b/tests/exception/t9657.nim @@ -1,6 +1,8 @@ discard """ action: run exitcode: 1 + target: "c" """ +# todo: remove `target: "c"` workaround once #10343 is properly fixed close stdmsg writeLine stdmsg, "exception!" From cceb28b5eb9bae4f0fc364899858018a1bf480ae Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 17 Jan 2019 20:37:39 +0100 Subject: [PATCH 150/350] remove queues (#10347) This module was deprecated and superseded by deques 2 years ago. --- lib/pure/collections/queues.nim | 257 -------------------------------- tools/kochdocs.nim | 1 - 2 files changed, 258 deletions(-) delete mode 100644 lib/pure/collections/queues.nim diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim deleted file mode 100644 index 9a1d169fbf..0000000000 --- a/lib/pure/collections/queues.nim +++ /dev/null @@ -1,257 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. -## -## None of the procs that get an individual value from the queue can be used -## on an empty queue. -## If compiled with `boundChecks` option, those procs will raise an `IndexError` -## on such access. This should not be relied upon, as `-d:release` will -## disable those checks and may return garbage or crash the program. -## -## As such, a check to see if the queue is empty is needed before any -## access, unless your program logic guarantees it indirectly. -## -## .. code-block:: Nim -## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` -## var q = initQueue[int]() # initializes the object -## for i in 1 ..< a: q.add i # populates the queue -## -## if b < q.len: # checking before indexed access -## echo "The element at index position ", b, " is ", q[b] -## -## # The following two lines don't need any checking on access due to the -## # logic of the program, but that would not be the case if `a` could be 0. -## assert q.front == 1 -## assert q.back == a -## -## while q.len > 0: # checking if the queue is empty -## echo q.pop() -## -## Note: For inter thread communication use -## a `Channel `_ instead. - -import math - -{.warning: "`queues` module is deprecated - use `deques` instead".} - -type - Queue* {.deprecated.} [T] = object ## A queue. - data: seq[T] - rd, wr, count, mask: int - -proc initQueue*[T](initialSize: int = 4): Queue[T] = - ## Create a new queue. - ## Optionally, the initial capacity can be reserved via `initialSize` as a - ## performance optimization. The length of a newly created queue will still - ## be 0. - ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module. - assert isPowerOfTwo(initialSize) - result.mask = initialSize-1 - newSeq(result.data, initialSize) - -proc len*[T](q: Queue[T]): int {.inline.}= - ## Return the number of elements of `q`. - result = q.count - -template emptyCheck(q) = - # Bounds check for the regular queue access. - when compileOption("boundChecks"): - if unlikely(q.count < 1): - raise newException(IndexError, "Empty queue.") - -template xBoundsCheck(q, i) = - # Bounds check for the array like accesses. - when compileOption("boundChecks"): # d:release should disable this. - if unlikely(i >= q.count): # x < q.low is taken care by the Natural parameter - raise newException(IndexError, - "Out of bounds: " & $i & " > " & $(q.count - 1)) - -proc front*[T](q: Queue[T]): T {.inline.}= - ## Return the oldest element of `q`. Equivalent to `q.pop()` but does not - ## remove it from the queue. - emptyCheck(q) - result = q.data[q.rd] - -proc back*[T](q: Queue[T]): T {.inline.} = - ## Return the newest element of `q` but does not remove it from the queue. - emptyCheck(q) - result = q.data[q.wr - 1 and q.mask] - -proc `[]`*[T](q: Queue[T], i: Natural) : T {.inline.} = - ## Access the i-th element of `q` by order of insertion. - ## q[0] is the oldest (the next one q.pop() will extract), - ## q[^1] is the newest (last one added to the queue). - xBoundsCheck(q, i) - return q.data[q.rd + i and q.mask] - -proc `[]`*[T](q: var Queue[T], i: Natural): var T {.inline.} = - ## Access the i-th element of `q` and returns a mutable - ## reference to it. - xBoundsCheck(q, i) - return q.data[q.rd + i and q.mask] - -proc `[]=`* [T] (q: var Queue[T], i: Natural, val : T) {.inline.} = - ## Change the i-th element of `q`. - xBoundsCheck(q, i) - q.data[q.rd + i and q.mask] = val - -iterator items*[T](q: Queue[T]): T = - ## Yield every element of `q`. - var i = q.rd - for c in 0 ..< q.count: - yield q.data[i] - i = (i + 1) and q.mask - -iterator mitems*[T](q: var Queue[T]): var T = - ## Yield every element of `q`. - var i = q.rd - for c in 0 ..< q.count: - yield q.data[i] - i = (i + 1) and q.mask - -iterator pairs*[T](q: Queue[T]): tuple[key: int, val: T] = - ## Yield every (position, value) of `q`. - var i = q.rd - for c in 0 ..< q.count: - yield (c, q.data[i]) - i = (i + 1) and q.mask - -proc contains*[T](q: Queue[T], item: T): bool {.inline.} = - ## Return true if `item` is in `q` or false if not found. Usually used - ## via the ``in`` operator. It is the equivalent of ``q.find(item) >= 0``. - ## - ## .. code-block:: Nim - ## if x in q: - ## assert q.contains x - for e in q: - if e == item: return true - return false - -proc add*[T](q: var Queue[T], item: T) = - ## Add an `item` to the end of the queue `q`. - var cap = q.mask+1 - if unlikely(q.count >= cap): - var n = newSeq[T](cap*2) - for i, x in pairs(q): # don't use copyMem because the GC and because it's slower. - shallowCopy(n[i], x) - shallowCopy(q.data, n) - q.mask = cap*2 - 1 - q.wr = q.count - q.rd = 0 - inc q.count - q.data[q.wr] = item - q.wr = (q.wr + 1) and q.mask - -template default[T](t: typedesc[T]): T = - var v: T - v - -proc pop*[T](q: var Queue[T]): T {.inline, discardable.} = - ## Remove and returns the first (oldest) element of the queue `q`. - emptyCheck(q) - dec q.count - result = q.data[q.rd] - q.data[q.rd] = default(type(result)) - q.rd = (q.rd + 1) and q.mask - -proc enqueue*[T](q: var Queue[T], item: T) = - ## Alias for the ``add`` operation. - q.add(item) - -proc dequeue*[T](q: var Queue[T]): T = - ## Alias for the ``pop`` operation. - q.pop() - -proc `$`*[T](q: Queue[T]): string = - ## Turn a queue into its string representation. - result = "[" - for x in items(q): # Don't remove the items here for reasons that don't fit in this margin. - if result.len > 1: result.add(", ") - result.add($x) - result.add("]") - -when isMainModule: - var q = initQueue[int](1) - q.add(123) - q.add(9) - q.enqueue(4) - var first = q.dequeue() - q.add(56) - q.add(6) - var second = q.pop() - q.add(789) - - assert first == 123 - assert second == 9 - assert($q == "[4, 56, 6, 789]") - - assert q[0] == q.front and q.front == 4 - q[0] = 42 - q[q.len - 1] = 7 - - assert 6 in q and 789 notin q - assert q.find(6) >= 0 - assert q.find(789) < 0 - - for i in -2 .. 10: - if i in q: - assert q.contains(i) and q.find(i) >= 0 - else: - assert(not q.contains(i) and q.find(i) < 0) - - when compileOption("boundChecks"): - try: - echo q[99] - assert false - except IndexError: - discard - - try: - assert q.len == 4 - for i in 0 ..< 5: q.pop() - assert false - except IndexError: - discard - - # grabs some types of resize error. - q = initQueue[int]() - for i in 1 .. 4: q.add i - q.pop() - q.pop() - for i in 5 .. 8: q.add i - assert $q == "[3, 4, 5, 6, 7, 8]" - - # Similar to proc from the documentation example - proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. - var q = initQueue[int]() - assert q.len == 0 - for i in 1 .. a: q.add i - - if b < q.len: # checking before indexed access. - assert q[b] == b + 1 - - # The following two lines don't need any checking on access due to the logic - # of the program, but that would not be the case if `a` could be 0. - assert q.front == 1 - assert q.back == a - - while q.len > 0: # checking if the queue is empty - assert q.pop() > 0 - - #foo(0,0) - foo(8,5) - foo(10,9) - foo(1,1) - foo(2,1) - foo(1,5) - foo(3,2) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index f75994f510..376f33ae50 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -177,7 +177,6 @@ lib/pure/collections/lists.nim lib/pure/collections/sharedlist.nim lib/pure/collections/sharedtables.nim lib/pure/collections/intsets.nim -lib/pure/collections/queues.nim lib/pure/collections/deques.nim lib/pure/encodings.nim lib/pure/collections/sequtils.nim From 214f48eae9b6a02d5ba68ddf0b1e6b9a26bddacb Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 18 Jan 2019 07:18:32 +0100 Subject: [PATCH 151/350] Remove long deprecated stuff (#10332) --- lib/impure/re.nim | 27 +--- lib/js/dom.nim | 30 +--- lib/pure/cgi.nim | 5 - lib/pure/collections/critbits.nim | 6 - lib/pure/collections/intsets.nim | 6 - lib/pure/collections/sets.nim | 7 - lib/pure/httpclient.nim | 243 ------------------------------ lib/pure/nativesockets.nim | 63 -------- lib/pure/net.nim | 39 +---- lib/pure/osproc.nim | 16 -- lib/pure/parseopt2.nim | 9 -- lib/pure/parseutils.nim | 12 -- lib/pure/uri.nim | 27 ---- tests/async/twinasyncrw.nim | 2 +- tests/manyloc/nake/nakefile.nim | 5 +- 15 files changed, 9 insertions(+), 488 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index dc4ee326f8..42be4a3c29 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -561,31 +561,6 @@ proc escapeRe*(s: string): string = result.add("\\x") result.add(toHex(ord(c), 2)) -const ## common regular expressions - reIdentifier* {.deprecated.} = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b" - ## describes an identifier - reNatural* {.deprecated.} = r"\b\d+\b" - ## describes a natural number - reInteger* {.deprecated.} = r"\b[-+]?\d+\b" - ## describes an integer - reHex* {.deprecated.} = r"\b0[xX][0-9a-fA-F]+\b" - ## describes a hexadecimal number - reBinary* {.deprecated.} = r"\b0[bB][01]+\b" - ## describes a binary number (example: 0b11101) - reOctal* {.deprecated.} = r"\b0[oO][0-7]+\b" - ## describes an octal number (example: 0o777) - reFloat* {.deprecated.} = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b" - ## describes a floating point number - reEmail* {.deprecated.} = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" & - r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@" & - r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" & - r"(?:[a-zA-Z]{2}|com|org|net|gov|mil|biz|" & - r"info|mobi|name|aero|jobs|museum)\b" - ## describes a common email address - reURL* {.deprecated.} = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms-help)" & - r":((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b" - ## describes an URL - when isMainModule: doAssert match("(a b c)", rex"\( .* \)") doAssert match("WHiLe", re("while", {reIgnoreCase})) @@ -595,7 +570,7 @@ when isMainModule: doAssert "ABC".match(rex"\d+ | \w+") {.push warnings:off.} - doAssert matchLen("key", re(reIdentifier)) == 3 + doAssert matchLen("key", re"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") == 3 {.pop.} var pattern = re"[a-z0-9]+\s*=\s*[a-z0-9]+" diff --git a/lib/js/dom.nim b/lib/js/dom.nim index cf219df3de..668dee822e 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -416,12 +416,12 @@ type BoundingRect* {.importc.} = ref object top*, bottom*, left*, right*, x*, y*, width*, height*: float - PerformanceMemory* {.importc.} = ref object + PerformanceMemory* {.importc.} = ref object jsHeapSizeLimit*: float totalJSHeapSize*: float usedJSHeapSize*: float - PerformanceTiming* {.importc.} = ref object + PerformanceTiming* {.importc.} = ref object connectStart*: float domComplete*: float domContentLoadedEventEnd*: float @@ -459,7 +459,6 @@ proc dispatchEvent*(et: EventTarget, ev: Event) proc alert*(w: Window, msg: cstring) proc back*(w: Window) proc blur*(w: Window) -proc captureEvents*(w: Window, eventMask: int) {.deprecated.} proc clearInterval*(w: Window, interval: ref TInterval) proc clearTimeout*(w: Window, timeout: ref TTimeOut) proc close*(w: Window) @@ -478,7 +477,6 @@ proc open*(w: Window, uri, windowname: cstring, properties: cstring = nil): Window proc print*(w: Window) proc prompt*(w: Window, text, default: cstring): cstring -proc releaseEvents*(w: Window, eventMask: int) {.deprecated.} proc resizeBy*(w: Window, x, y: int) proc resizeTo*(w: Window, x, y: int) proc routeEvent*(w: Window, event: Event) @@ -513,7 +511,6 @@ proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) # Document "methods" -proc captureEvents*(d: Document, eventMask: int) {.deprecated.} proc createAttribute*(d: Document, identifier: cstring): Node proc createElement*(d: Document, identifier: cstring): Element proc createTextNode*(d: Document, identifier: cstring): Node @@ -524,7 +521,6 @@ proc getElementsByClassName*(d: Document, name: cstring): seq[Element] proc getSelection*(d: Document): cstring proc handleEvent*(d: Document, event: Event) proc open*(d: Document) -proc releaseEvents*(d: Document, eventMask: int) {.deprecated.} proc routeEvent*(d: Document, event: Event) proc write*(d: Document, text: cstring) proc writeln*(d: Document, text: cstring) @@ -605,25 +601,3 @@ proc parseInt*(s: cstring): int {.importc, nodecl.} proc parseInt*(s: cstring, radix: int):int {.importc, nodecl.} proc newEvent*(name: cstring): Event {.importcpp: "new Event(@)", constructor.} - -type - TEventHandlers* {.deprecated.} = EventTargetObj - TWindow* {.deprecated.} = WindowObj - TFrame* {.deprecated.} = FrameObj - TNode* {.deprecated.} = NodeObj - TDocument* {.deprecated.} = DocumentObj - TElement* {.deprecated.} = ElementObj - TLink* {.deprecated.} = LinkObj - TEmbed* {.deprecated.} = EmbedObj - TAnchor* {.deprecated.} = AnchorObj - TOption* {.deprecated.} = OptionObj - TForm* {.deprecated.} = FormObj - TImage* {.deprecated.} = ImageObj - TNodeType* {.deprecated.} = NodeType - TEvent* {.deprecated.} = EventObj - TLocation* {.deprecated.} = LocationObj - THistory* {.deprecated.} = HistoryObj - TNavigator* {.deprecated.} = NavigatorObj - TStyle* {.deprecated.} = StyleObj - TScreen* {.deprecated.} = ScreenObj - TApplet* {.importc, deprecated.} = object of RootObj diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 869abc9cc7..ec3562c356 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -337,11 +337,6 @@ proc setStackTraceStdout*() = ## Makes Nim output stacktraces to stdout, instead of server log. errorMessageWriter = writeErrorMessage -proc setStackTraceNewLine*() {.deprecated.} = - ## Makes Nim output stacktraces to stdout, instead of server log. - ## Depracated alias for setStackTraceStdout. - setStackTraceStdout() - proc setCookie*(name, value: string) = ## Sets a cookie. write(stdout, "Set-Cookie: ", name, "=", value, "\n") diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 1bd13920d4..dd91fdb127 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -202,12 +202,6 @@ proc `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline, ## If `key` is not in `t`, the ``KeyError`` exception is raised. get(c, key) -proc mget*[T](c: var CritBitTree[T], key: string): var T {.inline, deprecated.} = - ## retrieves the value at ``c[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ``[]`` instead. - get(c, key) - iterator leaves[T](n: Node[T]): Node[T] = if n != nil: # XXX actually we could compute the necessary stack size in advance: diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index f6d3a3d11e..9d7950ea69 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -357,12 +357,6 @@ proc `$`*(s: IntSet): string = ## The `$` operator for int sets. dollarImpl() -proc empty*(s: IntSet): bool {.inline, deprecated.} = - ## Returns true if `s` is empty. This is safe to call even before - ## the set has been initialized with `initIntSet`. Note this never - ## worked reliably and so is deprecated. - result = s.counter == 0 - when isMainModule: import sequtils, algorithm diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 07fcfe6763..d1f941e92c 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -191,13 +191,6 @@ proc `[]`*[A](s: var HashSet[A], key: A): var A = else: raise newException(KeyError, "key not found") -proc mget*[A](s: var HashSet[A], key: A): var A {.deprecated.} = - ## returns the element that is actually stored in 's' which has the same - ## value as 'key' or raises the ``KeyError`` exception. This is useful - ## when one overloaded 'hash' and '==' but still needs reference semantics - ## for sharing. Use ``[]`` instead. - s[key] - proc contains*[A](s: HashSet[A], key: A): bool = ## Returns true iff `key` is in `s`. ## diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index b7498b1c53..269ae476fe 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -189,12 +189,6 @@ proc body*(response: Response): string = response.body = response.bodyStream.readAll() return response.body -proc `body=`*(response: Response, value: string) {.deprecated.} = - ## Setter for backward compatibility. - ## - ## **This is deprecated and should not be used**. - response.body = value - proc body*(response: AsyncResponse): Future[string] {.async.} = ## Reads the response's body and caches it. The read is performed only ## once. @@ -477,119 +471,6 @@ proc format(p: MultipartData): tuple[contentType, body: string] = result.body.add("--" & bound & "\c\L" & s) result.body.add("--" & bound & "--\c\L") -proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = getDefaultSSL(), timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated: "use HttpClient.request instead".} = - ## | Requests ``url`` with the custom method string specified by the - ## | ``httpMethod`` parameter. - ## | Extra headers can be specified and must be separated by ``\c\L`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. - var r = if proxy == nil: parseUri(url) else: proxy.url - var hostUrl = if proxy == nil: r else: parseUri(url) - var headers = httpMethod.toUpperAscii() - # TODO: Use generateHeaders further down once it supports proxies. - - var s = newSocket() - defer: s.close() - if s == nil: raiseOSError(osLastError()) - var port = net.Port(80) - if r.scheme == "https": - when defined(ssl): - sslContext.wrapSocket(s) - port = net.Port(443) - else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") - if r.port != "": - port = net.Port(r.port.parseInt) - - - # get the socket ready. If we are connecting through a proxy to SSL, - # send the appropriate CONNECT header. If not, simply connect to the proper - # host (which may still be the proxy, for normal HTTP) - if proxy != nil and hostUrl.scheme == "https": - when defined(ssl): - var connectHeaders = "CONNECT " - let targetPort = if hostUrl.port == "": 443 else: hostUrl.port.parseInt - connectHeaders.add(hostUrl.hostname) - connectHeaders.add(":" & $targetPort) - connectHeaders.add(" HTTP/1.1\c\L") - connectHeaders.add("Host: " & hostUrl.hostname & ":" & $targetPort & "\c\L") - if proxy.auth != "": - let auth = base64.encode(proxy.auth, newline = "") - connectHeaders.add("Proxy-Authorization: basic " & auth & "\c\L") - connectHeaders.add("\c\L") - if timeout == -1: - s.connect(r.hostname, port) - else: - s.connect(r.hostname, port, timeout) - - s.send(connectHeaders) - let connectResult = parseResponse(s, false, timeout) - if not connectResult.status.startsWith("200"): - raise newException(HttpRequestError, - "The proxy server rejected a CONNECT request, " & - "so a secure connection could not be established.") - sslContext.wrapConnectedSocket(s, handshakeAsClient, hostUrl.hostname) - else: - raise newException(HttpRequestError, "SSL support not available. Cannot " & - "connect via proxy over SSL. Compile with -d:ssl to enable.") - else: - if timeout == -1: - s.connect(r.hostname, port) - else: - s.connect(r.hostname, port, timeout) - - - # now that the socket is ready, prepare the headers - if proxy == nil: - headers.add ' ' - if r.path[0] != '/': headers.add '/' - headers.add(r.path) - if r.query.len > 0: - headers.add("?" & r.query) - else: - headers.add(" " & url) - - headers.add(" HTTP/1.1\c\L") - - if hostUrl.port == "": - add(headers, "Host: " & hostUrl.hostname & "\c\L") - else: - add(headers, "Host: " & hostUrl.hostname & ":" & hostUrl.port & "\c\L") - - if userAgent != "": - add(headers, "User-Agent: " & userAgent & "\c\L") - if proxy != nil and proxy.auth != "": - let auth = base64.encode(proxy.auth, newline = "") - add(headers, "Proxy-Authorization: basic " & auth & "\c\L") - add(headers, extraHeaders) - add(headers, "\c\L") - - # headers are ready. send them, await the result, and close the socket. - s.send(headers) - if body != "": - s.send(body) - - result = parseResponse(s, httpMethod != "HEAD", timeout) - -proc request*(url: string, httpMethod = HttpGET, extraHeaders = "", - body = "", sslContext = getDefaultSSL(), timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated.} = - ## | Requests ``url`` with the specified ``httpMethod``. - ## | Extra headers can be specified and must be separated by ``\c\L`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. - result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, - userAgent, proxy) - proc redirection(status: string): bool = const redirectionNRs = ["301", "302", "303", "307"] for i in items(redirectionNRs): @@ -608,130 +489,6 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = parsed.anchor = r.anchor result = $parsed -proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = getDefaultSSL(), - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response {.deprecated.} = - ## | GETs the ``url`` and returns a ``Response`` object - ## | This proc also handles redirection - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. - result = request(url, HttpGET, extraHeaders, "", sslContext, timeout, - userAgent, proxy) - var lastURL = url - for i in 1..maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - result = request(redirectTo, HttpGET, extraHeaders, "", sslContext, - timeout, userAgent, proxy) - lastURL = redirectTo - -proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = getDefaultSSL(), - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string {.deprecated.} = - ## | GETs the body and returns it as a string. - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.getContent`` instead. - var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent, - proxy) - if r.status[0] in {'4','5'}: - raise newException(HttpRequestError, r.status) - else: - return r.body - -proc post*(url: string, extraHeaders = "", body = "", - maxRedirects = 5, - sslContext: SSLContext = getDefaultSSL(), - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil, - multipart: MultipartData = nil): Response {.deprecated.} = - ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. - ## | This proc adds the necessary Content-Length header. - ## | This proc also handles redirection. - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## | The optional ``multipart`` parameter can be used to create - ## ``multipart/form-data`` POSTs comfortably. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.post`` instead. - let (mpContentType, mpBody) = format(multipart) - - template withNewLine(x): untyped = - if x.len > 0 and not x.endsWith("\c\L"): - x & "\c\L" - else: - x - - var xb = mpBody.withNewLine() & body - - var xh = extraHeaders.withNewLine() & - withNewLine("Content-Length: " & $len(xb)) - - if not multipart.isNil: - xh.add(withNewLine("Content-Type: " & mpContentType)) - - result = request(url, HttpPOST, xh, xb, sslContext, timeout, userAgent, - proxy) - var lastURL = url - for i in 1..maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - var meth = if result.status != "307": HttpGet else: HttpPost - result = request(redirectTo, meth, xh, xb, sslContext, timeout, - userAgent, proxy) - lastURL = redirectTo - -proc postContent*(url: string, extraHeaders = "", body = "", - maxRedirects = 5, - sslContext: SSLContext = getDefaultSSL(), - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil, - multipart: MultipartData = nil): string - {.deprecated.} = - ## | POSTs ``body`` to ``url`` and returns the response's body as a string - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## | The optional ``multipart`` parameter can be used to create - ## ``multipart/form-data`` POSTs comfortably. - ## - ## **Deprecated since version 0.15.0**: use ``HttpClient.postContent`` - ## instead. - var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, - userAgent, proxy, multipart) - if r.status[0] in {'4','5'}: - raise newException(HttpRequestError, r.status) - else: - return r.body - -proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = getDefaultSSL(), - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil) {.deprecated.} = - ## | Downloads ``url`` and saves it to ``outputFilename`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.16.2**: use ``HttpClient.downloadFile`` - ## instead. - var f: File - if open(f, outputFilename, fmWrite): - f.write(getContent(url, sslContext = sslContext, timeout = timeout, - userAgent = userAgent, proxy = proxy)) - f.close() - else: - fileError("Unable to open file") - proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 96c377187e..6d4df1c5df 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -265,10 +265,6 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) -proc dealloc*(ai: ptr AddrInfo) {.deprecated.} = - ## Deprecated since 0.16.2. Use ``freeAddrInfo`` instead. - freeaddrinfo(ai) - proc ntohl*(x: uint32): uint32 = ## Converts 32-bit unsigned integers from network to host byte order. ## On machines where the host byte order is the same as network byte order, @@ -279,15 +275,6 @@ proc ntohl*(x: uint32): uint32 = (x shl 8'u32 and 0xff0000'u32) or (x shl 24'u32) -template ntohl*(x: int32): untyped {.deprecated.} = - ## Converts 32-bit integers from network to host byte order. - ## On machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 4-byte swap operation. - ## **Warning**: This template is deprecated since 0.14.0, IPv4 - ## addresses are now treated as unsigned integers. Please use the unsigned - ## version of this template. - cast[int32](nativesockets.ntohl(cast[uint32](x))) - proc ntohs*(x: uint16): uint16 = ## Converts 16-bit unsigned integers from network to host byte order. On ## machines where the host byte order is the same as network byte order, @@ -295,39 +282,12 @@ proc ntohs*(x: uint16): uint16 = when cpuEndian == bigEndian: result = x else: result = (x shr 8'u16) or (x shl 8'u16) -template ntohs*(x: int16): untyped {.deprecated.} = - ## Converts 16-bit integers from network to host byte order. On - ## machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 2-byte swap operation. - ## **Warning**: This template is deprecated since 0.14.0, where port - ## numbers became unsigned integers. Please use the unsigned version of - ## this template. - cast[int16](nativesockets.ntohs(cast[uint16](x))) - -template htonl*(x: int32): untyped {.deprecated.} = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. - ## **Warning**: This template is deprecated since 0.14.0, IPv4 - ## addresses are now treated as unsigned integers. Please use the unsigned - ## version of this template. - nativesockets.ntohl(x) - template htonl*(x: uint32): untyped = ## Converts 32-bit unsigned integers from host to network byte order. On ## machines where the host byte order is the same as network byte order, ## this is a no-op; otherwise, it performs a 4-byte swap operation. nativesockets.ntohl(x) -template htons*(x: int16): untyped {.deprecated.} = - ## Converts 16-bit integers from host to network byte order. - ## On machines where the host byte order is the same as network byte - ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. - ## **Warning**: This template is deprecated since 0.14.0, where port - ## numbers became unsigned integers. Please use the unsigned version of - ## this template. - nativesockets.ntohs(x) - template htons*(x: uint16): untyped = ## Converts 16-bit unsigned integers from host to network byte order. ## On machines where the host byte order is the same as network byte @@ -649,29 +609,6 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = inc(i) setLen(s, L) -proc select*(readfds: var seq[SocketHandle], timeout = 500): int {.deprecated: "use selectRead instead".} = - ## When a socket in ``readfds`` is ready to be read from then a non-zero - ## value will be returned specifying the count of the sockets which can be - ## read from. The sockets which can be read from will also be removed - ## from ``readfds``. - ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for - ## an unlimited time. - ## **Warning:** This is deprecated since version 0.16.2. - ## Use the ``selectRead`` procedure instead. - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), nil, nil, nil)) - - pruneSocketSet(readfds, (rd)) - proc selectRead*(readfds: var seq[SocketHandle], timeout = 500): int = ## When a socket in ``readfds`` is ready to be read from then a non-zero ## value will be returned specifying the count of the sockets which can be diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 840a81f171..d4c5a88b7f 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -967,39 +967,6 @@ when defined(posix) or defined(nimdoc): raiseOSError(osLastError()) when defined(ssl): - proc handshake*(socket: Socket): bool - {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = - ## This proc needs to be called on a socket after it connects. This is - ## only applicable when using ``connectAsync``. - ## This proc performs the SSL handshake. - ## - ## Returns ``False`` whenever the socket is not yet ready for a handshake, - ## ``True`` whenever handshake completed successfully. - ## - ## A SslError error is raised on any other errors. - ## - ## **Note:** This procedure is deprecated since version 0.14.0. - result = true - if socket.isSSL: - var ret = SSLConnect(socket.sslHandle) - if ret <= 0: - var errret = SSLGetError(socket.sslHandle, ret) - case errret - of SSL_ERROR_ZERO_RETURN: - raiseSSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT, - SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSSLError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSSLError() - else: - raiseSSLError("Unknown Error") - socket.sslNoHandshake = false - else: - raiseSSLError("Socket is not an SSL socket.") - proc gotHandshake*(socket: Socket): bool = ## Determines whether a handshake has occurred between a client (``socket``) ## and the server that ``socket`` is connected to. @@ -1026,7 +993,7 @@ proc select(readfd: Socket, timeout = 500): int = return 1 var fds = @[readfd.fd] - result = select(fds, timeout) + result = selectRead(fds, timeout) proc isClosed(socket: Socket): bool = socket.fd == osInvalidSocket @@ -1694,7 +1661,5 @@ proc connect*(socket: Socket, address: string, port = Port(0), when defineSsl and not defined(nimdoc): if socket.isSSL: socket.fd.setBlocking(true) - {.warning[Deprecated]: off.} - doAssert socket.handshake() - {.warning[Deprecated]: on.} + doAssert socket.gotHandshake() socket.fd.setBlocking(true) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index dfe75d9982..e7ab395aec 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -60,9 +60,6 @@ type Process* = ref ProcessObj ## represents an operating system process -const poUseShell* {.deprecated.} = poUsePath - ## Deprecated alias for poUsePath. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], @@ -335,19 +332,6 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -proc select*(readfds: var seq[Process], timeout = 500): int - {.benign, deprecated.} - ## `select` with a sensible Nim interface. `timeout` is in milliseconds. - ## Specify -1 for no timeout. Returns the number of processes that are - ## ready to read from. The processes that are ready to be read from are - ## removed from `readfds`. - ## - ## **Warning**: This function may give unexpected or completely wrong - ## results on Windows. - ## - ## **Deprecated since version 0.17.0**: This procedure isn't cross-platform - ## and so should not be used in newly written code. - when not defined(useNimRtl): proc execProcess(command: string, workingDir: string = "", diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index a84943cf95..9fd6cd2c7f 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -55,15 +55,6 @@ proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} = result.cmd = @cmdline -proc initOptParser*(cmdline: string): OptParser {.rtl, deprecated.} = - ## Initalizes option parses with cmdline. Splits cmdline in on spaces - ## and calls initOptParser(openarray[string]) - ## Do not use. - if cmdline == "": # backward compatibility - return initOptParser(@[]) - else: - return initOptParser(cmdline.split) - when not defined(createNimRtl): proc initOptParser*(): OptParser = ## Initializes option parser from current command line arguments. diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index ee8453b216..bd8592a4c2 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -138,18 +138,6 @@ proc parseIdent*(s: string, start = 0): string = while i < s.len and s[i] in IdentChars: inc(i) result = substr(s, start, i-1) -proc parseToken*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline, deprecated.} = - ## parses a token and stores it in ``token``. Returns - ## the number of the parsed characters or 0 in case of an error. A token - ## consists of the characters in `validChars`. - ## - ## **Deprecated since version 0.8.12**: Use ``parseWhile`` instead. - var i = start - while i < s.len and s[i] in validChars: inc(i) - result = i-start - token = substr(s, start, i-1) - proc skipWhitespace*(s: string, start = 0): int {.inline.} = ## skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index d296017dd2..3347ed5462 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -18,33 +18,6 @@ type hostname*, port*, path*, query*, anchor*: string opaque*: bool -{.push warning[deprecated]: off.} -proc `$`*(url: Url): string {.deprecated: "use Uri instead".} = - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - return string(url) - -proc `/`*(a, b: Url): Url {.deprecated: "use Uri instead".} = - ## Joins two URLs together, separating them with / if needed. - ## - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - var urlS = $a - var bS = $b - if urlS == "": return b - if urlS[urlS.len-1] != '/': - urlS.add('/') - if bS[0] == '/': - urlS.add(bS.substr(1)) - else: - urlS.add(bs) - result = Url(urlS) - -proc add*(url: var Url, a: Url) {.deprecated: "use Uri instead".} = - ## Appends url to url. - ## - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - url = url / a -{.pop.} - proc encodeUrl*(s: string, usePlus=true): string = ## Encodes a URL according to RFC3986. ## diff --git a/tests/async/twinasyncrw.nim b/tests/async/twinasyncrw.nim index 64c5d6c269..6763eb5a26 100644 --- a/tests/async/twinasyncrw.nim +++ b/tests/async/twinasyncrw.nim @@ -46,7 +46,7 @@ when defined(windows): success = false it = it.ai_next - dealloc(aiList) + freeAddrInfo(aiList) if not success: retFuture.fail(newException(OSError, osErrorMsg(lastError))) return retFuture diff --git a/tests/manyloc/nake/nakefile.nim b/tests/manyloc/nake/nakefile.nim index 9c66ad71c9..35ed3cbb0b 100644 --- a/tests/manyloc/nake/nakefile.nim +++ b/tests/manyloc/nake/nakefile.nim @@ -83,6 +83,7 @@ task "download", "download game assets": var skipAssets = false path = expandFilename("data") + client = newHttpClient() path.add DirSep path.add(extractFilename(GameAssets)) if existsFile(path): @@ -101,7 +102,7 @@ task "download", "download game assets": echo "Downloading from ", GameAssets if not skipAssets: echo "Downloading to ", path - downloadFile GameAssets, path + client.downloadFile(GameAssets, path) echo "Download finished" let targetDir = parentDir(parentDir(path)) @@ -126,7 +127,7 @@ task "download", "download game assets": else: return path = extractFilename(BinLibs) - downloadFile BinLibs, path + client.downloadFile(BinLibs, path) echo "Downloaded dem libs ", path when true: echo "Unpack it yourself, sorry." else: ## this crashes, dunno why From 1e63f1edb3fa594bcdce39eb5195d5f5d2644e67 Mon Sep 17 00:00:00 2001 From: cooldome Date: Fri, 18 Jan 2019 07:51:22 +0000 Subject: [PATCH 152/350] destructors: first step towards fixing #9617 (#10341) --- compiler/semasgn.nim | 126 ++++++++++++++--------------- compiler/semdata.nim | 7 ++ compiler/semexprs.nim | 4 +- compiler/semstmts.nim | 17 +++- tests/destructor/helper.nim | 3 + tests/destructor/terror_module.nim | 20 +++++ 6 files changed, 110 insertions(+), 67 deletions(-) create mode 100644 tests/destructor/helper.nim create mode 100644 tests/destructor/terror_module.nim diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index c1ecaf8a27..9f1ef313b7 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -14,7 +14,7 @@ type TLiftCtx = object - c: PContext + graph: ModuleGraph info: TLineInfo # for construction kind: TTypeAttachedOp fn: PSym @@ -22,7 +22,7 @@ type recurse: bool proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) -proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; +proc liftBody(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym {.discardable.} proc at(a, i: PNode, elemType: PType): PNode = @@ -33,7 +33,7 @@ proc at(a, i: PNode, elemType: PType): PNode = proc liftBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) = for i in 0 ..< t.len: - let lit = lowerings.newIntLit(c.c.graph, x.info, i) + let lit = lowerings.newIntLit(c.graph, x.info, i) liftBodyAux(c, t.sons[i], body, x.at(lit, t.sons[i]), y.at(lit, t.sons[i])) proc dotField(x: PNode, f: PSym): PNode = @@ -77,22 +77,22 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) = of nkRecList: for t in items(n): liftBodyObj(c, t, body, x, y) else: - illFormedAstLocal(n, c.c.config) + illFormedAstLocal(n, c.graph.config) -proc genAddr(c: PContext; x: PNode): PNode = +proc genAddr(g: ModuleGraph; x: PNode): PNode = if x.kind == nkHiddenDeref: - checkSonsLen(x, 1, c.config) + checkSonsLen(x, 1, g.config) result = x.sons[0] else: - result = newNodeIT(nkHiddenAddr, x.info, makeVarType(c, x.typ)) + result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ)) addSon(result, x) -proc newAsgnCall(c: PContext; op: PSym; x, y: PNode): PNode = +proc newAsgnCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode = #if sfError in op.flags: # localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s) result = newNodeI(nkCall, x.info) result.add newSymNode(op) - result.add genAddr(c, x) + result.add genAddr(g, x) result.add y proc newAsgnStmt(le, ri: PNode): PNode = @@ -105,10 +105,10 @@ proc newOpCall(op: PSym; x: PNode): PNode = result.add(newSymNode(op)) result.add x -proc destructorCall(c: PContext; op: PSym; x: PNode): PNode = +proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode = result = newNodeIT(nkCall, x.info, op.typ.sons[0]) result.add(newSymNode(op)) - result.add genAddr(c, x) + result.add genAddr(g, x) proc newDeepCopyCall(op: PSym; x, y: PNode): PNode = result = newAsgnStmt(x, newOpCall(op, y)) @@ -127,13 +127,13 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode; else: op = field if op == nil: - op = liftBody(c.c, t, c.kind, c.info) + op = liftBody(c.graph, t, c.kind, c.info) if sfError in op.flags: incl c.fn.flags, sfError else: - markUsed(c.c.config, c.info, op, c.c.graph.usageSym) + markUsed(c.graph.config, c.info, op, c.graph.usageSym) onUse(c.info, op) - body.add newAsgnCall(c.c, op, x, y) + body.add newAsgnCall(c.graph, op, x, y) result = true proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = @@ -141,9 +141,9 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = of attachedDestructor: let op = t.destructor if op != nil: - markUsed(c.c.config, c.info, op, c.c.graph.usageSym) + markUsed(c.graph.config, c.info, op, c.graph.usageSym) onUse(c.info, op) - body.add destructorCall(c.c, op, x) + body.add destructorCall(c.graph, op, x) result = true of attachedAsgn: result = considerAsgnOrSink(c, t, body, x, y, t.assignment) @@ -152,7 +152,7 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = of attachedDeepCopy: let op = t.deepCopy if op != nil: - markUsed(c.c.config, c.info, op, c.c.graph.usageSym) + markUsed(c.graph.config, c.info, op, c.graph.usageSym) onUse(c.info, op) body.add newDeepCopyCall(op, x, y) result = true @@ -169,13 +169,13 @@ proc addVar(father, v, value: PNode) = addSon(father, vpart) proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode = - var temp = newSym(skTemp, getIdent(c.c.cache, lowerings.genPrefix), c.fn, c.info) - temp.typ = getSysType(c.c.graph, body.info, tyInt) + var temp = newSym(skTemp, getIdent(c.graph.cache, lowerings.genPrefix), c.fn, c.info) + temp.typ = getSysType(c.graph, body.info, tyInt) incl(temp.flags, sfFromGeneric) var v = newNodeI(nkVarSection, c.info) result = newSymNode(temp) - v.addVar(result, lowerings.newIntLit(c.c.graph, body.info, first)) + v.addVar(result, lowerings.newIntLit(c.graph, body.info, first)) body.add v proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode = @@ -185,22 +185,22 @@ proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode = proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode = result = newNodeI(nkWhileStmt, c.info, 2) - let cmp = genBuiltin(c.c.graph, mLeI, "<=", i) - cmp.add genHigh(c.c.graph, dest) - cmp.typ = getSysType(c.c.graph, c.info, tyBool) + let cmp = genBuiltin(c.graph, mLeI, "<=", i) + cmp.add genHigh(c.graph, dest) + cmp.typ = getSysType(c.graph, c.info, tyBool) result.sons[0] = cmp result.sons[1] = newNodeI(nkStmtList, c.info) proc addIncStmt(c: var TLiftCtx; body, i: PNode) = - let incCall = genBuiltin(c.c.graph, mInc, "inc", i) - incCall.add lowerings.newIntLit(c.c.graph, c.info, 1) + let incCall = genBuiltin(c.graph, mInc, "inc", i) + incCall.add lowerings.newIntLit(c.graph, c.info, 1) body.add incCall -proc newSeqCall(c: PContext; x, y: PNode): PNode = +proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode = # don't call genAddr(c, x) here: - result = genBuiltin(c.graph, mNewSeq, "newSeq", x) - let lenCall = genBuiltin(c.graph, mLengthSeq, "len", y) - lenCall.typ = getSysType(c.graph, x.info, tyInt) + result = genBuiltin(g, mNewSeq, "newSeq", x) + let lenCall = genBuiltin(g, mLengthSeq, "len", y) + lenCall.typ = getSysType(g, x.info, tyInt) result.add lenCall proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = @@ -211,7 +211,7 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = defaultOp(c, t, body, x, y) of tyArray: if tfHasAsgn in t.flags: - let i = declareCounter(c, body, firstOrd(c.c.config, t)) + let i = declareCounter(c, body, firstOrd(c.graph.config, t)) let whileLoop = genWhileLoop(c, i, x) let elemType = t.lastSon liftBodyAux(c, elemType, whileLoop.sons[1], x.at(i, elemType), @@ -223,12 +223,12 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = of tySequence: # note that tfHasAsgn is propagated so we need the check on # 'selectedGC' here to determine if we have the new runtime. - if c.c.config.selectedGC == gcDestructors: + if c.graph.config.selectedGC == gcDestructors: discard considerOverloadedOp(c, t, body, x, y) elif tfHasAsgn in t.flags: if c.kind != attachedDestructor: - body.add newSeqCall(c.c, x, y) - let i = declareCounter(c, body, firstOrd(c.c.config, t)) + body.add newSeqCall(c.graph, x, y) + let i = declareCounter(c, body, firstOrd(c.graph.config, t)) let whileLoop = genWhileLoop(c, i, x) let elemType = t.lastSon liftBodyAux(c, elemType, whileLoop.sons[1], x.at(i, elemType), @@ -258,20 +258,20 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = # have to go through some indirection; we delegate this to the codegen: let call = newNodeI(nkCall, c.info, 2) call.typ = t - call.sons[0] = newSymNode(createMagic(c.c.graph, "deepCopy", mDeepCopy)) + call.sons[0] = newSymNode(createMagic(c.graph, "deepCopy", mDeepCopy)) call.sons[1] = y body.add newAsgnStmt(x, call) of tyVarargs, tyOpenArray: - localError(c.c.config, c.info, "cannot copy openArray") + localError(c.graph.config, c.info, "cannot copy openArray") of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt, tyTypeDesc, tyGenericInvocation, tyForward: - internalError(c.c.config, c.info, "assignment requested for type: " & typeToString(t)) + internalError(c.graph.config, c.info, "assignment requested for type: " & typeToString(t)) of tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink: liftBodyAux(c, lastSon(t), body, x, y) - of tyOptAsRef: internalError(c.c.config, "liftBodyAux") + of tyOptAsRef: internalError(c.graph.config, "liftBodyAux") proc newProcType(info: TLineInfo; owner: PSym): PType = result = newType(tyProc, owner) @@ -287,54 +287,54 @@ proc addParam(procType: PType; param: PSym) = addSon(procType.n, newSymNode(param)) rawAddSon(procType, param.typ) -proc liftBodyDistinctType(c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = +proc liftBodyDistinctType(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = assert typ.kind == tyDistinct let baseType = typ[0] case kind of attachedAsgn: if baseType.assignment == nil: - discard liftBody(c, baseType, kind, info) + discard liftBody(g, baseType, kind, info) typ.assignment = baseType.assignment result = typ.assignment of attachedSink: if baseType.sink == nil: - discard liftBody(c, baseType, kind, info) + discard liftBody(g, baseType, kind, info) typ.sink = baseType.sink result = typ.sink of attachedDeepCopy: if baseType.deepCopy == nil: - discard liftBody(c, baseType, kind, info) + discard liftBody(g, baseType, kind, info) typ.deepCopy = baseType.deepCopy result = typ.deepCopy of attachedDestructor: if baseType.destructor == nil: - discard liftBody(c, baseType, kind, info) + discard liftBody(g, baseType, kind, info) typ.destructor = baseType.destructor result = typ.destructor -proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; +proc liftBody(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = if typ.kind == tyDistinct: - return liftBodyDistinctType(c, typ, kind, info) + return liftBodyDistinctType(g, typ, kind, info) var a: TLiftCtx a.info = info - a.c = c + a.graph = g a.kind = kind let body = newNodeI(nkStmtList, info) let procname = case kind - of attachedAsgn: getIdent(c.cache, "=") - of attachedSink: getIdent(c.cache, "=sink") - of attachedDeepCopy: getIdent(c.cache, "=deepcopy") - of attachedDestructor: getIdent(c.cache, "=destroy") + of attachedAsgn: getIdent(g.cache, "=") + of attachedSink: getIdent(g.cache, "=sink") + of attachedDeepCopy: getIdent(g.cache, "=deepcopy") + of attachedDestructor: getIdent(g.cache, "=destroy") result = newSym(skProc, procname, typ.owner, info) a.fn = result a.asgnForType = typ - let dest = newSym(skParam, getIdent(c.cache, "dest"), result, info) - let src = newSym(skParam, getIdent(c.cache, "src"), result, info) - dest.typ = makeVarType(c, typ) + let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info) + let src = newSym(skParam, getIdent(g.cache, "src"), result, info) + dest.typ = makeVarType(typ.owner, typ) src.typ = typ result.typ = newProcType(info, typ.owner) @@ -345,7 +345,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; liftBodyAux(a, typ, body, newSymNode(dest).newDeref, newSymNode(src)) # recursion is handled explicitly, do not register the type based operation # before 'liftBodyAux': - if c.config.selectedGC == gcDestructors and + if g.config.selectedGC == gcDestructors and typ.kind in {tySequence, tyString} and body.len == 0: discard "do not cache it yet" else: @@ -364,17 +364,17 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; incl result.flags, sfFromGeneric -proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym = +proc getAsgnOrLiftBody(g: ModuleGraph; typ: PType; info: TLineInfo): PSym = let t = typ.skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink}) result = t.assignment if result.isNil: - result = liftBody(c, t, attachedAsgn, info) + result = liftBody(g, t, attachedAsgn, info) -proc overloadedAsgn(c: PContext; dest, src: PNode): PNode = - let a = getAsgnOrLiftBody(c, dest.typ, dest.info) - result = newAsgnCall(c, a, dest, src) +proc overloadedAsgn(g: ModuleGraph; dest, src: PNode): PNode = + let a = getAsgnOrLiftBody(g, dest.typ, dest.info) + result = newAsgnCall(g, a, dest, src) -proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) = +proc liftTypeBoundOps*(g: ModuleGraph; typ: PType; info: TLineInfo) = ## In the semantic pass this is called in strategic places ## to ensure we lift assignment, destructors and moves properly. ## The later 'destroyer' pass depends on it. @@ -386,11 +386,11 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) = let typ = typ.skipTypes({tyGenericInst, tyAlias}) # we generate the destructor first so that other operators can depend on it: if typ.destructor == nil: - liftBody(c, typ, attachedDestructor, info) + liftBody(g, typ, attachedDestructor, info) if typ.assignment == nil: - liftBody(c, typ, attachedAsgn, info) + liftBody(g, typ, attachedAsgn, info) if typ.sink == nil: - liftBody(c, typ, attachedSink, info) + liftBody(g, typ, attachedSink, info) -#proc patchResolvedTypeBoundOp*(c: PContext; n: PNode): PNode = +#proc patchResolvedTypeBoundOp*(g: ModuleGraph; n: PNode): PNode = # if n.kind == nkCall and diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 735c6f6b16..a05bda32dd 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -286,6 +286,13 @@ proc makeVarType*(c: PContext, baseType: PType; kind = tyVar): PType = result = newTypeS(kind, c) addSonSkipIntLit(result, baseType) +proc makeVarType*(owner: PSym, baseType: PType; kind = tyVar): PType = + if baseType.kind == kind: + result = baseType + else: + result = newType(kind, owner) + addSonSkipIntLit(result, baseType) + proc makeTypeDesc*(c: PContext, typ: PType): PType = if typ.kind == tyTypeDesc: result = typ diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index e74d56a863..6f2981f632 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -802,7 +802,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = result = magicsAfterOverloadResolution(c, result, flags) if result.typ != nil and not (result.typ.kind == tySequence and result.typ.sons[0].kind == tyEmpty): - liftTypeBoundOps(c, result.typ, n.info) + liftTypeBoundOps(c.graph, result.typ, n.info) #result = patchResolvedTypeBoundOp(c, result) if c.matchedConcept == nil: result = evalAtCompileTime(c, result) @@ -1592,7 +1592,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = typeMismatch(c.config, n.info, lhs.typ, rhsTyp) n.sons[1] = fitNode(c, le, rhs, goodLineInfo(n[1])) - liftTypeBoundOps(c, lhs.typ, lhs.info) + liftTypeBoundOps(c.graph, lhs.typ, lhs.info) #liftTypeBoundOps(c, n.sons[0].typ, n.sons[0].info) fixAbstractType(c, n) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 70a16b290c..de39e95c8d 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -476,7 +476,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = # this can only happen for errornous var statements: if typ == nil: continue typeAllowedCheck(c.config, a.info, typ, symkind, if c.matchedConcept != nil: {taConcept} else: {}) - liftTypeBoundOps(c, typ, a.info) + liftTypeBoundOps(c.graph, typ, a.info) var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink}) if a.kind == nkVarTuple: if tup.kind != tyTuple: @@ -1479,7 +1479,8 @@ proc canonType(c: PContext, t: PType): PType = result = t proc semOverride(c: PContext, s: PSym, n: PNode) = - case s.name.s.normalize + let name = s.name.s.normalize + case name of "=destroy": let t = s.typ var noError = false @@ -1498,6 +1499,9 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = localError(c.config, n.info, errGenerated, "cannot bind another '" & s.name.s & "' to: " & typeToString(obj)) noError = true + if obj.owner.getModule != s.getModule: + localError(c.config, n.info, errGenerated, + "type bound operation `=destroy` can be defined only in the same module with its type (" & obj.typeToString() & ")") if not noError and sfSystemModule notin s.owner.flags: localError(c.config, n.info, errGenerated, "signature for '" & s.name.s & "' must be proc[T: object](x: var T)") @@ -1521,6 +1525,11 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = else: localError(c.config, n.info, errGenerated, "cannot bind 'deepCopy' to: " & typeToString(t)) + + if t.owner.getModule != s.getModule: + localError(c.config, n.info, errGenerated, + "type bound operation `" & name & "` can be defined only in the same module with its type (" & t.typeToString() & ")") + else: localError(c.config, n.info, errGenerated, "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T") @@ -1551,6 +1560,10 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = else: localError(c.config, n.info, errGenerated, "cannot bind another '" & s.name.s & "' to: " & typeToString(obj)) + if obj.owner.getModule != s.getModule: + localError(c.config, n.info, errGenerated, + "type bound operation `" & name & "` can be defined only in the same module with its type (" & obj.typeToString() & ")") + return if sfSystemModule notin s.owner.flags: localError(c.config, n.info, errGenerated, diff --git a/tests/destructor/helper.nim b/tests/destructor/helper.nim new file mode 100644 index 0000000000..466065747f --- /dev/null +++ b/tests/destructor/helper.nim @@ -0,0 +1,3 @@ +type + MyTestObject*[T] = object + p: ptr T diff --git a/tests/destructor/terror_module.nim b/tests/destructor/terror_module.nim new file mode 100644 index 0000000000..f3d7c9b26c --- /dev/null +++ b/tests/destructor/terror_module.nim @@ -0,0 +1,20 @@ +discard """ +joinable: false +cmd: "nim check $file" +errormsg: "type bound operation `=deepcopy` can be defined only in the same module with its type (MyTestObject)" +nimout: ''' +terror_module.nim(14, 1) Error: type bound operation `=destroy` can be defined only in the same module with its type (MyTestObject) +terror_module.nim(16, 1) Error: type bound operation `=sink` can be defined only in the same module with its type (MyTestObject) +terror_module.nim(18, 1) Error: type bound operation `=` can be defined only in the same module with its type (MyTestObject) +terror_module.nim(20, 1) Error: type bound operation `=deepcopy` can be defined only in the same module with its type (MyTestObject) +''' +""" +import helper + +proc `=destroy`[T](x: var MyTestObject[T]) = discard + +proc `=sink`[T](x: var MyTestObject[T], y:MyTestObject[T]) = discard + +proc `=`[T](x: var MyTestObject[T], y: MyTestObject[T]) = discard + +proc `=deepcopy`[T](x: ptr MyTestObject[T]): ptr MyTestObject[T] = discard From 4b4e4fc029d3f151c1e8bcdafc1d9b381a7142a6 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Fri, 18 Jan 2019 17:00:32 +0900 Subject: [PATCH 153/350] Fix unhandled exception that raised when nim was executed with --cc:vcc option (#10356) --- tools/vccexe/vccenv.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/vccexe/vccenv.nim b/tools/vccexe/vccenv.nim index 6ddf2e29a5..724fe992a4 100644 --- a/tools/vccexe/vccenv.nim +++ b/tools/vccexe/vccenv.nim @@ -44,4 +44,4 @@ proc vccEnvVcVarsAllPath*(version: VccEnvVersion = vsUndefined): string = let key = $version let val = getEnv key if val.len > 0: - result = expandFilename(val & vcvarsallRelativePath) + result = try: expandFilename(val & vcvarsallRelativePath) except OSError: "" From 27e2ed4375c21b196f5fd403c2199c63dcdb8bf0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jan 2019 00:03:26 -0800 Subject: [PATCH 154/350] fix #9629 every binary cmd line option allows on/off/empty=on (#10353) * fix #9629 every binary cmd line option allows on/off/empty=on * workaround refs #10359 --- compiler/commands.nim | 62 ++++++++++++++-------------------------- doc/advopt.txt | 30 +++++++++---------- doc/basicopt.txt | 4 +-- tests/misc/tparseopt.nim | 5 +++- 4 files changed, 43 insertions(+), 58 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 47687046f6..30521f9ca8 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -142,7 +142,7 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case arg.normalize - of "on": conf.options = conf.options + op + of "","on": conf.options = conf.options + op of "off": conf.options = conf.options - op else: localError(conf, info, errOnOrOffExpectedButXFound % arg) @@ -158,7 +158,7 @@ proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass: proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case arg.normalize - of "on": conf.globalOptions = conf.globalOptions + op + of "", "on": conf.globalOptions = conf.globalOptions + op of "off": conf.globalOptions = conf.globalOptions - op else: localError(conf, info, errOnOrOffExpectedButXFound % arg) @@ -414,26 +414,19 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: addExternalFileToLink(conf, AbsoluteFile arg) of "debuginfo": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optCDebug) + processOnOffSwitchG(conf, {optCDebug}, arg, pass, info) of "embedsrc": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optEmbedOrigSrc) + processOnOffSwitchG(conf, {optEmbedOrigSrc}, arg, pass, info) of "compileonly", "c": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optCompileOnly) + processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info) of "nolinking": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optNoLinking) + processOnOffSwitchG(conf, {optNoLinking}, arg, pass, info) of "nomain": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optNoMain) + processOnOffSwitchG(conf, {optNoMain}, arg, pass, info) of "forcebuild", "f": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optForceFullMake) + processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info) of "project": - expectNoArg(conf, switch, arg, pass, info) - incl conf.globalOptions, optWholeProject + processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info) of "gc": expectArg(conf, switch, arg, pass, info) case arg.normalize @@ -499,7 +492,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; else: undefSymbol(conf.symbols, "hotcodereloading") of "oldnewlines": case arg.normalize - of "on": + of "","on": conf.oldNewlines = true defineSymbol(conf.symbols, "nimOldNewlines") of "off": @@ -599,11 +592,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string of "listcmd": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optListCmd) + processOnOffSwitchG(conf, {optListCmd}, arg, pass, info) of "genmapping": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optGenMapping) + processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info) of "os": expectArg(conf, switch, arg, pass, info) if pass in {passCmd1, passPP}: @@ -619,8 +610,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; elif cpu != conf.target.hostCPU: setTarget(conf.target, conf.target.targetOS, cpu) of "run", "r": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optRun) + processOnOffSwitchG(conf, {optRun}, arg, pass, info) of "errormax": expectArg(conf, switch, arg, pass, info) # Note: `nim check` (etc) can overwrite this. @@ -668,21 +658,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "v2": conf.symbolFiles = v2Sf else: localError(conf, info, "invalid option for --incremental: " & arg) of "skipcfg": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optSkipSystemConfigFile) + processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info) of "skipprojcfg": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optSkipProjConfigFile) + processOnOffSwitchG(conf, {optSkipProjConfigFile}, arg, pass, info) of "skipusercfg": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optSkipUserConfigFile) + processOnOffSwitchG(conf, {optSkipUserConfigFile}, arg, pass, info) of "skipparentcfg": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optSkipParentConfigFiles) + processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info) of "genscript", "gendeps": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optGenScript) - incl(conf.globalOptions, optCompileOnly) + processOnOffSwitchG(conf, {optGenScript}, arg, pass, info) + processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info) of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info) of "lib": expectArg(conf, switch, arg, pass, info) @@ -716,16 +701,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideUse of "stdout": - expectNoArg(conf, switch, arg, pass, info) - incl(conf.globalOptions, optStdout) + processOnOffSwitchG(conf, {optStdout}, arg, pass, info) of "listfullpaths": - expectNoArg(conf, switch, arg, pass, info) - incl conf.globalOptions, optListFullPaths + processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info) of "dynliboverride": dynlibOverride(conf, switch, arg, pass, info) of "dynliboverrideall": - expectNoArg(conf, switch, arg, pass, info) - incl conf.globalOptions, optDynlibOverrideAll + processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info) of "cs": # only supported for compatibility. Does nothing. expectArg(conf, switch, arg, pass, info) diff --git a/doc/advopt.txt b/doc/advopt.txt index c2e09a3ffd..7ab85abfc5 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -18,9 +18,9 @@ Advanced commands: Advanced options: -o:FILE, --out:FILE set the output filename - --stdout output to stdout + --stdout:on|off output to stdout --colors:on|off turn compiler messages coloring on|off - --listFullPaths list full paths in messages + --listFullPaths:on|off list full paths in messages -w:on|off|list, --warnings:on|off|list turn all warnings on|off or list all available --warning[X]:on|off turn specific warning X on|off @@ -39,16 +39,16 @@ Advanced options: --nimcache:PATH set the path used for generated files --header:FILE the compiler should produce a .h file (FILE is optional) - -c, --compileOnly compile Nim files only; do not assemble or link - --noLinking compile Nim and generated files but do not link - --noMain do not generate a main procedure - --genScript generate a compile script (in the 'nimcache' + -c, --compileOnly:on|off compile Nim files only; do not assemble or link + --noLinking:on|off compile Nim and generated files but do not link + --noMain:on|off do not generate a main procedure + --genScript:on|off generate a compile script (in the 'nimcache' subdirectory named 'compile_$$project$$scriptext'), implies --compileOnly - --genDeps generate a '.deps' file containing the dependencies + --genDeps:on|off generate a '.deps' file containing the dependencies --os:SYMBOL set the target operating system (cross-compilation) --cpu:SYMBOL set the target processor (cross-compilation) - --debuginfo enables debug information + --debuginfo:on|off enables debug information -t, --passC:OPTION pass an option to the C compiler -l, --passL:OPTION pass an option to the linker --cincludes:DIR modify the C compiler header search path @@ -59,7 +59,7 @@ Advanced options: --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) --lineDir:on|off generation of #line directive on|off - --embedsrc embeds the original source code as comments + --embedsrc:on|off embeds the original source code as comments in the generated output --threadanalysis:on|off turn thread analysis on|off --tlsEmulation:on|off turn thread local storage emulation on|off @@ -77,10 +77,10 @@ Advanced options: --nilseqs:on|off allow 'nil' for strings/seqs for backwards compatibility --oldast:on|off use old AST for backwards compatibility - --skipCfg do not read the nim installation's configuration file - --skipUserCfg do not read the user's configuration file - --skipParentCfg do not read the parent dirs' configuration files - --skipProjCfg do not read the project's configuration file + --skipCfg:on|off do not read the nim installation's configuration file + --skipUserCfg:on|off do not read the user's configuration file + --skipParentCfg:on|off do not read the parent dirs' configuration files + --skipProjCfg:on|off do not read the project's configuration file --gc:refc|markAndSweep|boehm|go|none|regions select the GC to use; default is 'refc' --index:on|off turn index file generation on|off @@ -97,8 +97,8 @@ Advanced options: symbol matching is fuzzy so that --dynlibOverride:lua matches dynlib: "liblua.so.3" - --dynlibOverrideAll makes the dynlib pragma have no effect - --listCmd list the commands used to execute external programs + --dynlibOverrideAll:on|off makes the dynlib pragma have no effect + --listCmd:on|off list the commands used to execute external programs --parallelBuild:0|1|... perform a parallel build value = number of processors (0 for auto-detect) --incremental:on|off only recompile the changed modules (experimental!) diff --git a/doc/basicopt.txt b/doc/basicopt.txt index a9166d36cb..96c9ced3df 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -15,7 +15,7 @@ Options: (Optionally: Define the value for that symbol, see: "compile time define pragmas") -u, --undef:SYMBOL undefine a conditional symbol - -f, --forceBuild force rebuilding of all modules + -f, --forceBuild:on|off force rebuilding of all modules --stackTrace:on|off turn stack tracing on|off --lineTrace:on|off turn line tracing on|off --threads:on|off turn support for multi-threading on|off @@ -35,7 +35,7 @@ Options: --debugger:native|endb use native debugger (gdb) | ENDB (experimental) --app:console|gui|lib|staticlib generate a console app|GUI app|DLL|static library - -r, --run run the compiled program with given arguments + -r, --run:on|off run the compiled program with given arguments --fullhelp show all command line switches -h, --help show this help diff --git a/tests/misc/tparseopt.nim b/tests/misc/tparseopt.nim index d8824e5220..7d5071c3ec 100644 --- a/tests/misc/tparseopt.nim +++ b/tests/misc/tparseopt.nim @@ -121,7 +121,10 @@ else: block: # fix #9842 let exe = buildDir / "D20190112T145450".addFileExt(ExeExt) - defer: removeFile exe + defer: + when not defined(windows): + # workaround #10359 ; innocuous to skip since we're saving under `buildDir` + removeFile exe let args = @["a1b", "a2 b", "", "a4\"b", "a5'b", r"a6\b", "a7\'b"] let cmd = "$# c -r --verbosity:0 -o:$# -d:testament_tparseopt $# $#" % [getCurrentCompilerExe(), exe, currentSourcePath(), From f11f36e7d5c2dc9a8f13f84ead360369fb05eaf3 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Fri, 18 Jan 2019 09:04:12 +0100 Subject: [PATCH 155/350] Fixed getCustomPragmaVal to allow multiple fields in custom annotations (#10289) --- lib/core/macros.nim | 15 ++++++++++++--- tests/pragmas/tcustom_pragma.nim | 9 +++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 08ee051522..3a85324bac 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1500,9 +1500,18 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = let pragmaNode = customPragmaNode(n) for p in pragmaNode: if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: - return p[1] - - error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error, + if p.len == 2: + result = p[1] + else: + let def = p[0].getImpl[3] + result = newTree(nnkPar) + for i in 1.. Date: Fri, 18 Jan 2019 14:17:25 +0100 Subject: [PATCH 156/350] better docs: parseutils --- lib/pure/parseutils.nim | 215 +++++++++++++++++++++++++++++++++------- 1 file changed, 180 insertions(+), 35 deletions(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index bd8592a4c2..c2d52ebcce 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -7,9 +7,46 @@ # distribution, for details about the copyright. # -## This module contains helpers for parsing tokens, numbers, identifiers, etc. +## This module contains helpers for parsing tokens, numbers, integers, floats, +## identifiers, etc. ## ## To unpack raw bytes look at the `streams `_ module. +## +## +## .. code-block:: +## import parseutils +## +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## echo res & " - " & captureBetween(log, ' ', '_') +## # => 2019-01-10 - OK +## +## +## .. code-block:: +## import parseutils +## from strutils import Digits, parseInt +## +## let userInput1 = "2019 school start" +## let userInput2 = "3 years back" +## +## let startYear = input1[0..skipWhile(input1, Digits)-1] # 2019 +## let yearsBack = input2[0..skipWhile(input2, Digits)-1] # 3 +## +## echo "Examination is in " & $(parseInt(startYear) + parseInt(yearsBack)) +## +## +## **See also:** +## * `strutils module`_ for combined and identical parsing proc's +## * `json module`_ for a JSON parser +## * `parsecfg module`_ for a configuration file parser +## * `parsecsv module`_ for a simple CSV (comma separated value) parser +## * `parseopt module`_ for a command line parser +## * `parsexml module`_ for a XML / HTML parser +## * `other parsers`_ for other parsers + {.deadCodeElim: on.} # dce option deprecated @@ -35,21 +72,20 @@ proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. ## proc is sensitive to the already existing value of ``number`` and will ## likely not do what you want unless you make sure ``number`` is zero. You ## can use this feature to *chain* calls, though the result int will quickly - ## overflow. Example: - ## - ## .. code-block:: nim - ## var value = 0 - ## discard parseHex("0x38", value) - ## assert value == 56 - ## discard parseHex("0x34", value) - ## assert value == 56 * 256 + 52 - ## value = -1 - ## discard parseHex("0x38", value) - ## assert value == -200 + ## overflow. ## ## If ``maxLen == 0`` the length of the hexadecimal number has no upper bound. ## Else no more than ``start + maxLen`` characters are parsed, up to the ## length of the string. + runnableExamples: + var value = 0 + discard parseHex("0x38", value) + assert value == 56 + discard parseHex("0x34", value) + assert value == 56 * 256 + 52 + value = -1 + discard parseHex("0x38", value) + assert value == -200 var i = start var foundDigit = false # get last index based on minimum `start + maxLen` or `s.len` @@ -80,6 +116,11 @@ proc parseOct*(s: string, number: var int, start = 0, maxLen = 0): int {. ## If ``maxLen == 0`` the length of the octal number has no upper bound. ## Else no more than ``start + maxLen`` characters are parsed, up to the ## length of the string. + runnableExamples: + var res: int + doAssert parseOct("12", res) == 2 + doAssert res == 10 + doAssert parseOct("9", res) == 0 var i = start var foundDigit = false # get last index based on minimum `start + maxLen` or `s.len` @@ -103,6 +144,10 @@ proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {. ## If ``maxLen == 0`` the length of the binary number has no upper bound. ## Else no more than ``start + maxLen`` characters are parsed, up to the ## length of the string. + runnableExamples: + var res: int + doAssert parseBin("010011100110100101101101", res) == 24 + doAssert parseBin("3", res) == 0 var i = start var foundDigit = false # get last index based on minimum `start + maxLen` or `s.len` @@ -119,8 +164,16 @@ proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {. if foundDigit: result = i-start proc parseIdent*(s: string, ident: var string, start = 0): int = - ## parses an identifier and stores it in ``ident``. Returns + ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" var i = start if i < s.len and s[i] in IdentStartChars: inc(i) @@ -129,8 +182,13 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = result = i-start proc parseIdent*(s: string, start = 0): string = - ## parses an identifier and returns it or an empty string in + ## Parses an identifier and returns it or an empty string in ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" result = "" var i = start if i < s.len and s[i] in IdentStartChars: @@ -139,20 +197,34 @@ proc parseIdent*(s: string, start = 0): string = result = substr(s, start, i-1) proc skipWhitespace*(s: string, start = 0): int {.inline.} = - ## skips the whitespace starting at ``s[start]``. Returns the number of + ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 while start+result < s.len and s[start+result] in Whitespace: inc(result) proc skip*(s, token: string, start = 0): int {.inline.} = - ## skips the `token` starting at ``s[start]``. Returns the length of `token` + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 while start+result < s.len and result < token.len and s[result+start] == token[result]: inc(result) if result != token.len: result = 0 proc skipIgnoreCase*(s, token: string, start = 0): int = - ## same as `skip` but case is ignored for token matching. + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 while start+result < s.len and result < token.len and toLower(s[result+start]) == toLower(token[result]): inc(result) if result != token.len: result = 0 @@ -161,24 +233,45 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 while start+result < s.len and s[result+start] notin until: inc(result) proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 while start+result < s.len and s[result+start] != until: inc(result) proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = ## Skips all characters while one char from the set `token` is found. ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 while start+result < s.len and s[result+start] in toSkip: inc(result) proc parseUntil*(s: string, token: var string, until: set[char], start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " var i = start while i < s.len and s[i] notin until: inc(i) result = i-start @@ -186,9 +279,17 @@ proc parseUntil*(s: string, token: var string, until: set[char], proc parseUntil*(s: string, token: var string, until: char, start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" var i = start while i < s.len and s[i] != until: inc(i) result = i-start @@ -196,9 +297,15 @@ proc parseUntil*(s: string, token: var string, until: char, proc parseUntil*(s: string, token: var string, until: string, start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " if until.len == 0: token.setLen(0) return 0 @@ -215,9 +322,15 @@ proc parseUntil*(s: string, token: var string, until: string, proc parseWhile*(s: string, token: var string, validChars: set[char], start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" var i = start while i < s.len and s[i] in validChars: inc(i) result = i-start @@ -226,6 +339,10 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" var i = skipUntil(s, first, start)+1+start result = "" discard s.parseUntil(result, if second == '\0': first else: second, i) @@ -260,9 +377,13 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = - ## parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 var res: BiggestInt # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): @@ -272,9 +393,15 @@ proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. proc parseInt*(s: string, number: var int, start = 0): int {. rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = - ## parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("2019", res, 0) == 4 + doAssert res == 2019 + doAssert parseInt("2019", res, 2) == 2 + doAssert res == 19 var res: BiggestInt result = parseBiggestInt(s, res, start) when sizeof(int) <= 4: @@ -285,16 +412,14 @@ proc parseInt*(s: string, number: var int, start = 0): int {. proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. raises: [].}= - ## parses a natural number into ``b``. This cannot raise an overflow + ## Parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. ## The number of processed character is returned. ## This is usually what you really want to use instead of `parseInt`:idx:. - ## Example: - ## - ## .. code-block:: nim - ## var res = 0 - ## discard parseSaturatedNatural("848", res) - ## doAssert res == 848 + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 var i = start if i < s.len and s[i] == '+': inc(i) if i < s.len and s[i] in {'0'..'9'}: @@ -331,9 +456,15 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = - ## parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer starting at `start` and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 var res: BiggestUInt # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): @@ -343,9 +474,15 @@ proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. proc parseUInt*(s: string, number: var uint, start = 0): int {. rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = - ## parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer starting at `start` and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 var res: BiggestUInt result = parseBiggestUInt(s, res, start) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: @@ -356,22 +493,30 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## parses a float starting at `start` and stores the value into `number`. + ## Parses a float starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. proc parseFloat*(s: string, number: var float, start = 0): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## parses a float starting at `start` and stores the value into `number`. + ## Parses a float starting at `start` and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 var bf: BiggestFloat result = parseBiggestFloat(s, bf, start) if result != 0: number = bf type - InterpolatedKind* = enum ## describes for `interpolatedFragments` + InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is ## yielded; for example in "str$$$var${expr}" ikStr, ## ``str`` part of the interpolated string From 8399e0f78ad7ef81183cb6797fa78f8c8116e502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Fri, 18 Jan 2019 14:36:38 +0100 Subject: [PATCH 157/350] deprecate += and friends for bool and enum (#10336) * fixes #10257 * add version to deprecation --- lib/system.nim | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index b479b920a3..d4a86e53be 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3886,15 +3886,21 @@ proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, ## Same as `gorge` but also returns the precious exit code. discard -proc `+=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. +proc `+=`*[T: SomeInteger](x: var T, y: T) {. magic: "Inc", noSideEffect.} - ## Increments an ordinal + ## Increments an integer -proc `-=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. +proc `+=`*[T: enum|bool](x: var T, y: T) {. + magic: "Inc", noSideEffect, deprecated: "use `inc` instead".} + +proc `-=`*[T: SomeInteger](x: var T, y: T) {. magic: "Dec", noSideEffect.} ## Decrements an ordinal -proc `*=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. +proc `-=`*[T: enum|bool](x: var T, y: T) {. + magic: "Dec", noSideEffect, deprecated: "0.20.0, use `dec` instead".} + +proc `*=`*[T: SomeInteger](x: var T, y: T) {. inline, noSideEffect.} = ## Binary `*=` operator for ordinals x = x * y From 3cc39c2eb90fa391c3da732525db0cdb52d7f958 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Fri, 18 Jan 2019 23:16:35 +0530 Subject: [PATCH 158/350] Show filename in exception raised by expandFilename on windows (#10365) --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 32fa2885ea..181bc5728d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1427,7 +1427,7 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", for x in walkFiles(result.string): result = x if not existsFile(result) and not existsDir(result): - raise newException(OSError, "file does not exist") + raise newException(OSError, "file '" & result & "' does not exist") else: # according to Posix we don't need to allocate space for result pathname. # But we need to free return value with free(3). From 86a91c1a4a38a669e9ed5409975936f64ad3e532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Fri, 18 Jan 2019 19:15:29 +0100 Subject: [PATCH 159/350] Fix parseutils.parseBiggestInt regression (#10348) --- lib/pure/parseutils.nim | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index c2d52ebcce..ba09347a29 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -350,6 +350,10 @@ proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = proc integerOutOfRangeError() {.noinline.} = raise newException(ValueError, "Parsed integer outside of valid range") +# See #6752 +when defined(js): + {.push overflowChecks: off.} + proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = var sign: BiggestInt = -1 @@ -363,18 +367,21 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = b = 0 while i < s.len and s[i] in {'0'..'9'}: let c = ord(s[i]) - ord('0') - if b >= (low(int) + c) div 10: + if b >= (low(BiggestInt) + c) div 10: b = b * 10 - c else: integerOutOfRangeError() inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - if sign == -1 and b == low(int): + if sign == -1 and b == low(BiggestInt): integerOutOfRangeError() else: b = b * sign result = i - start +when defined(js): + {.pop.} # overflowChecks: off + proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = ## Parses an integer starting at `start` and stores the value into `number`. @@ -632,4 +639,8 @@ when isMainModule: doAssert(parseSaturatedNatural("1_000_000", value) == 9) doAssert value == 1_000_000 + var i64Value: int64 + discard parseBiggestInt("9223372036854775807", i64Value) + doAssert i64Value == 9223372036854775807 + {.pop.} From f7c0360aba4e8ac2857adb85ff838d94cf30ff53 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 19 Jan 2019 12:46:49 +0100 Subject: [PATCH 160/350] allocators: introduce --define:nimMinHeapPages for tuning mmap calls (omg they are slow on OSX...) --- lib/system/alloc.nim | 6 +++--- lib/system/gc_regions.nim | 29 ++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index b090117a92..e938dc4751 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -20,7 +20,7 @@ template track(op, address, size) = # Each chunk starts at an address that is divisible by the page size. const - InitialMemoryRequest = 128 * PageSize # 0.5 MB + nimMinHeapPages {.intdefine.} = 128 # 0.5 MB SmallChunkSize = PageSize MaxFli = 30 MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' @@ -588,8 +588,8 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") result = findSuitableBlock(a, fl, sl) if result == nil: - if size < InitialMemoryRequest: - result = requestOsChunks(a, InitialMemoryRequest) + if size < nimMinHeapPages * PageSize: + result = requestOsChunks(a, nimMinHeapPages * PageSize) splitChunk(a, result, size) else: result = requestOsChunks(a, size) diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 8a14469449..3b908fb08c 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -23,6 +23,21 @@ when defined(nimphpext): proc osDeallocPages(p: pointer, size: int) {.inline.} = efree(p) +elif defined(useMalloc): + proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + proc emalloc(size: int): pointer {.importc: "malloc", header: "".} + proc efree(mem: pointer) {.importc: "free", header: "".} + + proc osAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osTryAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + efree(p) + else: include osalloc @@ -108,6 +123,8 @@ template `+!`(p: pointer, s: int): pointer = template `-!`(p: pointer, s: int): pointer = cast[pointer](cast[int](p) -% s) +const nimMinHeapPages {.intdefine.} = 4 + proc allocSlowPath(r: var MemRegion; size: int) = # we need to ensure that the underlying linked list # stays small. Say we want to grab 16GB of RAM with some @@ -116,9 +133,8 @@ proc allocSlowPath(r: var MemRegion; size: int) = # 8MB, 16MB, 32MB, 64MB, 128MB, 512MB, 1GB, 2GB, 4GB, 8GB, # 16GB --> list contains only 20 elements! That's reasonable. if (r.totalSize and 1) == 0: - r.nextChunkSize = - if r.totalSize < 64 * 1024: PageSize*4 - else: r.nextChunkSize*2 + r.nextChunkSize = if r.totalSize < 64 * 1024: PageSize*nimMinHeapPages + else: r.nextChunkSize*2 var s = roundup(size+sizeof(BaseChunk), PageSize) var fresh: Chunk if s > r.nextChunkSize: @@ -242,6 +258,13 @@ proc deallocAll*() = tlRegion.deallocAll() proc deallocOsPages(r: var MemRegion) = r.deallocAll() template withScratchRegion*(body: untyped) = + let obs = obstackPtr() + try: + body + finally: + setObstackPtr(obs) + +when false: var scratch: MemRegion let oldRegion = tlRegion tlRegion = scratch From 9985706fb812156fa6f805989063293ca71599df Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 19 Jan 2019 12:47:44 +0100 Subject: [PATCH 161/350] GC tests: make them take less time to save CI cycles --- tests/gc/gcleak.nim | 2 +- tests/gc/tlists.nim | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/gc/gcleak.nim b/tests/gc/gcleak.nim index 24ac1036a2..0b2e6e14db 100644 --- a/tests/gc/gcleak.nim +++ b/tests/gc/gcleak.nim @@ -12,7 +12,7 @@ type proc makeObj(): TTestObj = result.x = "Hello" -for i in 1 .. 1_000_000: +for i in 1 .. 100_000: when defined(gcMarkAndSweep) or defined(boehmgc): GC_fullcollect() var obj = makeObj() diff --git a/tests/gc/tlists.nim b/tests/gc/tlists.nim index 26b32396c0..959cc5f7c4 100644 --- a/tests/gc/tlists.nim +++ b/tests/gc/tlists.nim @@ -10,15 +10,13 @@ import lists import strutils proc mkleak() = - # allocate 10 MB via linked lists + # allocate 1 MB via linked lists let numberOfLists = 100 for i in countUp(1, numberOfLists): var leakList = initDoublyLinkedList[string]() - let numberOfLeaks = 50000 + let numberOfLeaks = 5000 for j in countUp(1, numberOfLeaks): - let leakSize = 200 - let leaked = newString(leakSize) - leakList.append(leaked) + leakList.append(newString(200)) proc mkManyLeaks() = for i in 0..0: @@ -29,7 +27,7 @@ proc mkManyLeaks() = # lists and bring the memory usage down to a few MB's. GC_fullCollect() when false: echo getOccupiedMem() - if getOccupiedMem() > 8 * 200 * 50_000 * 2: + if getOccupiedMem() > 8 * 200 * 5000 * 2: echo GC_getStatistics() quit "leaking" echo "Success" From 2371b4be966c5b6e3682295393a3e1566ac3f06b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 19 Jan 2019 12:48:25 +0100 Subject: [PATCH 162/350] ported havlak and gcbench benchmarks to work with --gc:regions --- tests/gc/gcbench.nim | 68 ++++++++++++++++++++++++-------------------- tests/gc/thavlak.nim | 27 +++++++++++------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/tests/gc/gcbench.nim b/tests/gc/gcbench.nim index 782daf793b..fc5d6864f7 100644 --- a/tests/gc/gcbench.nim +++ b/tests/gc/gcbench.nim @@ -65,55 +65,58 @@ proc newNode(L, r: PNode): PNode = const kStretchTreeDepth = 18 # about 16Mb kLongLivedTreeDepth = 16 # about 4Mb - kArraySize = 500000 # about 4Mb + kArraySize = 500000 # about 4Mb kMinTreeDepth = 4 kMaxTreeDepth = 16 +when not declared(withScratchRegion): + template withScratchRegion(body: untyped) = body + # Nodes used by a tree of a given size -proc TreeSize(i: int): int = return ((1 shl (i + 1)) - 1) +proc treeSize(i: int): int = return ((1 shl (i + 1)) - 1) # Number of iterations to use for a given tree depth -proc NumIters(i: int): int = - return 2 * TreeSize(kStretchTreeDepth) div TreeSize(i) +proc numIters(i: int): int = + return 2 * treeSize(kStretchTreeDepth) div treeSize(i) # Build tree top down, assigning to older objects. -proc Populate(iDepth: int, thisNode: PNode) = +proc populate(iDepth: int, thisNode: PNode) = if iDepth <= 0: return else: new(thisNode.left) new(thisNode.right) - Populate(iDepth-1, thisNode.left) - Populate(iDepth-1, thisNode.right) + populate(iDepth-1, thisNode.left) + populate(iDepth-1, thisNode.right) # Build tree bottom-up -proc MakeTree(iDepth: int): PNode = +proc makeTree(iDepth: int): PNode = if iDepth <= 0: new(result) else: - return newNode(MakeTree(iDepth-1), MakeTree(iDepth-1)) + return newNode(makeTree(iDepth-1), makeTree(iDepth-1)) -proc PrintDiagnostics() = +proc printDiagnostics() = echo("Total memory available: " & $getTotalMem() & " bytes") echo("Free memory: " & $getFreeMem() & " bytes") -proc TimeConstruction(depth: int) = +proc timeConstruction(depth: int) = var root, tempTree: PNode iNumIters: int - iNumIters = NumIters(depth) + iNumIters = numIters(depth) echo("Creating " & $iNumIters & " trees of depth " & $depth) var t = epochTime() for i in 0..iNumIters-1: new(tempTree) - Populate(depth, tempTree) + populate(depth, tempTree) tempTree = nil echo("\tTop down construction took " & $(epochTime() - t) & "msecs") t = epochTime() for i in 0..iNumIters-1: - tempTree = MakeTree(depth) + tempTree = makeTree(depth) tempTree = nil echo("\tBottom up construction took " & $(epochTime() - t) & "msecs") @@ -127,39 +130,42 @@ proc main() = echo("Garbage Collector Test") echo(" Stretching memory with a binary tree of depth " & $kStretchTreeDepth) - PrintDiagnostics() + printDiagnostics() var t = epochTime() # Stretch the memory space quickly - tempTree = MakeTree(kStretchTreeDepth) - tempTree = nil + withScratchRegion: + tempTree = makeTree(kStretchTreeDepth) + tempTree = nil # Create a long lived object echo(" Creating a long-lived binary tree of depth " & $kLongLivedTreeDepth) new(longLivedTree) - Populate(kLongLivedTreeDepth, longLivedTree) + populate(kLongLivedTreeDepth, longLivedTree) # Create long-lived array, filling half of it echo(" Creating a long-lived array of " & $kArraySize & " doubles") - newSeq(myarray, kArraySize) - for i in 0..kArraySize div 2 - 1: - myarray[i] = 1.0 / toFloat(i) + withScratchRegion: + newSeq(myarray, kArraySize) + for i in 0..kArraySize div 2 - 1: + myarray[i] = 1.0 / toFloat(i) - PrintDiagnostics() + printDiagnostics() - var d = kMinTreeDepth - while d <= kMaxTreeDepth: - TimeConstruction(d) - inc(d, 2) + var d = kMinTreeDepth + while d <= kMaxTreeDepth: + withScratchRegion: + timeConstruction(d) + inc(d, 2) - if longLivedTree == nil or myarray[1000] != 1.0/1000.0: - echo("Failed") - # fake reference to LongLivedTree - # and array to keep them from being optimized away + if longLivedTree == nil or myarray[1000] != 1.0/1000.0: + echo("Failed") + # fake reference to LongLivedTree + # and array to keep them from being optimized away var elapsed = epochTime() - t - PrintDiagnostics() + printDiagnostics() echo("Completed in " & $elapsed & "ms. Success!") when defined(GC_setMaxPause): diff --git a/tests/gc/thavlak.nim b/tests/gc/thavlak.nim index 09c07785e8..2d8df7c1a3 100644 --- a/tests/gc/thavlak.nim +++ b/tests/gc/thavlak.nim @@ -12,9 +12,10 @@ Found 1 loops (including artificial root node) (5)''' # bug #3184 -import tables -import sequtils -import sets +import tables, sequtils, sets, strutils + +when not declared(withScratchRegion): + template withScratchRegion(body: untyped) = body type BasicBlock = object @@ -418,8 +419,9 @@ proc run(self: var LoopTesterApp) = echo "15000 dummy loops" for i in 1..15000: - var h = newHavlakLoopFinder(self.cfg, newLsg()) - var res = h.findLoops + withScratchRegion: + var h = newHavlakLoopFinder(self.cfg, newLsg()) + var res = h.findLoops echo "Constructing CFG..." var n = 2 @@ -446,12 +448,17 @@ proc run(self: var LoopTesterApp) = var sum = 0 for i in 1..5: - write stdout, "." - flushFile(stdout) - var hlf = newHavlakLoopFinder(self.cfg, newLsg()) - sum += hlf.findLoops - #echo getOccupiedMem() + withScratchRegion: + write stdout, "." + flushFile(stdout) + var hlf = newHavlakLoopFinder(self.cfg, newLsg()) + sum += hlf.findLoops + #echo getOccupiedMem() echo "\nFound ", loops, " loops (including artificial root node) (", sum, ")" + when false: + echo("Total memory available: " & formatSize(getTotalMem()) & " bytes") + echo("Free memory: " & formatSize(getFreeMem()) & " bytes") + var l = newLoopTesterApp() l.run From 15aea78405f7e3ba8777984d9531166b6dfa00ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?= Date: Sat, 19 Jan 2019 15:21:07 +0100 Subject: [PATCH 163/350] better docs: base64 (#10371) --- lib/pure/base64.nim | 84 +++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index bfb8a16663..427f93926e 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -9,37 +9,49 @@ ## This module implements a base64 encoder and decoder. ## +## Base64 is an encoding and decoding technique used to convert binary +## data to an ASCII string format. +## Each Base64 digit represents exactly 6 bits of data. Three 8-bit +## bytes (i.e., a total of 24 bits) can therefore be represented by +## four 6-bit Base64 digits. +## +## +## Basic usage +## =========== +## ## Encoding data ## ------------- ## -## In order to encode some text simply call the ``encode`` procedure: -## -## .. code-block::nim -## import base64 -## let encoded = encode("Hello World") -## echo(encoded) # SGVsbG8gV29ybGQ= +## .. code-block::nim +## import base64 +## let encoded = encode("Hello World") +## assert encoded == "SGVsbG8gV29ybGQ=" ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim -## import base64 -## let encodedInts = encode([1,2,3]) -## echo(encodedInts) # AQID -## let encodedChars = encode(['h','e','y']) -## echo(encodedChars) # aGV5 +## .. code-block::nim +## import base64 +## let encodedInts = encode([1,2,3]) +## assert encodedInts == "AQID" +## let encodedChars = encode(['h','e','y']) +## assert encodedChars == "aGV5" ## -## The ``encode`` procedure takes an ``openarray`` so both arrays and sequences -## can be passed as parameters. ## ## Decoding data ## ------------- ## -## To decode a base64 encoded data string simply call the ``decode`` -## procedure: +## .. code-block::nim +## import base64 +## let decoded = decode("SGVsbG8gV29ybGQ=") +## assert decoded == "Hello World" ## -## .. code-block::nim -## import base64 -## echo(decode("SGVsbG8gV29ybGQ=")) # Hello World +## +## See also +## ======== +## +## * `hashes module`_ for efficient computations of hash values for diverse Nim types +## * `md5 module`_ implements the MD5 checksum algorithm +## * `sha1 module`_ implements a sha1 encoder and decoder const cb64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" @@ -100,18 +112,33 @@ template encodeInternal(s: typed, lineLen: int, newLine: string): untyped = discard proc encode*[T:SomeInteger|char](s: openarray[T], lineLen = 75, newLine="\13\10"): string = - ## encodes `s` into base64 representation. After `lineLen` characters, a - ## `newline` is added. + ## Encodes ``s`` into base64 representation. After ``lineLen`` characters, a + ## ``newline`` is added. ## ## This procedure encodes an openarray (array or sequence) of either integers ## or characters. + ## + ## **See also:** + ## * `encode proc<#encode,string,int,string>`_ for encoding a string + ## * `decode proc<#decode,string>`_ for decoding a string + runnableExamples: + assert encode(['n', 'i', 'm']) == "bmlt" + assert encode(@['n', 'i', 'm']) == "bmlt" + assert encode([1, 2, 3, 4, 5]) == "AQIDBAU=" encodeInternal(s, lineLen, newLine) proc encode*(s: string, lineLen = 75, newLine="\13\10"): string = - ## encodes `s` into base64 representation. After `lineLen` characters, a - ## `newline` is added. + ## Encodes ``s`` into base64 representation. After ``lineLen`` characters, a + ## ``newline`` is added. ## ## This procedure encodes a string. + ## + ## **See also:** + ## * `encode proc<#encode,openArray[T],int,string>`_ for encoding an openarray + ## * `decode proc<#decode,string>`_ for decoding a string + runnableExamples: + assert encode("Hello World") == "SGVsbG8gV29ybGQ=" + assert encode("Hello World", 3, "\n") == "SGVs\nbG8g\nV29ybGQ=" encodeInternal(s, lineLen, newLine) proc decodeByte(b: char): int {.inline.} = @@ -123,8 +150,15 @@ proc decodeByte(b: char): int {.inline.} = else: result = 63 proc decode*(s: string): string = - ## decodes a string in base64 representation back into its original form. - ## Whitespace is skipped. + ## Decodes string ``s`` in base64 representation back into its original form. + ## The initial whitespace is skipped. + ## + ## **See also:** + ## * `encode proc<#encode,openArray[T],int,string>`_ for encoding an openarray + ## * `encode proc<#encode,string,int,string>`_ for encoding a string + runnableExamples: + assert decode("SGVsbG8gV29ybGQ=") == "Hello World" + assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} var total = ((len(s) + 3) div 4) * 3 # total is an upper bound, as we will skip arbitrary whitespace: From 4a04470450ab6e6ca59d427837f4bef0c9aea189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?= Date: Sat, 19 Jan 2019 15:24:39 +0100 Subject: [PATCH 164/350] better docs: sha1 (#10374) --- lib/std/sha1.nim | 65 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index e7e6697cf2..b5660f244a 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -7,7 +7,32 @@ # distribution, for details about the copyright. # -## Note: Import ``std/sha1`` to use this module +## **Note:** Import ``std/sha1`` to use this module +## +## SHA-1 (Secure Hash Algorithm 1) is a cryptographic hash function which +## takes an input and produces a 160-bit (20-byte) hash value known as a +## message digest. +## +## .. code-block:: +## import std/sha1 +## +## let accessName = secureHash("John Doe") +## assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" +## +## .. code-block:: +## import std/sha1 +## +## let +## a = secureHashFile("myFile.nim") +## b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") +## +## if a == b: +## echo "Files match" +## +## **See also:** +## * `base64 module`_ implements a base64 encoder and decoder +## * `hashes module`_ for efficient computations of hash values for diverse Nim types +## * `md5 module`_ implements the MD5 checksum algorithm import strutils from endians import bigEndian32, bigEndian64 @@ -170,23 +195,61 @@ proc finalize(ctx: var Sha1State): Sha1Digest = # Public API proc secureHash*(str: string): SecureHash = + ## Generates a ``SecureHash`` from a ``str``. + ## + ## **See also:** + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a ``SecureHash`` from a file + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string ``hash`` to ``SecureHash`` + runnableExamples: + let hash = secureHash("Hello World") + assert hash == parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") var state = newSha1State() state.update(str) SecureHash(state.finalize()) proc secureHashFile*(filename: string): SecureHash = + ## Generates a ``SecureHash`` from a file. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,string>`_ for generating a ``SecureHash`` from a string + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string ``hash`` to ``SecureHash`` secureHash(readFile(filename)) proc `$`*(self: SecureHash): string = + ## Returns the string representation of a ``SecureHash``. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,string>`_ for generating a ``SecureHash`` from a string + runnableExamples: + let hash = secureHash("Hello World") + assert $hash == "0A4D55A8D778E5022FAB701977C5D840BBC486D0" result = "" for v in Sha1Digest(self): result.add(toHex(int(v), 2)) proc parseSecureHash*(hash: string): SecureHash = + ## Converts a string ``hash`` to ``SecureHash``. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,string>`_ for generating a ``SecureHash`` from a string + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a ``SecureHash`` from a file + runnableExamples: + let + hashStr = "0A4D55A8D778E5022FAB701977C5D840BBC486D0" + secureHash = secureHash("Hello World") + assert secureHash == parseSecureHash(hashStr) for i in 0 ..< Sha1DigestSize: Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) proc `==`*(a, b: SecureHash): bool = + ## Checks if two ``SecureHash`` values are identical. + runnableExamples: + let + a = secureHash("Hello World") + b = secureHash("Goodbye World") + c = parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") + assert a != b + assert a == c # Not a constant-time comparison, but that's acceptable in this context Sha1Digest(a) == Sha1Digest(b) From bfeade97917da667f34704de205633dcfe911f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?= Date: Sat, 19 Jan 2019 15:31:14 +0100 Subject: [PATCH 165/350] better docs: uri (#10373) --- lib/pure/uri.nim | 147 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 112 insertions(+), 35 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 3347ed5462..bfb411fc84 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -8,6 +8,35 @@ # ## This module implements URI parsing as specified by RFC 3986. +## +## A Uniform Resource Identifier (URI) provides a simple and extensible +## means for identifying a resource. A URI can be further classified +## as a locator, a name, or both. The term “Uniform Resource Locator” +## (URL) refers to the subset of URIs. +## +## Basic usage +## =========== +## +## Combine URIs +## ------------- +## .. code-block:: +## import uri +## let host = parseUri("https://nim-lang.org") +## let blog = "/blog.html" +## let bloguri = host / blog +## assert $host == "https://nim-lang.org" +## assert $bloguri == "https://nim-lang.org/blog.html" +## +## Access URI item +## --------------- +## .. code-block:: +## import uri +## let res = parseUri("sftp://127.0.0.1:4343") +## if isAbsolute(res): +## echo "Connect to port: " & res.port +## # --> Connect to port: 4343 +## else: +## echo "Wrong format" import strutils, parseutils type @@ -24,11 +53,18 @@ proc encodeUrl*(s: string, usePlus=true): string = ## This means that characters in the set ## ``{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}`` are ## carried over to the result. - ## All other characters are encoded as ``''%xx'`` where ``xx`` + ## All other characters are encoded as ``%xx`` where ``xx`` ## denotes its hexadecimal value. ## ## As a special rule, when the value of ``usePlus`` is true, - ## spaces are encoded as ``'+'`` instead of ``'%20'``. + ## spaces are encoded as ``+`` instead of ``%20``. + ## + ## **See also:** + ## * `decodeUrl proc<#decodeUrl,string>`_ + runnableExamples: + assert encodeUrl("https://nim-lang.org") == "https%3A%2F%2Fnim-lang.org" + assert encodeUrl("https://nim-lang.org/this is a test") == "https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test" + assert encodeUrl("https://nim-lang.org/this is a test", false) == "https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test" result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars let fromSpace = if usePlus: "+" else: "%20" for c in s: @@ -43,12 +79,19 @@ proc encodeUrl*(s: string, usePlus=true): string = proc decodeUrl*(s: string, decodePlus=true): string = ## Decodes a URL according to RFC3986. ## - ## This means that any ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## This means that any ``%xx`` (where ``xx`` denotes a hexadecimal ## value) are converted to the character with ordinal number ``xx``, ## and every other character is carried over. ## - ## As a special rule, when the value of ``decodePlus`` is true, ``'+'`` + ## As a special rule, when the value of ``decodePlus`` is true, ``+`` ## characters are converted to a space. + ## + ## **See also:** + ## * `encodeUrl proc<#encodeUrl,string>`_ + runnableExamples: + assert decodeUrl("https%3A%2F%2Fnim-lang.org") == "https://nim-lang.org" + assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test") == "https://nim-lang.org/this is a test" + assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test", false) == "https://nim-lang.org/this is a test" proc handleHexChar(c: char, x: var int) {.inline.} = case c of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) @@ -123,7 +166,14 @@ proc parsePath(uri: string, i: var int, result: var Uri) = i.inc parseUntil(uri, result.anchor, {}, i) proc initUri*(): Uri = - ## Initializes a URI. + ## Initializes a URI with ``scheme``, ``username``, ``password``, + ## ``hostname``, ``port``, ``path``, ``query`` and ``anchor``. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + runnableExamples: + var uri: Uri + assert initUri() == uri result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", path: "", query: "", anchor: "") @@ -136,6 +186,16 @@ proc resetUri(uri: var Uri) = proc parseUri*(uri: string, result: var Uri) = ## Parses a URI. The `result` variable will be cleared before. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + ## * `initUri proc <#initUri,>`_ for initializing a URI + runnableExamples: + var res = initUri() + parseUri("https://nim-lang.org/docs/manual.html", res) + assert res.scheme == "https" + assert res.hostname == "nim-lang.org" + assert res.path == "/docs/manual.html" resetUri(result) var i = 0 @@ -174,6 +234,14 @@ proc parseUri*(uri: string, result: var Uri) = proc parseUri*(uri: string): Uri = ## Parses a URI and returns it. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + runnableExamples: + let res = parseUri("ftp://Username:Password@Hostname") + assert res.username == "Username" + assert res.password == "Password" + assert res.scheme == "ftp" result = initUri() parseUri(uri, result) @@ -224,22 +292,18 @@ proc combine*(base: Uri, reference: Uri): Uri = ## This uses the algorithm specified in ## `section 5.2.2 of RFC 3986 `_. ## - ## This means that the slashes inside the base URI's path as well as reference - ## URI's path affect the resulting URI. + ## This means that the slashes inside the base URIs path as well as reference + ## URIs path affect the resulting URI. ## - ## For building URIs you may wish to use \`/\` instead. - ## - ## Examples: - ## - ## .. code-block:: - ## let foo = combine(parseUri("http://example.com/foo/bar"), parseUri("/baz")) - ## assert foo.path == "/baz" - ## - ## let bar = combine(parseUri("http://example.com/foo/bar"), parseUri("baz")) - ## assert bar.path == "/foo/baz" - ## - ## let bar = combine(parseUri("http://example.com/foo/bar/"), parseUri("baz")) - ## assert bar.path == "/foo/bar/baz" + ## **See also:** + ## * `/ proc <#/,Uri,string>`_ for building URIs + runnableExamples: + let foo = combine(parseUri("https://nim-lang.org/foo/bar"), parseUri("/baz")) + assert foo.path == "/baz" + let bar = combine(parseUri("https://nim-lang.org/foo/bar"), parseUri("baz")) + assert bar.path == "/foo/baz" + let qux = combine(parseUri("https://nim-lang.org/foo/bar/"), parseUri("baz")) + assert qux.path == "/foo/bar/baz" template setAuthority(dest, src): untyped = dest.hostname = src.hostname @@ -275,32 +339,42 @@ proc combine*(base: Uri, reference: Uri): Uri = proc combine*(uris: varargs[Uri]): Uri = ## Combines multiple URIs together. + ## + ## **See also:** + ## * `/ proc <#/,Uri,string>`_ for building URIs + runnableExamples: + let foo = combine(parseUri("https://nim-lang.org/blog.html"), parseUri("/install.html")) + assert foo.hostname == "nim-lang.org" + assert foo.path == "/install.html" result = uris[0] for i in 1 ..< uris.len: result = combine(result, uris[i]) proc isAbsolute*(uri: Uri): bool = - ## returns true if URI is absolute, false otherwise + ## Returns true if URI is absolute, false otherwise. + runnableExamples: + let foo = parseUri("https://nim-lang.org") + assert isAbsolute(foo) == true + let bar = parseUri("nim-lang") + assert isAbsolute(bar) == false return uri.scheme != "" and (uri.hostname != "" or uri.path != "") proc `/`*(x: Uri, path: string): Uri = - ## Concatenates the path specified to the specified URI's path. + ## Concatenates the path specified to the specified URIs path. ## - ## Contrary to the ``combine`` procedure you do not have to worry about - ## the slashes at the beginning and end of the path and URI's path + ## Contrary to the `combine proc <#combine,Uri,Uri>`_ you do not have to worry about + ## the slashes at the beginning and end of the path and URIs path ## respectively. ## - ## Examples: - ## - ## .. code-block:: - ## let foo = parseUri("http://example.com/foo/bar") / "/baz" - ## assert foo.path == "/foo/bar/baz" - ## - ## let bar = parseUri("http://example.com/foo/bar") / "baz" - ## assert bar.path == "/foo/bar/baz" - ## - ## let bar = parseUri("http://example.com/foo/bar/") / "baz" - ## assert bar.path == "/foo/bar/baz" + ## **See also:** + ## * `combine proc <#combine,Uri,Uri>`_ + runnableExamples: + let foo = parseUri("https://nim-lang.org/foo/bar") / "/baz" + assert foo.path == "/foo/bar/baz" + let bar = parseUri("https://nim-lang.org/foo/bar") / "baz" + assert bar.path == "/foo/bar/baz" + let qux = parseUri("https://nim-lang.org/foo/bar/") / "baz" + assert qux.path == "/foo/bar/baz" result = x if result.path.len == 0: @@ -321,6 +395,9 @@ proc `/`*(x: Uri, path: string): Uri = proc `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. + runnableExamples: + let foo = parseUri("https://nim-lang.org") + assert $foo == "https://nim-lang.org" result = "" if u.scheme.len > 0: result.add(u.scheme) From 095eaacf21a80c72d15ba9a7a422ddc192124ae1 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 19 Jan 2019 15:01:27 +0000 Subject: [PATCH 166/350] Fix spelling errors (#10379) --- compiler/semexprs.nim | 2 +- lib/pure/includes/osseps.nim | 4 ++-- lib/pure/net.nim | 38 ++++++++++++++++++------------------ tests/stdlib/ttimes.nim | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 6f2981f632..5e1e4cbbd0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -230,7 +230,7 @@ proc semConv(c: PContext, n: PNode): PNode = # special case to make MyObject(x = 3) produce a nicer error message: if n[1].kind == nkExprEqExpr and targetType.skipTypes(abstractPtrs).kind == tyObject: - localError(c.config, n.info, "object contruction uses ':', not '='") + localError(c.config, n.info, "object construction uses ':', not '='") var op = semExprWithType(c, n.sons[1]) if targetType.isMetaType: let final = inferWithMetatype(c, targetType, op, true) diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim index 9a79fe303d..944ad123ef 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/pure/includes/osseps.nim @@ -85,9 +85,9 @@ elif doslikeFileSystem: const CurDir* = '.' ParDir* = ".." - DirSep* = '\\' # seperator within paths + DirSep* = '\\' # separator within paths AltSep* = '/' - PathSep* = ';' # seperator between paths + PathSep* = ';' # separator between paths FileSystemCaseSensitive* = false ExeExt* = "exe" ScriptExt* = "bat" diff --git a/lib/pure/net.nim b/lib/pure/net.nim index d4c5a88b7f..a5bdf3ce6a 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -233,7 +233,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = var byteCount = 0 currentByte:uint16 = 0 - seperatorValid = false + separatorValid = false result.family = IpAddressFamily.IPv4 @@ -244,20 +244,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = if currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") - seperatorValid = true + separatorValid = true elif addressStr[i] == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: + if not separatorValid or byteCount >= 3: raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") result.address_v4[byteCount] = cast[uint8](currentByte) currentByte = 0 byteCount.inc - seperatorValid = false + separatorValid = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") - if byteCount != 3 or not seperatorValid: + if byteCount != 3 or not separatorValid: raise newException(ValueError, "Invalid IP Address") result.address_v4[byteCount] = cast[uint8](currentByte) @@ -272,7 +272,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = groupCount = 0 currentGroupStart = 0 currentShort:uint32 = 0 - seperatorValid = true + separatorValid = true dualColonGroup = -1 lastWasColon = false v4StartPos = -1 @@ -280,15 +280,15 @@ proc parseIPv6Address(addressStr: string): IpAddress = for i,c in addressStr: if c == ':': - if not seperatorValid: + if not separatorValid: raise newException(ValueError, - "Invalid IP Address. Address contains an invalid seperator") + "Invalid IP Address. Address contains an invalid separator") if lastWasColon: if dualColonGroup != -1: raise newException(ValueError, - "Invalid IP Address. Address contains more than one \"::\" seperator") + "Invalid IP Address. Address contains more than one \"::\" separator") dualColonGroup = groupCount - seperatorValid = false + separatorValid = false elif i != 0 and i != high(addressStr): if groupCount >= 8: raise newException(ValueError, @@ -297,7 +297,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) currentShort = 0 groupCount.inc() - if dualColonGroup != -1: seperatorValid = false + if dualColonGroup != -1: separatorValid = false elif i == 0: # only valid if address starts with :: if addressStr[1] != ':': raise newException(ValueError, @@ -309,11 +309,11 @@ proc parseIPv6Address(addressStr: string): IpAddress = lastWasColon = true currentGroupStart = i + 1 elif c == '.': # Switch to parse IPv4 mode - if i < 3 or not seperatorValid or groupCount >= 7: + if i < 3 or not separatorValid or groupCount >= 7: raise newException(ValueError, "Invalid IP Address") v4StartPos = currentGroupStart currentShort = 0 - seperatorValid = false + separatorValid = false break elif c in strutils.HexDigits: if c in strutils.Digits: # Normal digit @@ -326,14 +326,14 @@ proc parseIPv6Address(addressStr: string): IpAddress = raise newException(ValueError, "Invalid IP Address. Value is out of range") lastWasColon = false - seperatorValid = true + separatorValid = true else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff - if seperatorValid: # Copy remaining data + if separatorValid: # Copy remaining data if groupCount >= 8: raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") @@ -347,19 +347,19 @@ proc parseIPv6Address(addressStr: string): IpAddress = if currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") - seperatorValid = true + separatorValid = true elif c == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: + if not separatorValid or byteCount >= 3: raise newException(ValueError, "Invalid IP Address") result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) currentShort = 0 byteCount.inc() - seperatorValid = false + separatorValid = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") - if byteCount != 3 or not seperatorValid: + if byteCount != 3 or not separatorValid: raise newException(ValueError, "Invalid IP Address") result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) groupCount += 2 diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 32f39953d2..3999c968f8 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -253,7 +253,7 @@ suite "ttimes": parseTestExcp("+1", "mm") parseTestExcp("+1", "ss") - test "_ as a seperator": + test "_ as a separator": discard parse("2000_01_01", "YYYY'_'MM'_'dd") test "dynamic timezone": From 28b3c8d74dcc3b812c2a386af68a39d3f746edcc Mon Sep 17 00:00:00 2001 From: Kobi Date: Sat, 19 Jan 2019 14:31:14 +0200 Subject: [PATCH 167/350] prevent index out of bounds error in oserr.nim --- lib/pure/includes/oserr.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index 9ae06f5bc5..db7d84c1e0 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -59,7 +59,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = e.errorCode = errorCode.int32 e.msg = osErrorMsg(errorCode) if additionalInfo.len > 0: - if e.msg[^1] != '\n': e.msg.add '\n' + if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' e.msg.add "Additional info: " e.msg.addQuoted additionalInfo if e.msg == "": From 32a90f74063c3ee238a4909d8678720d93545014 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jan 2019 23:24:06 -0800 Subject: [PATCH 168/350] fix json bug `[]=` misplaced (#10397) --- lib/pure/json.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 02fa6dc678..cf979e2d0c 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -357,6 +357,11 @@ when false: assert false notin elements, "usage error: only empty sets allowed" assert true notin elements, "usage error: only empty sets allowed" +proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = + ## Sets a field from a `JObject`. + assert(obj.kind == JObject) + obj.fields[key] = val + #[ Note: could use simply: proc `%`*(o: object|tuple): JsonNode @@ -522,11 +527,6 @@ proc contains*(node: JsonNode, val: JsonNode): bool = proc existsKey*(node: JsonNode, key: string): bool {.deprecated: "use hasKey instead".} = node.hasKey(key) ## **Deprecated:** use `hasKey` instead. -proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = - ## Sets a field from a `JObject`. - assert(obj.kind == JObject) - obj.fields[key] = val - proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the From 5491f40d544494d4277d17669ca1fe0713d66f35 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Mon, 21 Jan 2019 14:35:23 +0530 Subject: [PATCH 169/350] fix vccexe not using correct path for detecting vcvarsall (#10364) --- tools/vccexe/vccenv.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/vccexe/vccenv.nim b/tools/vccexe/vccenv.nim index 724fe992a4..bd3b0b30a0 100644 --- a/tools/vccexe/vccenv.nim +++ b/tools/vccexe/vccenv.nim @@ -16,7 +16,7 @@ type vs140 = (140, "VS140COMNTOOLS") ## Visual Studio 2015 const - vcvarsallRelativePath = joinPath("..", "..", "VC", "vcvarsall") ## Relative path from the COMNTOOLS path to the vcvarsall file. + vcvarsallRelativePath = joinPath("..", "..", "VC", "vcvarsall.bat") ## Relative path from the COMNTOOLS path to the vcvarsall file. proc vccEnvVcVarsAllPath*(version: VccEnvVersion = vsUndefined): string = ## Returns the path to the VCC Developer Command Prompt executable for the specified VCC version. @@ -44,4 +44,4 @@ proc vccEnvVcVarsAllPath*(version: VccEnvVersion = vsUndefined): string = let key = $version let val = getEnv key if val.len > 0: - result = try: expandFilename(val & vcvarsallRelativePath) except OSError: "" + result = try: expandFilename(joinPath(val, vcvarsallRelativePath)) except OSError: "" From ea0c1811c52355335b4a884a1c963bfe332aebc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Mon, 21 Jan 2019 14:22:16 +0100 Subject: [PATCH 170/350] gdb pretty printer survive reload --- tools/nim-gdb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index 1a0d89fcbe..fb7eff565c 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -1,4 +1,3 @@ - import gdb import re import sys @@ -16,8 +15,6 @@ def printErrorOnce(id, message): errorSet.add(id) gdb.write(message, gdb.STDERR) -nimobjfile = gdb.current_objfile() or gdb.objfiles()[0] -nimobjfile.type_printers = [] ################################################################################ ##### Type pretty printers @@ -121,9 +118,6 @@ class NimTypePrinter: def instantiate(self): return NimTypeRecognizer() - -nimobjfile.type_printers = [NimTypePrinter()] - ################################################################################ ##### GDB Function, equivalent of Nim's $ operator ################################################################################ @@ -528,5 +522,11 @@ def makematcher(klass): printErrorOnce(typeName, "No matcher for type '" + typeName + "': " + str(e) + "\n") return matcher -nimobjfile.pretty_printers = [] -nimobjfile.pretty_printers.extend([makematcher(var) for var in list(vars().values()) if hasattr(var, 'pattern')]) +def new_object_handler(event): + nimobjfile = gdb.current_objfile() or gdb.objfiles()[0] + nimobjfile.type_printers = [] + nimobjfile.type_printers = [NimTypePrinter()] + nimobjfile.pretty_printers = [] + nimobjfile.pretty_printers.extend([makematcher(var) for var in list(globals().values()) if hasattr(var, 'pattern')]) + +gdb.events.new_objfile.connect(new_object_handler) From 9a003bae06014beb017c7a77b351f676cee6ce5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Mon, 21 Jan 2019 15:14:38 +0100 Subject: [PATCH 171/350] Fix error lexer error messages for to large numbers (#10394) --- compiler/lexer.nim | 60 ++++++++++++++--------------- tests/errmsgs/tinteger_literals.nim | 15 ++++++++ 2 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 tests/errmsgs/tinteger_literals.nim diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 635e6f08da..a4414d1864 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -17,7 +17,7 @@ import hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream, - wordrecg, lineinfos, pathutils + wordrecg, lineinfos, pathutils, parseutils const MaxLineLength* = 80 # lines longer than this lead to a warning @@ -307,20 +307,6 @@ template tokenEndPrevious(tok, pos) = when defined(nimpretty): tok.offsetB = L.offsetBase + pos -{.push overflowChecks: off.} -# We need to parse the largest uint literal without overflow checks -proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int = - var i = start - if i < s.len and s[i] in {'0'..'9'}: - b = 0 - while i < s.len and s[i] in {'0'..'9'}: - b = b * 10 + (ord(s[i]) - ord('0')) - inc(i) - while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - result = i - start -{.pop.} # overflowChecks - - template eatChar(L: var TLexer, t: var TToken, replacementChar: char) = add(t.literal, replacementChar) inc(L.bufpos) @@ -586,33 +572,43 @@ proc getNumber(L: var TLexer, result: var TToken) = of floatTypes: result.fNumber = parseFloat(result.literal) of tkUint64Lit: - xi = 0 - let len = unsafeParseUInt(result.literal, xi) - if len != result.literal.len or len == 0: - raise newException(ValueError, "invalid integer: " & $xi) - result.iNumber = xi + var iNumber: uint64 + var len: int + try: + len = parseBiggestUInt(result.literal, iNumber) + except ValueError: + raise newException(OverflowError, "number out of range: " & $result.literal) + if len != result.literal.len: + raise newException(ValueError, "invalid integer: " & $result.literal) + result.iNumber = cast[int64](iNumber) else: - result.iNumber = parseBiggestInt(result.literal) + var iNumber: int64 + var len: int + try: + len = parseBiggestInt(result.literal, iNumber) + except ValueError: + raise newException(OverflowError, "number out of range: " & $result.literal) + if len != result.literal.len: + raise newException(ValueError, "invalid integer: " & $result.literal) + result.iNumber = iNumber - # Explicit bounds checks + # Explicit bounds checks. Only T.high needs to be considered + # since result.iNumber can't be negative. let outOfRange = case result.tokType - of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high) - of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or - result.iNumber > BiggestInt(uint8.high)) - of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high) - of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or - result.iNumber > BiggestInt(uint16.high)) - of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high) - of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or - result.iNumber > BiggestInt(uint32.high)) + of tkInt8Lit: result.iNumber > int8.high + of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) + of tkInt16Lit: result.iNumber > int16.high + of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) + of tkInt32Lit: result.iNumber > int32.high + of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) else: false if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos) # Promote int literal to int64? Not always necessary, but more consistent if result.tokType == tkIntLit: - if (result.iNumber < low(int32)) or (result.iNumber > high(int32)): + if result.iNumber > high(int32): result.tokType = tkInt64Lit except ValueError: diff --git a/tests/errmsgs/tinteger_literals.nim b/tests/errmsgs/tinteger_literals.nim new file mode 100644 index 0000000000..98c92a2277 --- /dev/null +++ b/tests/errmsgs/tinteger_literals.nim @@ -0,0 +1,15 @@ +discard """ +cmd: "nim check $file" +errormsg: "number out of range: '300'u8'" +nimout: ''' +tinteger_literals.nim(12, 9) Error: number out of range: '18446744073709551616'u64' +tinteger_literals.nim(13, 9) Error: number out of range: '9223372036854775808'i64' +tinteger_literals.nim(14, 9) Error: number out of range: '9223372036854775808' +tinteger_literals.nim(15, 9) Error: number out of range: '300'u8' +''' +""" + +discard 18446744073709551616'u64 # high(uint64) + 1 +discard 9223372036854775808'i64 # high(int64) + 1 +discard 9223372036854775808 # high(int64) + 1 +discard 300'u8 \ No newline at end of file From 4bea8dd674c87b49195220bd6c5d72def467d8e5 Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 21 Jan 2019 15:22:53 +0100 Subject: [PATCH 172/350] better docs for lists and deques (#10390) * better docs: lists * better docs: deques --- lib/pure/collections/deques.nim | 302 +++++++++++++++--- lib/pure/collections/lists.nim | 533 +++++++++++++++++++++++++++++--- 2 files changed, 757 insertions(+), 78 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 5243b96a64..cb05e5112b 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -20,41 +20,59 @@ ## access, unless your program logic guarantees it indirectly. ## ## .. code-block:: Nim -## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` -## var deq = initDeque[int]() # initializes the object -## for i in 1 ..< a: deq.addLast i # populates the deque +## import deques ## -## if b < deq.len: # checking before indexed access -## echo "The element at index position ", b, " is ", deq[b] +## var a = initDeque[int]() ## -## # The following two lines don't need any checking on access due to the -## # logic of the program, but that would not be the case if `a` could be 0. -## assert deq.peekFirst == 1 -## assert deq.peekLast == a +## doAssertRaises(IndexError, echo a[0]) ## -## while deq.len > 0: # checking if the deque is empty -## echo deq.popLast() +## for i in 1 .. 5: +## a.addLast(10*i) +## assert $a == "[10, 20, 30, 40, 50]" ## -## Note: For inter thread communication use -## a `Channel `_ instead. +## assert a.peekFirst == 10 +## assert a.peekLast == 50 +## assert len(a) == 5 +## +## assert a.popFirst == 10 +## assert a.popLast == 50 +## assert len(a) == 3 +## +## a.addFirst(11) +## a.addFirst(22) +## a.addFirst(33) +## assert $a == "[33, 22, 11, 20, 30, 40]" +## +## a.shrink(fromFirst = 1, fromLast = 2) +## assert $a == "[22, 11, 20]" +## +## +## **See also:** +## * `lists module `_ for singly and doubly linked lists and rings +## * `channels module `_ for inter-thread communication + import math, typetraits type Deque*[T] = object ## A double-ended queue backed with a ringed seq buffer. + ## + ## To initialize an empty deque use `initDeque proc <#initDeque,int>`_. data: seq[T] head, tail, count, mask: int proc initDeque*[T](initialSize: int = 4): Deque[T] = - ## Create a new deque. - ## Optionally, the initial capacity can be reserved via `initialSize` as a - ## performance optimization. The length of a newly created deque will still - ## be 0. + ## Create a new empty deque. ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math `_ module. + ## Optionally, the initial capacity can be reserved via `initialSize` + ## as a performance optimization. + ## The length of a newly created deque will still be 0. + ## + ## ``initialSize`` must be a power of two (default: 4). + ## If you need to accept runtime values for this you could use the + ## `nextPowerOfTwo proc`_ from the + ## `math module`_. assert isPowerOfTwo(initialSize) result.mask = initialSize-1 newSeq(result.data, initialSize) @@ -75,45 +93,128 @@ template xBoundsCheck(deq, i) = if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter raise newException(IndexError, "Out of bounds: " & $i & " > " & $(deq.count - 1)) + if unlikely(i < 0): # when used with BackwardsIndex + raise newException(IndexError, + "Out of bounds: " & $i & " < 0") proc `[]`*[T](deq: Deque[T], i: Natural) : T {.inline.} = - ## Access the i-th element of `deq` by order from first to last. - ## deq[0] is the first, deq[^1] is the last. + ## Access the i-th element of `deq`. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert a[0] == 10 + assert a[3] == 40 + doAssertRaises(IndexError, echo a[8]) + xBoundsCheck(deq, i) return deq.data[(deq.head + i) and deq.mask] proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = - ## Access the i-th element of `deq` and returns a mutable + ## Access the i-th element of `deq` and return a mutable ## reference to it. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert a[0] == 10 + assert a[3] == 40 + doAssertRaises(IndexError, echo a[8]) + xBoundsCheck(deq, i) return deq.data[(deq.head + i) and deq.mask] proc `[]=`*[T](deq: var Deque[T], i: Natural, val : T) {.inline.} = ## Change the i-th element of `deq`. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + a[0] = 99 + a[3] = 66 + assert $a == "[99, 20, 30, 66, 50]" + xBoundsCheck(deq, i) deq.data[(deq.head + i) and deq.mask] = val -proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = - ## Access the backwards indexed i-th element. - return deq[deq.len - int(i)] - proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = ## Access the backwards indexed i-th element. + ## + ## `deq[^1]` is the last element. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert a[^1] == 50 + assert a[^4] == 20 + doAssertRaises(IndexError, echo a[^9]) + + xBoundsCheck(deq, deq.len - int(i)) + return deq[deq.len - int(i)] + +proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = + ## Access the backwards indexed i-th element. + ## + ## `deq[^1]` is the last element. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert a[^1] == 50 + assert a[^4] == 20 + doAssertRaises(IndexError, echo a[^9]) + + xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = ## Change the backwards indexed i-th element. + ## + ## `deq[^1]` is the last element. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + a[^1] = 99 + a[^3] = 77 + assert $a == "[10, 20, 77, 40, 99]" + + xBoundsCheck(deq, deq.len - int(i)) deq[deq.len - int(i)] = x iterator items*[T](deq: Deque[T]): T = ## Yield every element of `deq`. + ## + ## **Examples:** + ## + ## .. code-block:: + ## var a = initDeque[int]() + ## for i in 1 .. 3: + ## a.addLast(10*i) + ## + ## for x in a: # the same as: for x in items(a): + ## echo x + ## + ## # 10 + ## # 20 + ## # 30 + ## var i = deq.head for c in 0 ..< deq.count: yield deq.data[i] i = (i + 1) and deq.mask iterator mitems*[T](deq: var Deque[T]): var T = - ## Yield every element of `deq`. + ## Yield every element of `deq`, which can be modified. + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5*x - 1 + assert $a == "[49, 99, 149, 199, 249]" + var i = deq.head for c in 0 ..< deq.count: yield deq.data[i] @@ -121,18 +222,35 @@ iterator mitems*[T](deq: var Deque[T]): var T = iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = ## Yield every (position, value) of `deq`. + ## + ## **Examples:** + ## + ## .. code-block:: + ## var a = initDeque[int]() + ## for i in 1 .. 3: + ## a.addLast(10*i) + ## + ## for k, v in pairs(a): + ## echo "key: ", k, ", value: ", v + ## + ## # key: 0, value: 10 + ## # key: 1, value: 20 + ## # key: 2, value: 30 + ## var i = deq.head for c in 0 ..< deq.count: yield (c, deq.data[i]) i = (i + 1) and deq.mask proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = - ## Return true if `item` is in `deq` or false if not found. Usually used - ## via the ``in`` operator. It is the equivalent of ``deq.find(item) >= 0``. + ## Return true if `item` is in `deq` or false if not found. + ## + ## Usually used via the ``in`` operator. + ## It is the equivalent of ``deq.find(item) >= 0``. ## ## .. code-block:: Nim ## if x in q: - ## assert q.contains x + ## assert q.contains(x) for e in deq: if e == item: return true return false @@ -150,6 +268,19 @@ proc expandIfNeeded[T](deq: var Deque[T]) = proc addFirst*[T](deq: var Deque[T], item: T) = ## Add an `item` to the beginning of the `deq`. + ## + ## See also: + ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addFirst(10*i) + assert $a == "[50, 40, 30, 20, 10]" + expandIfNeeded(deq) inc deq.count deq.head = (deq.head - 1) and deq.mask @@ -157,6 +288,19 @@ proc addFirst*[T](deq: var Deque[T], item: T) = proc addLast*[T](deq: var Deque[T], item: T) = ## Add an `item` to the end of the `deq`. + ## + ## See also: + ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + expandIfNeeded(deq) inc deq.count deq.data[deq.tail] = item @@ -164,11 +308,41 @@ proc addLast*[T](deq: var Deque[T], item: T) = proc peekFirst*[T](deq: Deque[T]): T {.inline.}= ## Returns the first element of `deq`, but does not remove it from the deque. + ## + ## See also: + ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + assert a.peekFirst == 10 + assert len(a) == 5 + emptyCheck(deq) result = deq.data[deq.head] proc peekLast*[T](deq: Deque[T]): T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. + ## + ## See also: + ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + assert a.peekLast == 50 + assert len(a) == 5 + emptyCheck(deq) result = deq.data[(deq.tail - 1) and deq.mask] @@ -177,6 +351,23 @@ template destroy(x: untyped) = proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Remove and returns the first element of the `deq`. + ## + ## See also: + ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + ## * `clear proc <#clear,Deque[T]>`_ + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + assert a.popFirst == 10 + assert $a == "[20, 30, 40, 50]" + emptyCheck(deq) dec deq.count result = deq.data[deq.head] @@ -185,6 +376,23 @@ proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Remove and returns the last element of the `deq`. + ## + ## See also: + ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `clear proc <#clear,Deque[T]>`_ + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10*i) + assert $a == "[10, 20, 30, 40, 50]" + assert a.popLast == 50 + assert $a == "[10, 20, 30, 40]" + emptyCheck(deq) dec deq.count deq.tail = (deq.tail - 1) and deq.mask @@ -193,17 +401,39 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. + ## + ## See also: + ## * `clear proc <#clear,Deque[T]>`_ + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addFirst(10*i) + assert $a == "[50, 40, 30, 20, 10]" + clear(a) + assert len(a) == 0 + for el in mitems(deq): destroy(el) deq.count = 0 deq.tail = deq.head proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = ## Remove `fromFirst` elements from the front of the deque and - ## `fromLast` elements from the back. If the supplied number of - ## elements exceeds the total number of elements in the deque, - ## the deque will remain empty. + ## `fromLast` elements from the back. ## - ## Any user defined destructors + ## If the supplied number of elements exceeds the total number of elements + ## in the deque, the deque will remain empty. + ## + ## See also: + ## * `clear proc <#clear,Deque[T]>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addFirst(10*i) + assert $a == "[50, 40, 30, 20, 10]" + a.shrink(fromFirst = 2, fromLast = 1) + assert $a == "[30, 20]" + if fromFirst + fromLast > deq.count: clear(deq) return @@ -226,6 +456,8 @@ proc `$`*[T](deq: Deque[T]): string = result.addQuoted(x) result.add("]") + + when isMainModule: var deq = initDeque[int](1) deq.addLast(4) diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 15ce5d0741..182eb8442d 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -7,34 +7,112 @@ # distribution, for details about the copyright. # -## Implementation of singly and doubly linked lists. Because it makes no sense -## to do so, the 'next' and 'prev' pointers are not hidden from you and can -## be manipulated directly for efficiency. +## Implementation of: +## * `singly linked lists <#SinglyLinkedList>`_ +## * `doubly linked lists <#DoublyLinkedList>`_ +## * `singly linked rings <#SinglyLinkedRing>`_ (circular lists) +## * `doubly linked rings <#DoublyLinkedRing>`_ (circular lists) +## +## +## Basic Usage +## =========== +## +## Because it makes no sense to do otherwise, the `next` and `prev` pointers +## are not hidden from you and can be manipulated directly for efficiency. +## +## Lists +## ----- +## +## .. code-block:: +## import lists +## +## var +## l = initDoublyLinkedList[int]() +## a = newDoublyLinkedNode[int](3) +## b = newDoublyLinkedNode[int](7) +## c = newDoublyLinkedNode[int](9) +## +## l.append(a) +## l.append(b) +## l.prepend(c) +## +## assert a.next == b +## assert a.prev == c +## assert c.next == a +## assert c.next.next == b +## assert c.prev == nil +## assert b.next == nil +## +## +## Rings +## ----- +## +## .. code-block:: +## import lists +## +## var +## l = initSinglyLinkedRing[int]() +## a = newSinglyLinkedNode[int](3) +## b = newSinglyLinkedNode[int](7) +## c = newSinglyLinkedNode[int](9) +## +## l.append(a) +## l.append(b) +## l.prepend(c) +## +## assert c.next == a +## assert a.next == b +## assert c.next.next == b +## assert b.next == c +## assert c.next.next.next == c +## +## See also +## ======== +## +## * `deques module <#deques.html>`_ for double-ended queues +## * `sharedlist module <#sharedlist.html>`_ for shared singly-linked lists + when not defined(nimhygiene): {.pragma: dirty.} type - DoublyLinkedNodeObj*[T] = object ## a node a doubly linked list consists of + DoublyLinkedNodeObj*[T] = object ## A node a doubly linked list consists of. + ## + ## It consists of a `value` field, and pointers to `next` and `prev`. next*, prev*: ref DoublyLinkedNodeObj[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] - SinglyLinkedNodeObj*[T] = object ## a node a singly linked list consists of + SinglyLinkedNodeObj*[T] = object ## A node a singly linked list consists of. + ## + ## It consists of a `value` field, and a pointer to `next`. next*: ref SinglyLinkedNodeObj[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] - SinglyLinkedList*[T] = object ## a singly linked list + SinglyLinkedList*[T] = object ## A singly linked list. + ## + ## Use `initSinglyLinkedList proc <#initSinglyLinkedList,>`_ to create + ## a new empty list. head*, tail*: SinglyLinkedNode[T] - DoublyLinkedList*[T] = object ## a doubly linked list + DoublyLinkedList*[T] = object ## A doubly linked list. + ## + ## Use `initDoublyLinkedList proc <#initDoublyLinkedList,>`_ to create + ## a new empty list. head*, tail*: DoublyLinkedNode[T] - SinglyLinkedRing*[T] = object ## a singly linked ring + SinglyLinkedRing*[T] = object ## A singly linked ring. + ## + ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing,>`_ to create + ## a new empty ring. head*, tail*: SinglyLinkedNode[T] - DoublyLinkedRing*[T] = object ## a doubly linked ring + DoublyLinkedRing*[T] = object ## A doubly linked ring. + ## + ## Use `initDoublyLinkedRing proc <#initDoublyLinkedRing,>`_ to create + ## a new empty ring. head*: DoublyLinkedNode[T] SomeLinkedList*[T] = SinglyLinkedList[T] | DoublyLinkedList[T] @@ -46,28 +124,44 @@ type SomeLinkedNode*[T] = SinglyLinkedNode[T] | DoublyLinkedNode[T] proc initSinglyLinkedList*[T](): SinglyLinkedList[T] = - ## creates a new singly linked list that is empty. + ## Creates a new singly linked list that is empty. + runnableExamples: + var a = initSinglyLinkedList[int]() discard proc initDoublyLinkedList*[T](): DoublyLinkedList[T] = - ## creates a new doubly linked list that is empty. + ## Creates a new doubly linked list that is empty. + runnableExamples: + var a = initDoublyLinkedList[int]() discard proc initSinglyLinkedRing*[T](): SinglyLinkedRing[T] = - ## creates a new singly linked ring that is empty. + ## Creates a new singly linked ring that is empty. + runnableExamples: + var a = initSinglyLinkedRing[int]() discard proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = - ## creates a new doubly linked ring that is empty. + ## Creates a new doubly linked ring that is empty. + runnableExamples: + var a = initDoublyLinkedRing[int]() discard proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = - ## creates a new doubly linked node with the given `value`. + ## Creates a new doubly linked node with the given `value`. + runnableExamples: + var n = newDoublyLinkedNode[int](5) + assert n.value == 5 + new(result) result.value = value proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = - ## creates a new singly linked node with the given `value`. + ## Creates a new singly linked node with the given `value`. + runnableExamples: + var n = newSinglyLinkedNode[int](5) + assert n.value == 5 + new(result) result.value = value @@ -86,24 +180,100 @@ template itemsRingImpl() {.dirty.} = if it == L.head: break iterator items*[T](L: SomeLinkedList[T]): T = - ## yields every value of `L`. + ## Yields every value of `L`. + ## + ## See also: + ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## var a = initSinglyLinkedList[int]() + ## for i in 1 .. 3: + ## a.append(10*i) + ## + ## for x in a: # the same as: for x in items(a): + ## echo x + ## + ## # 10 + ## # 20 + ## # 30 itemsListImpl() iterator items*[T](L: SomeLinkedRing[T]): T = - ## yields every value of `L`. + ## Yields every value of `L`. + ## + ## See also: + ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## var a = initSinglyLinkedRing[int]() + ## for i in 1 .. 3: + ## a.append(10*i) + ## + ## for x in a: # the same as: for x in items(a): + ## echo x + ## + ## # 10 + ## # 20 + ## # 30 itemsRingImpl() iterator mitems*[T](L: var SomeLinkedList[T]): var T = - ## yields every value of `L` so that you can modify it. + ## Yields every value of `L` so that you can modify it. + ## + ## See also: + ## * `items iterator <#items.i,SomeLinkedList[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ + runnableExamples: + var a = initSinglyLinkedList[int]() + for i in 1 .. 5: + a.append(10*i) + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5*x - 1 + assert $a == "[49, 99, 149, 199, 249]" itemsListImpl() iterator mitems*[T](L: var SomeLinkedRing[T]): var T = - ## yields every value of `L` so that you can modify it. + ## Yields every value of `L` so that you can modify it. + ## + ## See also: + ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ + runnableExamples: + var a = initSinglyLinkedRing[int]() + for i in 1 .. 5: + a.append(10*i) + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5*x - 1 + assert $a == "[49, 99, 149, 199, 249]" itemsRingImpl() iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the + ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. + ## + ## See also: + ## * `items iterator <#items.i,SomeLinkedList[T]>`_ + ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ + runnableExamples: + var a = initDoublyLinkedList[int]() + for i in 1 .. 5: + a.append(10*i) + assert $a == "[10, 20, 30, 40, 50]" + for x in nodes(a): + if x.value == 30: + a.remove(x) + else: + x.value = 5*x.value - 1 + assert $a == "[49, 99, 199, 249]" + var it = L.head while it != nil: var nxt = it.next @@ -111,8 +281,24 @@ iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = it = nxt iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the + ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. + ## + ## See also: + ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ + ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ + runnableExamples: + var a = initDoublyLinkedRing[int]() + for i in 1 .. 5: + a.append(10*i) + assert $a == "[10, 20, 30, 40, 50]" + for x in nodes(a): + if x.value == 30: + a.remove(x) + else: + x.value = 5*x.value - 1 + assert $a == "[49, 99, 199, 249]" + var it = L.head if it != nil: while true: @@ -122,7 +308,7 @@ iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = if it == L.head: break proc `$`*[T](L: SomeLinkedCollection[T]): string = - ## turns a list into its string representation. + ## Turns a list into its string representation for logging and printing. result = "[" for x in nodes(L): if result.len > 1: result.add(", ") @@ -130,19 +316,54 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string = result.add("]") proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not + ## Searches in the list for a value. Returns `nil` if the value does not ## exist. + ## + ## See also: + ## * `contains proc <#contains,SomeLinkedCollection[T],T>`_ + runnableExamples: + var a = initSinglyLinkedList[int]() + a.append(9) + a.append(8) + assert a.find(9).value == 9 + assert a.find(1) == nil + for x in nodes(L): if x.value == value: return x proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. + ## Searches in the list for a value. Returns `false` if the value does not + ## exist, `true` otherwise. + ## + ## See also: + ## * `find proc <#find,SomeLinkedCollection[T],T>`_ + runnableExamples: + var a = initSinglyLinkedList[int]() + a.append(9) + a.append(8) + assert a.contains(9) + assert 8 in a + assert(not a.contains(1)) + assert 2 notin a + result = find(L, value) != nil proc append*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## appends a node `n` to `L`. Efficiency: O(1). + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var + a = initSinglyLinkedList[int]() + n = newSinglyLinkedNode[int](9) + a.append(n) + assert a.contains(9) + n.next = nil if L.tail != nil: assert(L.tail.next == nil) @@ -151,22 +372,75 @@ proc append*[T](L: var SinglyLinkedList[T], if L.head == nil: L.head = n proc append*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = - ## appends a value to `L`. Efficiency: O(1). + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedList[int]() + a.append(9) + a.append(8) + assert a.contains(9) append(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## prepends a node to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var + a = initSinglyLinkedList[int]() + n = newSinglyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + n.next = L.head L.head = n if L.tail == nil: L.tail = n proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = - ## prepends a node to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + runnableExamples: + var a = initSinglyLinkedList[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) prepend(L, newSinglyLinkedNode(value)) + + proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var + a = initDoublyLinkedList[int]() + n = newDoublyLinkedNode[int](9) + a.append(n) + assert a.contains(9) + n.next = nil n.prev = L.tail if L.tail != nil: @@ -176,11 +450,40 @@ proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = if L.head == nil: L.head = n proc append*[T](L: var DoublyLinkedList[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + a.append(9) + a.append(8) + assert a.contains(9) append(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var + a = initDoublyLinkedList[int]() + n = newDoublyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + n.prev = nil n.next = L.head if L.head != nil: @@ -190,18 +493,56 @@ proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = if L.tail == nil: L.tail = n proc prepend*[T](L: var DoublyLinkedList[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) prepend(L, newDoublyLinkedNode(value)) proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## removes `n` from `L`. Efficiency: O(1). + ## Removes a node `n` from `L`. Efficiency: O(1). + runnableExamples: + var + a = initDoublyLinkedList[int]() + n = newDoublyLinkedNode[int](5) + a.append(n) + assert 5 in a + a.remove(n) + assert 5 notin a + if n == L.tail: L.tail = n.prev if n == L.head: L.head = n.next if n.next != nil: n.next.prev = n.prev if n.prev != nil: n.prev.next = n.next + + proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var + a = initSinglyLinkedRing[int]() + n = newSinglyLinkedNode[int](9) + a.append(n) + assert a.contains(9) + if L.head != nil: n.next = L.head assert(L.tail != nil) @@ -213,11 +554,36 @@ proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = L.tail = n proc append*[T](L: var SinglyLinkedRing[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedRing[int]() + a.append(9) + a.append(8) + assert a.contains(9) append(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var + a = initSinglyLinkedRing[int]() + n = newSinglyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + if L.head != nil: n.next = L.head assert(L.tail != nil) @@ -228,11 +594,40 @@ proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = L.head = n proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + runnableExamples: + var a = initSinglyLinkedRing[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) prepend(L, newSinglyLinkedNode(value)) + + proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var + a = initDoublyLinkedRing[int]() + n = newDoublyLinkedNode[int](9) + a.append(n) + assert a.contains(9) + if L.head != nil: n.next = L.head n.prev = L.head.prev @@ -244,11 +639,40 @@ proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = L.head = n proc append*[T](L: var DoublyLinkedRing[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + a.append(9) + a.append(8) + assert a.contains(9) append(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var + a = initDoublyLinkedRing[int]() + n = newDoublyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + if L.head != nil: n.next = L.head n.prev = L.head.prev @@ -260,11 +684,34 @@ proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = L.head = n proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## See also: + ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) prepend(L, newDoublyLinkedNode(value)) proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## removes `n` from `L`. Efficiency: O(1). + ## Removes `n` from `L`. Efficiency: O(1). + runnableExamples: + var + a = initDoublyLinkedRing[int]() + n = newDoublyLinkedNode[int](5) + a.append(n) + assert 5 in a + a.remove(n) + assert 5 notin a + n.next.prev = n.prev n.prev.next = n.next if n == L.head: From 413755fd45f5a77f9c3323cf7185830249c3f310 Mon Sep 17 00:00:00 2001 From: alaviss Date: Mon, 21 Jan 2019 22:57:48 +0700 Subject: [PATCH 173/350] Correct lineinfo for accent quoted symbols in proc definition (#10399) * compiler/parser: preserve lineinfo for accent quoted symbols Previously the lineinfo for symbol $$$ in this example is: proc `$$$` ^ After this commit: proc `$$$` ^ * compiler/semstmts: correct lineinfo for accent quoted idents Previously nimsuggest would highlight this as: proc `$$$` ^~~ After this commit: proc `$$$` ^~~ * nimsuggest/tests: add a test for accent quoted proc Disabled by default --- compiler/parser.nim | 5 ++++- compiler/semstmts.nim | 20 ++++++++++---------- nimsuggest/tests/taccent_highlight.nim | 8 ++++++++ 3 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 nimsuggest/tests/taccent_highlight.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index afc49c0d37..c9626c527c 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -332,12 +332,15 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode = parMessage(p, errIdentifierExpected, p.tok) break of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi: + let lineinfo = parLineinfo(p) var accm = "" while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi}: accm.add(tokToStr(p.tok)) getTok(p) - result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p)) + let node = newNodeI(nkIdent, lineinfo) + node.ident = p.lex.cache.getIdent(accm) + result.add(node) of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p)) getTok(p) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index de39e95c8d..288820d862 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -318,16 +318,16 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = result = semIdentWithPragma(c, kind, n, {}) if result.owner.kind == skModule: incl(result.flags, sfGlobal) - let info = case n.kind - of nkPostfix: - n.sons[1].info - of nkPragmaExpr: - if n.sons[0].kind == nkPostfix: - n.sons[0].sons[1].info - else: - n.sons[0].info - else: - n.info + + proc getLineInfo(n: PNode): TLineInfo = + case n.kind + of nkPostfix: + getLineInfo(n.sons[1]) + of nkAccQuoted, nkPragmaExpr: + getLineInfo(n.sons[0]) + else: + n.info + let info = getLineInfo(n) suggestSym(c.config, info, result, c.graph.usageSym) proc checkNilable(c: PContext; v: PSym) = diff --git a/nimsuggest/tests/taccent_highlight.nim b/nimsuggest/tests/taccent_highlight.nim new file mode 100644 index 0000000000..2f03d7c860 --- /dev/null +++ b/nimsuggest/tests/taccent_highlight.nim @@ -0,0 +1,8 @@ +proc `$$$`#[!]# + +discard """ +disabled:true +$nimsuggest --tester $file +>highlight $1 +highlight;;skProc;;1;;6;;3 +""" From a4cdd25b19b0ec98826a01e1f57da1c2fb8920af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Mon, 21 Jan 2019 17:00:33 +0100 Subject: [PATCH 174/350] Support system.reset in vm (#10400) --- compiler/vm.nim | 2 -- compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 4 +++- tests/vm/treset.nim | 28 ++++++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 tests/vm/treset.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 180f3800b2..10d38fe77f 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1249,8 +1249,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let newLen = regs[rb].intVal.int if regs[ra].node.isNil: stackTrace(c, tos, pc, errNilAccess) else: c.setLenSeq(regs[ra].node, newLen, c.debug[pc]) - of opcReset: - internalError(c.config, c.debug[pc], "too implement") of opcNarrowS: decodeB(rkInt) let min = -(1.BiggestInt shl (rb-1)) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 493078f742..a43f8dbba5 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -73,7 +73,7 @@ type opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq, opcIsNil, opcOf, opcIs, opcSubStr, opcParseFloat, opcConv, opcCast, - opcQuit, opcReset, + opcQuit, opcNarrowS, opcNarrowU, opcAddStrCh, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 033cc81f0d..e7993dfb21 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1105,7 +1105,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mReset: unused(c, n, dest) var d = c.genx(n.sons[1]) - c.gABC(n, opcReset, d) + c.gABx(n, opcLdNull, d, c.genType(n.sons[1].typ)) + c.gABx(n, opcNodeToReg, d, d) + c.genAsgnPatch(n.sons[1], d) of mOf, mIs: if dest < 0: dest = c.getTemp(n.typ) var tmp = c.genx(n.sons[1]) diff --git a/tests/vm/treset.nim b/tests/vm/treset.nim new file mode 100644 index 0000000000..56fe19b19a --- /dev/null +++ b/tests/vm/treset.nim @@ -0,0 +1,28 @@ +static: + type Obj = object + field: int + var o = Obj(field: 1) + reset(o) + doAssert o.field == 0 + +static: + var i = 2 + reset(i) + doAssert i == 0 + +static: + var i = new int + reset(i) + doAssert i.isNil + +static: + var s = @[1, 2, 3] + reset(s) + doAssert s == @[] + +static: + proc f() = + var i = 2 + reset(i) + doAssert i == 0 + f() \ No newline at end of file From ae5d8fbd9d6b0c6469c2245da8a54d7b4c12f54f Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 21 Jan 2019 17:27:36 +0100 Subject: [PATCH 175/350] Proper check for tyStatic[T] -> U conversions (#10382) Drop the outer tyStatic shell then perform the check. Fixes #7609 --- compiler/semexprs.nim | 2 ++ tests/statictypes/tstatictypes.nim | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 5e1e4cbbd0..82f948492c 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -108,6 +108,8 @@ const proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus = result = convOK + # We're interested in the inner type and not in the static tag + var src = src.skipTypes({tyStatic}) if sameType(castDest, src) and castDest.sym == src.sym: # don't annoy conversions that may be needed on another processor: if castDest.kind notin IntegralTypes+{tyRange}: diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index 9888165cce..b7cde61247 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -132,3 +132,8 @@ block: var x = foo(y, 10, 15, [1, 2, 3]) doAssert x == (20, 10, 15, 3) +# #7609 +block: + type + Coord[N: static[int]] = tuple[col, row: range[0'i8 .. (N.int8-1)]] + Point[N: static[int]] = range[0'i16 .. N.int16 * N.int16 - 1] From b4d329d69f9172114212a2225ff68dbd789ebf68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Mon, 21 Jan 2019 17:38:17 +0100 Subject: [PATCH 176/350] precise printer injection --- tools/nim-gdb.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index fb7eff565c..2abaf69265 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -522,11 +522,20 @@ def makematcher(klass): printErrorOnce(typeName, "No matcher for type '" + typeName + "': " + str(e) + "\n") return matcher +def register_nim_pretty_printers_for_object(objfile): + nimMainSym = gdb.lookup_global_symbol("NimMain", gdb.SYMBOL_FUNCTIONS_DOMAIN) + if nimMainSym and nimMainSym.symtab.objfile == objfile: + print("set Nim pretty printers for ", objfile.filename) + + objfile.type_printers = [NimTypePrinter()] + objfile.pretty_printers = [makematcher(var) for var in list(globals().values()) if hasattr(var, 'pattern')] + +# Register pretty printers for all objfiles that are already loaded. +for old_objfile in gdb.objfiles(): + register_nim_pretty_printers_for_object(old_objfile) + +# Register an event handler to register nim pretty printers for all future objfiles. def new_object_handler(event): - nimobjfile = gdb.current_objfile() or gdb.objfiles()[0] - nimobjfile.type_printers = [] - nimobjfile.type_printers = [NimTypePrinter()] - nimobjfile.pretty_printers = [] - nimobjfile.pretty_printers.extend([makematcher(var) for var in list(globals().values()) if hasattr(var, 'pattern')]) + register_nim_pretty_printers_for_object(event.new_objfile) gdb.events.new_objfile.connect(new_object_handler) From ee89ba6bdb664fe4972f2917499cff1afdac0bab Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 21 Jan 2019 19:12:17 +0100 Subject: [PATCH 177/350] Fix subtype conversion w/ varargs arguments (#10402) The type matching is done on the `T` of the `varargs[T]` so the conversion must be performed to `T` and not to the whole type. This problem is only noticeable with the cpp backend since C doesn't give a damn shit about your fucking (wrong) types. Fixes #9845 --- compiler/sigmatch.nim | 5 +++-- tests/typerel/t4799.nim | 1 + tests/typerel/t4799_1.nim | 1 + tests/typerel/t4799_2.nim | 1 + tests/typerel/t4799_3.nim | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 9aae254f33..e08559db61 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2071,6 +2071,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, # constructor in a call: if result == nil and f.kind == tyVarargs: if f.n != nil: + # Forward to the varargs converter result = localConvMatch(c, m, f, a, arg) else: r = typeRel(m, base(f), a) @@ -2083,10 +2084,10 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, # bug #4799, varargs accepting subtype relation object elif r == isSubtype: inc(m.subtypeMatches) - if f.kind == tyTypeDesc: + if base(f).kind == tyTypeDesc: result = arg else: - result = implicitConv(nkHiddenSubConv, f, arg, m, c) + result = implicitConv(nkHiddenSubConv, base(f), arg, m, c) m.baseTypeMatch = true else: result = userConvMatch(c, m, base(f), a, arg) diff --git a/tests/typerel/t4799.nim b/tests/typerel/t4799.nim index 0758934763..814ad361d0 100644 --- a/tests/typerel/t4799.nim +++ b/tests/typerel/t4799.nim @@ -1,4 +1,5 @@ discard """ + targets: "c cpp" output: "OK" """ diff --git a/tests/typerel/t4799_1.nim b/tests/typerel/t4799_1.nim index 549b6bf3c5..e66aa1a9a4 100644 --- a/tests/typerel/t4799_1.nim +++ b/tests/typerel/t4799_1.nim @@ -1,4 +1,5 @@ discard """ + targets: "c cpp" outputsub: '''ObjectAssignmentError''' exitcode: "1" """ diff --git a/tests/typerel/t4799_2.nim b/tests/typerel/t4799_2.nim index cfd399a6e0..ff20c24260 100644 --- a/tests/typerel/t4799_2.nim +++ b/tests/typerel/t4799_2.nim @@ -1,4 +1,5 @@ discard """ + targets: "c cpp" outputsub: '''ObjectAssignmentError''' exitcode: "1" """ diff --git a/tests/typerel/t4799_3.nim b/tests/typerel/t4799_3.nim index 784eee8fc1..4a8a158dd1 100644 --- a/tests/typerel/t4799_3.nim +++ b/tests/typerel/t4799_3.nim @@ -1,4 +1,5 @@ discard """ + targets: "c cpp" outputsub: '''ObjectAssignmentError''' exitcode: "1" """ From 4b1e227531dcc056c209f9430c615232d7be2618 Mon Sep 17 00:00:00 2001 From: genotrance Date: Mon, 21 Jan 2019 23:24:45 -0600 Subject: [PATCH 178/350] Fix gorge caching (#10407) --- compiler/gorgeimpl.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index 44636f3829..534ef9fdce 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -24,11 +24,11 @@ proc readOutput(p: Process): (string, int) = proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (string, int) = let workingDir = parentDir(toFullPath(conf, info)) - if cache.len > 0:# and optForceFullMake notin gGlobalOptions: + if cache.len > 0: let h = secureHash(cmd & "\t" & input & "\t" & cache) let filename = toGeneratedFile(conf, AbsoluteFile("gorge_" & $h), "txt").string var f: File - if open(f, filename): + if optForceFullMake notin conf.globalOptions and open(f, filename): result = (f.readAll, 0) f.close return From 7a3f382517cfc0f2b676dc6704b423e7b167b565 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Jan 2019 07:33:27 +0100 Subject: [PATCH 179/350] Fix corner case of checked div/mod on x86 (#10406) Fixes #10377 --- lib/system/arithm.nim | 44 ++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim index 69c5587996..a875e95a7e 100644 --- a/lib/system/arithm.nim +++ b/lib/system/arithm.nim @@ -197,26 +197,40 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): proc divInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - idiv ecx - jno theEnd - call `raiseOverflow` - theEnd: + test edx, edx + jne L_NOT_ZERO + call `raiseDivByZero` + L_NOT_ZERO: + cmp ecx, 0x80000000 + jne L_DO_DIV + cmp edx, -1 + jne L_DO_DIV + call `raiseOverflow` + L_DO_DIV: + mov eax, ecx + mov ecx, edx + cdq + idiv ecx ret """ proc modInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - idiv ecx - jno theEnd - call `raiseOverflow` - theEnd: - mov eax, edx + test edx, edx + jne L_NOT_ZERO + call `raiseDivByZero` + L_NOT_ZERO: + cmp ecx, 0x80000000 + jne L_DO_DIV + cmp edx, -1 + jne L_DO_DIV + call `raiseOverflow` + L_DO_DIV: + mov eax, ecx + mov ecx, edx + cdq + idiv ecx + mov eax, edx ret """ From 3ea099bc7f75d35ee127932208d9d012b57f11f8 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Jan 2019 07:35:52 +0100 Subject: [PATCH 180/350] Finalizer proc must be global (#10388) Fixes #10376 --- compiler/semmagic.nim | 5 +++++ tests/errmsgs/t10376.nim | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/errmsgs/t10376.nim diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 05c8b181c0..2311136b49 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -410,4 +410,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n else: result = plugin(c, n) + of mNewFinalize: + # Make sure the finalizer procedure refers to a procedure + if n[^1].kind == nkSym and n[^1].sym.kind notin {skProc, skFunc}: + localError(c.config, n.info, "finalizer must be a direct reference to a procedure") + result = n else: result = n diff --git a/tests/errmsgs/t10376.nim b/tests/errmsgs/t10376.nim new file mode 100644 index 0000000000..a33d5e40f0 --- /dev/null +++ b/tests/errmsgs/t10376.nim @@ -0,0 +1,31 @@ +discard """ + errormsg: "finalizer must be a direct reference to a procedure" + line: 29 +""" + +type + A = ref object + +proc my_callback(a: A) {. nimcall .} = + discard + +proc foo(callback: proc(a: A) {. nimcall .}) = + var x1: A + new(x1, proc (x: A) {.nimcall.} = discard) + var x2: A + new(x2, func (x: A) {.nimcall.} = discard) + + var x3: A + proc foo1(a: A) {.nimcall.} = discard + new(x3, foo1) + var x4: A + func foo2(a: A) {.nimcall.} = discard + new(x4, foo2) + + var x5: A + new(x5, my_callback) + + var x6: A + new(x6, callback) + +foo(my_callback) From 44c04b35717336595dad03ac532ee9e141a0e374 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Jan 2019 07:36:40 +0100 Subject: [PATCH 181/350] Object downconversion in VM should not copy (#10378) Hopefully the type-check phase already rejected all the invalid conversions by the time we execute the VM bytecode. Problem reported by chrisheller on the Nim Forum --- compiler/vm.nim | 5 +++++ tests/vm/tconstobj.nim | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/compiler/vm.nim b/compiler/vm.nim index 10d38fe77f..131501380f 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -399,6 +399,11 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): dest.floatVal = toBiggestFloat(src.intVal) else: dest.floatVal = src.floatVal + of tyObject: + if srctyp.skipTypes(abstractRange).kind != tyObject: + internalError(c.config, "invalid object-to-object conversion") + # A object-to-object conversion is essentially a no-op + moveConst(dest, src) else: asgnComplex(dest, src) diff --git a/tests/vm/tconstobj.nim b/tests/vm/tconstobj.nim index 021fcb7289..3cf256eedf 100644 --- a/tests/vm/tconstobj.nim +++ b/tests/vm/tconstobj.nim @@ -48,3 +48,20 @@ let people = { }.toTable() echo people["001"] + +# Object downconversion should not copy + +type + SomeBaseObj {.inheritable.} = object of RootObj + txt : string + InheritedFromBase = object of SomeBaseObj + other : string + +proc initBase(sbo: var SomeBaseObj) = + sbo.txt = "Initialized string from base" + +static: + var ifb2: InheritedFromBase + initBase(SomeBaseObj(ifb2)) + echo repr(ifb2) + doAssert(ifb2.txt == "Initialized string from base") From 226c15499fe829d533b4811aa5c68b15de870b04 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Tue, 22 Jan 2019 12:25:11 +0530 Subject: [PATCH 182/350] Fix compileTime pragma applying to whole var/let section (#10389) --- compiler/semstmts.nim | 10 ++++------ tests/vm/tvarsection.nim | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 tests/vm/tvarsection.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 288820d862..17f88d0393 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -432,7 +432,6 @@ proc setVarType(c: PContext; v: PSym, typ: PType) = proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = var b: PNode result = copyNode(n) - var hasCompileTime = false for i in countup(0, sonsLen(n)-1): var a = n.sons[i] if c.config.cmd == cmdIdeTools: suggestStmt(c, a) @@ -556,13 +555,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = else: v.typ = tup b.sons[j] = newSymNode(v) checkNilable(c, v) - if sfCompileTime in v.flags: hasCompileTime = true + if sfCompileTime in v.flags: + var x = newNodeI(result.kind, v.info) + addSon(x, result[i]) + vm.setupCompileTimeVar(c.module, c.graph, x) if v.flags * {sfGlobal, sfThread} == {sfGlobal}: message(c.config, v.info, hintGlobalVar) - if hasCompileTime: - vm.setupCompileTimeVar(c.module, c.graph, result) - # handled by the VM codegen: - #c.graph.recordStmt(c.graph, c.module, result) proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) diff --git a/tests/vm/tvarsection.nim b/tests/vm/tvarsection.nim new file mode 100644 index 0000000000..d1c4926a04 --- /dev/null +++ b/tests/vm/tvarsection.nim @@ -0,0 +1,15 @@ +discard """ + output: '''-1abc''' +""" + +var + a {.compileTime.} = 2 + b = -1 + c {.compileTime.} = 3 + d = "abc" + +static: + assert a == 2 + assert c == 3 + +echo b, d From 792dbed57e2f4475cab8dcc72114ea5f844113c9 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Jan 2019 09:41:11 +0100 Subject: [PATCH 183/350] Restore the docstring during the .async. transform (#10404) [backport] Fixes #9816 --- lib/pure/asyncmacro.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index b18d20d554..f1e0aa5682 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -245,6 +245,12 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = var outerProcBody = newNimNode(nnkStmtList, prc.body) + # Extract the documentation comment from the original procedure declaration. + # Note that we're not removing it from the body in order not to make this + # transformation even more complex. + if prc.body.len > 1 and prc.body[0].kind == nnkCommentStmt: + outerProcBody.add(prc.body[0]) + # -> var retFuture = newFuture[T]() var retFutureSym = genSym(nskVar, "retFuture") var subRetType = From 39e1cd2bf60a3626f73ee34e91127a8ea4f18a58 Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 22 Jan 2019 10:18:12 +0100 Subject: [PATCH 184/350] Better docs for sets and intsets (#10362) * better docs: sets * better docs: intsets --- lib/pure/collections/intsets.nim | 362 ++++++-- lib/pure/collections/sets.nim | 1351 ++++++++++++++++++------------ 2 files changed, 1081 insertions(+), 632 deletions(-) diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 9d7950ea69..654b297f62 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -7,13 +7,16 @@ # distribution, for details about the copyright. # -## The ``intsets`` module implements an efficient int set implemented as a +## The ``intsets`` module implements an efficient `int` set implemented as a ## `sparse bit set`:idx:. - -## **Note**: Currently the assignment operator ``=`` for ``intsets`` +## +## **Note**: Currently the assignment operator ``=`` for ``IntSet`` ## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use ``assign`` to -## get a deep copy. +## not allow the assignment operator to be overloaded, use `assign proc +## <#assign,IntSet,IntSet>`_ to get a deep copy. +## +## **See also:** +## * `sets module `_ for more general hash sets import @@ -40,7 +43,7 @@ type bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector TrunkSeq = seq[PTrunk] - IntSet* = object ## an efficient set of 'int' implemented as a sparse bit set + IntSet* = object ## An efficient set of `int` implemented as a sparse bit set. elems: int # only valid for small numbers counter, max: int head: PTrunk @@ -96,18 +99,33 @@ proc intSetPut(t: var IntSet, key: int): PTrunk = t.head = result t.data[h] = result -proc contains*(s: IntSet, key: int): bool = - ## Returns true iff `key` is in `s`. +proc bitincl(s: var IntSet, key: int) {.inline.} = + var t = intSetPut(s, `shr`(key, TrunkShift)) + var u = key and TrunkMask + t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or + `shl`(1, u and IntMask) + +proc exclImpl(s: var IntSet, key: int) = if s.elems <= s.a.len: for i in 0.. 1: result.add(", ") + result.add($key) + result.add("}") + iterator items*(s: IntSet): int {.inline.} = ## Iterates over any included element of `s`. @@ -131,14 +149,62 @@ iterator items*(s: IntSet): int {.inline.} = inc(i) r = r.next -proc bitincl(s: var IntSet, key: int) {.inline.} = - var t = intSetPut(s, `shr`(key, TrunkShift)) - var u = key and TrunkMask - t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or - `shl`(1, u and IntMask) + +proc initIntSet*: IntSet = + ## Returns an empty IntSet. + runnableExamples: + var a = initIntSet() + assert len(a) == 0 + + # newSeq(result.data, InitIntSetSize) + # result.max = InitIntSetSize-1 + result = IntSet( + elems: 0, + counter: 0, + max: 0, + head: nil, + data: when defined(nimNoNilSeqs): @[] else: nil) + # a: array[0..33, int] # profiling shows that 34 elements are enough + +proc contains*(s: IntSet, key: int): bool = + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of `in` operator. + runnableExamples: + var a = initIntSet() + for x in [1, 3, 5]: + a.incl(x) + assert a.contains(3) + assert 3 in a + assert(not a.contains(8)) + assert 8 notin a + + if s.elems <= s.a.len: + for i in 0..`_ for excluding an element + ## * `incl proc <#incl,IntSet,IntSet>`_ for including other set + ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ + runnableExamples: + var a = initIntSet() + a.incl(3) + a.incl(3) + assert len(a) == 1 + if s.elems <= s.a.len: for i in 0..`_. + ## + ## See also: + ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set + ## * `incl proc <#incl,IntSet,int>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1) + b.incl(5) + a.incl(b) + assert len(a) == 2 + assert 5 in a + for item in other: incl(s, item) -proc exclImpl(s: var IntSet, key: int) = - if s.elems <= s.a.len: - for i in 0..`_ is + ## that this proc returns `true` if `s` already contained `key`. The + ## proc will return `false` if `key` was added as a new value to `s` during + ## this call. + ## + ## See also: + ## * `incl proc <#incl,IntSet,int>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ + runnableExamples: + var a = initIntSet() + assert a.containsOrIncl(3) == false + assert a.containsOrIncl(3) == true + assert a.containsOrIncl(4) == false + if s.elems <= s.a.len: for i in 0..`_ for including an element + ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set + ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ + runnableExamples: + var a = initIntSet() + a.incl(3) + a.excl(3) + a.excl(3) + a.excl(99) + assert len(a) == 0 + exclImpl(s, key) - # newSeq(result.data, InitIntSetSize) - # result.max = InitIntSetSize-1 - result = IntSet( - elems: 0, - counter: 0, - max: 0, - head: nil, - data: when defined(nimNoNilSeqs): @[] else: nil) - # a: array[0..33, int] # profiling shows that 34 elements are enough +proc excl*(s: var IntSet, other: IntSet) = + ## Excludes all elements from `other` from `s`. + ## + ## This is the in-place version of `s - other <#-,IntSet,IntSet>`_. + ## + ## See also: + ## * `incl proc <#incl,IntSet,IntSet>`_ for including other set + ## * `excl proc <#excl,IntSet,int>`_ for excluding an element + ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1) + a.incl(5) + b.incl(5) + a.excl(b) + assert len(a) == 1 + assert 5 notin a + + for item in other: excl(s, item) + +proc missingOrExcl*(s: var IntSet, key: int) : bool = + ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. + ## + ## The difference with regards to the `excl proc <#excl,IntSet,int>`_ is + ## that this proc returns `true` if `key` was missing from `s`. + ## The proc will return `false` if `key` was in `s` and it was removed + ## during this call. + ## + ## See also: + ## * `excl proc <#excl,IntSet,int>`_ for excluding an element + ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set + ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ + runnableExamples: + var a = initIntSet() + a.incl(5) + assert a.missingOrExcl(5) == false + assert a.missingOrExcl(5) == true + + var count = s.elems + exclImpl(s, key) + result = count == s.elems proc clear*(result: var IntSet) = ## Clears the IntSet back to an empty state. + runnableExamples: + var a = initIntSet() + a.incl(5) + a.incl(7) + clear(a) + assert len(a) == 0 # setLen(result.data, InitIntSetSize) # for i in 0..InitIntSetSize-1: result.data[i] = nil @@ -243,8 +362,17 @@ proc clear*(result: var IntSet) = proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil and x.elems == 0 proc assign*(dest: var IntSet, src: IntSet) = - ## copies `src` to `dest`. `dest` does not need to be initialized by - ## `initIntSet`. + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by `initIntSet proc <#initIntSet,>`_. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + if src.elems <= src.a.len: when defined(nimNoNilSeqs): dest.data = @[] @@ -276,11 +404,33 @@ proc assign*(dest: var IntSet, src: IntSet) = proc union*(s1, s2: IntSet): IntSet = ## Returns the union of the sets `s1` and `s2`. + ## + ## The same as `s1 + s2 <#+,IntSet,IntSet>`_. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1); a.incl(2); a.incl(3) + b.incl(3); b.incl(4); b.incl(5) + assert union(a, b).len == 5 + ## {1, 2, 3, 4, 5} + result.assign(s1) incl(result, s2) proc intersection*(s1, s2: IntSet): IntSet = ## Returns the intersection of the sets `s1` and `s2`. + ## + ## The same as `s1 * s2 <#*,IntSet,IntSet>`_. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1); a.incl(2); a.incl(3) + b.incl(3); b.incl(4); b.incl(5) + assert intersection(a, b).len == 1 + ## {3} + result = initIntSet() for item in s1: if contains(s2, item): @@ -288,6 +438,17 @@ proc intersection*(s1, s2: IntSet): IntSet = proc difference*(s1, s2: IntSet): IntSet = ## Returns the difference of the sets `s1` and `s2`. + ## + ## The same as `s1 - s2 <#-,IntSet,IntSet>`_. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1); a.incl(2); a.incl(3) + b.incl(3); b.incl(4); b.incl(5) + assert difference(a, b).len == 2 + ## {1, 2} + result = initIntSet() for item in s1: if not contains(s2, item): @@ -295,31 +456,50 @@ proc difference*(s1, s2: IntSet): IntSet = proc symmetricDifference*(s1, s2: IntSet): IntSet = ## Returns the symmetric difference of the sets `s1` and `s2`. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1); a.incl(2); a.incl(3) + b.incl(3); b.incl(4); b.incl(5) + assert symmetricDifference(a, b).len == 4 + ## {1, 2, 4, 5} + result.assign(s1) for item in s2: if containsOrIncl(result, item): excl(result, item) proc `+`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `union(s1, s2) <#union>`_. + ## Alias for `union(s1, s2) <#union,IntSet,IntSet>`_. result = union(s1, s2) proc `*`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `intersection(s1, s2) <#intersection>`_. + ## Alias for `intersection(s1, s2) <#intersection,IntSet,IntSet>`_. result = intersection(s1, s2) proc `-`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `difference(s1, s2) <#difference>`_. + ## Alias for `difference(s1, s2) <#difference,IntSet,IntSet>`_. result = difference(s1, s2) proc disjoint*(s1, s2: IntSet): bool = ## Returns true if the sets `s1` and `s2` have no items in common. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1); a.incl(2) + b.incl(2); b.incl(3) + assert disjoint(a, b) == false + b.excl(2) + assert disjoint(a, b) == true + for item in s1: if contains(s2, item): return false return true proc len*(s: IntSet): int {.inline.} = - ## Returns the number of keys in `s`. + ## Returns the number of elements in `s`. if s.elems < s.a.len: result = s.elems else: @@ -328,35 +508,59 @@ proc len*(s: IntSet): int {.inline.} = inc(result) proc card*(s: IntSet): int {.inline.} = - ## Alias for `len() <#len>` _. + ## Alias for `len() <#len,IntSet>`_. result = s.len() proc `<=`*(s1, s2: IntSet): bool = - ## Returns true iff `s1` is subset of `s2`. + ## Returns true if `s1` is subset of `s2`. + ## + ## A subset `s1` has all of its elements in `s2`, and `s2` doesn't necessarily + ## have more elements than `s1`. That is, `s1` can be equal to `s2`. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1) + b.incl(1); b.incl(2) + assert a <= b + a.incl(2) + assert a <= b + a.incl(3) + assert(not (a <= b)) + for item in s1: if not s2.contains(item): return false return true proc `<`*(s1, s2: IntSet): bool = - ## Returns true iff `s1` is proper subset of `s2`. + ## Returns true if `s1` is proper subset of `s2`. + ## + ## A strict or proper subset `s1` has all of its elements in `s2`, but `s2` has + ## more elements than `s1`. + runnableExamples: + var + a = initIntSet() + b = initIntSet() + a.incl(1) + b.incl(1); b.incl(2) + assert a < b + a.incl(2) + assert(not (a < b)) return s1 <= s2 and not (s2 <= s1) proc `==`*(s1, s2: IntSet): bool = - ## Returns true if both `s` and `t` have the same members and set size. + ## Returns true if both `s1` and `s2` have the same elements and set size. return s1 <= s2 and s2 <= s1 -template dollarImpl(): untyped = - result = "{" - for key in items(s): - if result.len > 1: result.add(", ") - result.add($key) - result.add("}") - proc `$`*(s: IntSet): string = ## The `$` operator for int sets. + ## + ## Converts the set `s` to a string, mostly for logging and printing purposes. dollarImpl() + + when isMainModule: import sequtils, algorithm diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index d1f941e92c..5da5d92434 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -14,8 +14,38 @@ ## `_. Sets allow you to store any value that can be ## `hashed `_ and they don't contain duplicate entries. ## -## **Note**: The data types declared here have *value semantics*: This means +## Common usages of sets: +## * removing duplicates from a container by converting it with `toSet proc +## <#toSet,openArray[A]>`_ (see also `sequtils.deduplicate proc +## `_) +## * membership testing +## * mathematical operations on two sets, such as +## `union <#union,HashSet[A],HashSet[A]>`_, +## `intersection <#intersection,HashSet[A],HashSet[A]>`_, +## `difference <#difference,HashSet[A],HashSet[A]>`_, and +## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ +## +## .. code-block:: +## echo toSet([9, 5, 1]) # {9, 1, 5} +## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} +## +## let +## s1 = toSet([9, 5, 1]) +## s2 = toSet([3, 5, 7]) +## +## echo s1 + s2 # {9, 1, 3, 5, 7} +## echo s1 - s2 # {1, 9} +## echo s1 * s2 # {5} +## echo s1 -+- s2 # {9, 1, 3, 7} +## +## +## Note: The data types declared here have *value semantics*: This means ## that ``=`` performs a copy of the set. +## +## **See also:** +## * `intsets module `_ for efficient int sets +## * `tables module `_ for hash tables + import hashes, math @@ -31,27 +61,24 @@ when not defined(nimhygiene): type KeyValuePair[A] = tuple[hcode: Hash, key: A] KeyValuePairSeq[A] = seq[KeyValuePair[A]] - HashSet* {.myShallow.}[A] = object ## \ + HashSet* {.myShallow.} [A] = object ## \ ## A generic hash set. ## - ## Use `init() <#init,HashSet[A],int>`_ or `initSet[type]() <#initSet>`_ + ## Use `init proc <#init,HashSet[A],int>`_ or `initSet proc <#initSet,int>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int + +# ---------------------- helpers ----------------------------------- + +const growthFactor = 2 + template default[T](t: typedesc[T]): T = ## Used by clear methods to get a default value. var v: T v -proc clear*[A](s: var HashSet[A]) = - ## Clears the HashSet back to an empty state, without shrinking - ## any of the existing storage. O(n) where n is the size of the hash bucket. - s.counter = 0 - for i in 0..`_. - ## - ## Most operations over an uninitialized set will crash at runtime and - ## `assert `_ in debug builds. You can use this proc in - ## your own procs to verify that sets passed to your procs are correctly - ## initialized. Example: - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! - result = s.data.len > 0 - -proc len*[A](s: HashSet[A]): int = - ## Returns the number of keys in `s`. - ## - ## Due to an implementation detail you can call this proc on variables which - ## have not been initialized yet. The proc will return zero as the length - ## then. Example: - ## - ## .. code-block:: - ## - ## var values: HashSet[int] - ## assert(not values.isValid) - ## assert values.len == 0 - result = s.counter - -proc card*[A](s: HashSet[A]): int = - ## Alias for `len() <#len,TSet[A]>`_. - ## - ## Card stands for the `cardinality - ## `_ of a set. - result = s.counter - -iterator items*[A](s: HashSet[A]): A = - ## Iterates over keys in the set `s`. - ## - ## If you need a sequence with the keys you can use `sequtils.toSeq() - ## `_ on the iterator. Usage example: - ## - ## .. code-block:: - ## type - ## pair = tuple[a, b: int] - ## var - ## a, b = initSet[pair]() - ## a.incl((2, 3)) - ## a.incl((3, 2)) - ## a.incl((2, 3)) - ## for x, y in a.items: - ## b.incl((x - 2, y + 1)) - ## assert a.len == 2 - ## echo b - ## # --> {(a: 1, b: 3), (a: 0, b: 4)} - assert s.isValid, "The set needs to be initialized." - for h in 0..high(s.data): - if isFilled(s.data[h].hcode): yield s.data[h].key - -proc hash*[A](s: HashSet[A]): Hash = - ## hashing of HashSet - assert s.isValid, "The set needs to be initialized." - for h in 0..high(s.data): - result = result xor s.data[h].hcode - result = !$result - -const - growthFactor = 2 - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc rightSize*(count: Natural): int {.inline.} = - ## Return the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - ## - ## Internally, we want mustRehash(rightSize(x), x) == false. - result = nextPowerOfTwo(count * 3 div 2 + 4) - proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = (h + 1) and maxHash @@ -176,38 +122,6 @@ proc rawGetKnownHC[A](s: HashSet[A], key: A, hc: Hash): int {.inline.} = proc rawGet[A](s: HashSet[A], key: A, hc: var Hash): int {.inline.} = rawGetImpl() -proc `[]`*[A](s: var HashSet[A], key: A): var A = - ## returns the element that is actually stored in 's' which has the same - ## value as 'key' or raises the ``KeyError`` exception. This is useful - ## when one overloaded 'hash' and '==' but still needs reference semantics - ## for sharing. - assert s.isValid, "The set needs to be initialized." - var hc: Hash - var index = rawGet(s, key, hc) - if index >= 0: result = s.data[index].key - else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") - -proc contains*[A](s: HashSet[A], key: A): bool = - ## Returns true iff `key` is in `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## assert(not values.contains(2)) - ## values.incl(2) - ## assert values.contains(2) - ## values.excl(2) - ## assert(not values.contains(2)) - assert s.isValid, "The set needs to be initialized." - var hc: Hash - var index = rawGet(s, key, hc) - result = index >= 0 - proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A, hc: Hash, h: Hash) = rawInsertImpl() @@ -243,34 +157,6 @@ template containsOrInclImpl() {.dirty.} = rawInsert(s, s.data, key, hc, -1 - index) inc(s.counter) -proc incl*[A](s: var HashSet[A], key: A) = - ## Includes an element `key` in `s`. - ## - ## This doesn't do anything if `key` is already in `s`. Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## values.incl(2) - ## values.incl(2) - ## assert values.len == 1 - assert s.isValid, "The set needs to be initialized." - inclImpl() - -proc incl*[A](s: var HashSet[A], other: HashSet[A]) = - ## Includes all elements from `other` into `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## values.incl(2) - ## var others = toSet([6, 7]) - ## values.incl(others) - ## assert values.len == 3 - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: incl(s, item) - template doWhile(a, b) = while true: b @@ -302,119 +188,9 @@ proc exclImpl[A](s: var HashSet[A], key: A) : bool {. inline .} = r = s.data[i].hcode and msk # "home" location of key@i shallowCopy(s.data[j], s.data[i]) # data[i] will be marked EMPTY next loop -proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = - ## Excludes `key` in the set `s` and tells if `key` was removed from `s`. - ## - ## The difference with regards to the `excl() <#excl,TSet[A],A>`_ proc is - ## that this proc returns `true` if `key` was not present in `s`. Example: - ## - ## .. code-block:: - ## var s = toSet([2, 3, 6, 7]) - ## assert s.missingOrExcl(4) == true - ## assert s.missingOrExcl(6) == false - exclImpl(s, key) - -proc excl*[A](s: var HashSet[A], key: A) = - ## Excludes `key` from the set `s`. - ## - ## This doesn't do anything if `key` is not found in `s`. Example: - ## - ## .. code-block:: - ## var s = toSet([2, 3, 6, 7]) - ## s.excl(2) - ## s.excl(2) - ## assert s.len == 3 - discard exclImpl(s, key) - -proc excl*[A](s: var HashSet[A], other: HashSet[A]) = - ## Excludes everything in `other` from `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var - ## numbers = toSet([1, 2, 3, 4, 5]) - ## even = toSet([2, 4, 6, 8]) - ## numbers.excl(even) - ## echo numbers - ## # --> {1, 3, 5} - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: discard exclImpl(s, item) - -proc pop*[A](s: var HashSet[A]): A = - ## Remove and return an arbitrary element from the set `s`. - ## - ## Raises KeyError if the set `s` is empty. - ## - for h in 0..high(s.data): - if isFilled(s.data[h].hcode): - result = s.data[h].key - excl(s, result) - return result - raise newException(KeyError, "set is empty") - -proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = - ## Includes `key` in the set `s` and tells if `key` was added to `s`. - ## - ## The difference with regards to the `incl() <#incl,TSet[A],A>`_ proc is - ## that this proc returns `true` if `key` was already present in `s`. The - ## proc will return false if `key` was added as a new value to `s` during - ## this call. Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## assert values.containsOrIncl(2) == false - ## assert values.containsOrIncl(2) == true - assert s.isValid, "The set needs to be initialized." - containsOrInclImpl() - -proc init*[A](s: var HashSet[A], initialSize=64) = - ## Initializes a hash set. - ## - ## The `initialSize` parameter needs to be a power of two. You can use - ## `math.nextPowerOfTwo() `_ or `rightSize` to - ## guarantee that at runtime. All set variables must be initialized before - ## use with other procs from this module with the exception of `isValid() - ## <#isValid,TSet[A]>`_ and `len() <#len,TSet[A]>`_. - ## - ## You can call this proc on a previously initialized hash set, which will - ## discard all its values. This might be more convenient than iterating over - ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: - ## - ## .. code-block :: - ## var a: HashSet[int] - ## a.init(4) - ## a.incl(2) - ## a.init - ## assert a.len == 0 and a.isValid - assert isPowerOfTwo(initialSize) - s.counter = 0 - newSeq(s.data, initialSize) - -proc initSet*[A](initialSize=64): HashSet[A] = - ## Wrapper around `init() <#init,TSet[A],int>`_ for initialization of hash - ## sets. - ## - ## Returns an empty hash set you can assign directly in ``var`` blocks in a - ## single line. Example: - ## - ## .. code-block :: - ## var a = initSet[int](4) - ## a.incl(2) - result.init(initialSize) - -proc toSet*[A](keys: openArray[A]): HashSet[A] = - ## Creates a new hash set that contains the given `keys`. - ## - ## Example: - ## - ## .. code-block:: - ## var numbers = toSet([1, 2, 3, 4, 5]) - ## assert numbers.contains(2) - ## assert numbers.contains(4) - result = initSet[A](rightSize(keys.len)) - for key in items(keys): result.incl(key) +proc mustRehash(length, counter: int): bool {.inline.} = + assert(length > counter) + result = (length * 2 < counter * 3) or (length - counter < 4) template dollarImpl() {.dirty.} = result = "{" @@ -423,34 +199,333 @@ template dollarImpl() {.dirty.} = result.addQuoted(key) result.add("}") -proc `$`*[A](s: HashSet[A]): string = - ## Converts the set `s` to a string, mostly for logging purposes. +proc rightSize*(count: Natural): int {.inline.} + + + + + + + + +# --------------------------------------------------------------------- +# ------------------------------ HashSet ------------------------------ +# --------------------------------------------------------------------- + + +proc init*[A](s: var HashSet[A], initialSize=64) = + ## Initializes a hash set. ## - ## Don't use this proc for serialization, the representation may change at - ## any moment and values are not escaped. Example: + ## The `initialSize` parameter needs to be a power of two (default: 64). + ## If you need to accept runtime values for this, you can use + ## `math.nextPowerOfTwo proc `_ or `rightSize proc + ## <#rightSize,Natural>`_ from this module. ## - ## Example: + ## All set variables must be initialized before + ## use with other procs from this module, with the exception of `isValid proc + ## <#isValid,HashSet[A]>`_ and `len() <#len,HashSet[A]>`_. ## - ## .. code-block:: - ## echo toSet([2, 4, 5]) - ## # --> {2, 4, 5} - ## echo toSet(["no", "esc'aping", "is \" provided"]) - ## # --> {no, esc'aping, is " provided} + ## You can call this proc on a previously initialized hash set, which will + ## discard all its values. This might be more convenient than iterating over + ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. + ## + ## See also: + ## * `initSet proc <#initSet,int>`_ + ## * `toSet proc <#toSet,openArray[A]>`_ + runnableExamples: + var a: HashSet[int] + assert(not a.isValid) + init(a) + assert a.isValid + + assert isPowerOfTwo(initialSize) + s.counter = 0 + newSeq(s.data, initialSize) + +proc initSet*[A](initialSize=64): HashSet[A] = + ## Wrapper around `init proc <#init,HashSet[A],int>`_ for initialization of + ## hash sets. + ## + ## Returns an empty hash set you can assign directly in ``var`` blocks in a + ## single line. + ## + ## See also: + ## * `toSet proc <#toSet,openArray[A]>`_ + runnableExamples: + var a = initSet[int]() + assert a.isValid + a.incl(3) + assert len(a) == 1 + result.init(initialSize) + +proc toSet*[A](keys: openArray[A]): HashSet[A] = + ## Creates a new hash set that contains the members of the given + ## collection (seq, array, or string) `keys`. + ## + ## Duplicates are removed. + ## + ## See also: + ## * `initSet proc <#initSet,int>`_ + runnableExamples: + let + a = toSet([5, 3, 2]) + b = toSet("abracadabra") + assert len(a) == 3 + ## a == {2, 3, 5} + assert len(b) == 5 + ## b == {'a', 'b', 'c', 'd', 'r'} + + result = initSet[A](rightSize(keys.len)) + for key in items(keys): result.incl(key) + +proc isValid*[A](s: HashSet[A]): bool = + ## Returns `true` if the set has been initialized (with `initSet proc + ## <#initSet,int>`_ or `init proc <#init,HashSet[A],int>`_). + ## + ## Most operations over an uninitialized set will crash at runtime and + ## `assert `_ in debug builds. You can use this proc in + ## your own procs to verify that sets passed to your procs are correctly + ## initialized. + ## + ## **Examples:** + ## + ## .. code-block :: + ## proc savePreferences(options: HashSet[string]) = + ## assert options.isValid, "Pass an initialized set!" + ## # Do stuff here, may crash in release builds! + result = s.data.len > 0 + +proc `[]`*[A](s: var HashSet[A], key: A): var A = + ## Returns the element that is actually stored in `s` which has the same + ## value as `key` or raises the ``KeyError`` exception. + ## + ## This is useful when one overloaded `hash` and `==` but still needs + ## reference semantics for sharing. assert s.isValid, "The set needs to be initialized." - dollarImpl() + var hc: Hash + var index = rawGet(s, key, hc) + if index >= 0: result = s.data[index].key + else: + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") + +proc contains*[A](s: HashSet[A], key: A): bool = + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of `in` operator. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var values = initSet[int]() + assert(not values.contains(2)) + assert 2 notin values + + values.incl(2) + assert values.contains(2) + assert 2 in values + + assert s.isValid, "The set needs to be initialized." + var hc: Hash + var index = rawGet(s, key, hc) + result = index >= 0 + +proc incl*[A](s: var HashSet[A], key: A) = + ## Includes an element `key` in `s`. + ## + ## This doesn't do anything if `key` is already in `s`. + ## + ## See also: + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var values = initSet[int]() + values.incl(2) + values.incl(2) + assert values.len == 1 + + assert s.isValid, "The set needs to be initialized." + inclImpl() + +proc incl*[A](s: var HashSet[A], other: HashSet[A]) = + ## Includes all elements from `other` set into `s` (must be declared as `var`). + ## + ## This is the in-place version of `s + other <#+,HashSet[A],HashSet[A]>`_. + ## + ## See also: + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var + values = toSet([1, 2, 3]) + others = toSet([3, 4, 5]) + values.incl(others) + assert values.len == 5 + + assert s.isValid, "The set `s` needs to be initialized." + assert other.isValid, "The set `other` needs to be initialized." + for item in other: incl(s, item) + +proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = + ## Includes `key` in the set `s` and tells if `key` was already in `s`. + ## + ## The difference with regards to the `incl proc <#incl,HashSet[A],A>`_ is + ## that this proc returns `true` if `s` already contained `key`. The + ## proc will return `false` if `key` was added as a new value to `s` during + ## this call. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var values = initSet[int]() + assert values.containsOrIncl(2) == false + assert values.containsOrIncl(2) == true + assert values.containsOrIncl(3) == false + + assert s.isValid, "The set needs to be initialized." + containsOrInclImpl() + +proc excl*[A](s: var HashSet[A], key: A) = + ## Excludes `key` from the set `s`. + ## + ## This doesn't do anything if `key` is not found in `s`. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var s = toSet([2, 3, 6, 7]) + s.excl(2) + s.excl(2) + assert s.len == 3 + discard exclImpl(s, key) + +proc excl*[A](s: var HashSet[A], other: HashSet[A]) = + ## Excludes all elements of `other` set from `s`. + ## + ## This is the in-place version of `s - other <#-,HashSet[A],HashSet[A]>`_. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var + numbers = toSet([1, 2, 3, 4, 5]) + even = toSet([2, 4, 6, 8]) + numbers.excl(even) + assert len(numbers) == 3 + ## numbers == {1, 3, 5} + + assert s.isValid, "The set `s` needs to be initialized." + assert other.isValid, "The set `other` needs to be initialized." + for item in other: discard exclImpl(s, item) + +proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = + ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. + ## + ## The difference with regards to the `excl proc <#excl,HashSet[A],A>`_ is + ## that this proc returns `true` if `key` was missing from `s`. + ## The proc will return `false` if `key` was in `s` and it was removed + ## during this call. + ## + ## See also: + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var s = toSet([2, 3, 6, 7]) + assert s.missingOrExcl(4) == true + assert s.missingOrExcl(6) == false + assert s.missingOrExcl(6) == true + exclImpl(s, key) + +proc pop*[A](s: var HashSet[A]): A = + ## Remove and return an arbitrary element from the set `s`. + ## + ## Raises KeyError if the set `s` is empty. + ## + ## See also: + ## * `clear proc <#clear,HashSet[A]>`_ + runnableExamples: + var s = toSet([2, 1]) + assert s.pop == 1 + assert s.pop == 2 + doAssertRaises(KeyError, echo s.pop) + + for h in 0..high(s.data): + if isFilled(s.data[h].hcode): + result = s.data[h].key + excl(s, result) + return result + raise newException(KeyError, "set is empty") + +proc clear*[A](s: var HashSet[A]) = + ## Clears the HashSet back to an empty state, without shrinking + ## any of the existing storage. + ## + ## `O(n)` operation, where `n` is the size of the hash bucket. + ## + ## See also: + ## * `pop proc <#pop,HashSet[A]>`_ + runnableExamples: + var s = toSet([3, 5, 7]) + clear(s) + assert len(s) == 0 + + s.counter = 0 + for i in 0..`_. + ## + ## Card stands for the `cardinality + ## `_ of a set. + result = s.counter + proc union*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the union of the sets `s1` and `s2`. ## - ## The union of two sets is represented mathematically as *A ∪ B* and is the - ## set of all objects that are members of `s1`, `s2` or both. Example: + ## The same as `s1 + s2 <#+,HashSet[A],HashSet[A]>`_. ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = union(a, b) - ## assert c == toSet(["a", "b", "c"]) + ## The union of two sets is represented mathematically as *A ∪ B* and is the + ## set of all objects that are members of `s1`, `s2` or both. + ## + ## See also: + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = union(a, b) + assert c == toSet(["a", "b", "c"]) + assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." result = s1 @@ -459,16 +534,23 @@ proc union*[A](s1, s2: HashSet[A]): HashSet[A] = proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the intersection of the sets `s1` and `s2`. ## + ## The same as `s1 * s2 <#*,HashSet[A],HashSet[A]>`_. + ## ## The intersection of two sets is represented mathematically as *A ∩ B* and ## is the set of all objects that are members of `s1` and `s2` at the same - ## time. Example: + ## time. ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c == toSet(["b"]) + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = intersection(a, b) + assert c == toSet(["b"]) + assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." result = initSet[A](min(s1.data.len, s2.data.len)) @@ -478,16 +560,22 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. ## + ## The same as `s1 - s2 <#-,HashSet[A],HashSet[A]>`_. + ## ## The difference of two sets is represented mathematically as *A \ B* and is ## the set of all objects that are members of `s1` and not members of `s2`. - ## Example: ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = difference(a, b) - ## assert c == toSet(["a"]) + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = difference(a, b) + assert c == toSet(["a"]) + assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." result = initSet[A]() @@ -498,16 +586,23 @@ proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = proc symmetricDifference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the symmetric difference of the sets `s1` and `s2`. ## + ## The same as `s1 -+- s2 <#-+-,HashSet[A],HashSet[A]>`_. + ## ## The symmetric difference of two sets is represented mathematically as *A △ ## B* or *A ⊖ B* and is the set of all objects that are members of `s1` or - ## `s2` but not both at the same time. Example: + ## `s2` but not both at the same time. ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = symmetricDifference(a, b) - ## assert c == toSet(["a", "c"]) + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = symmetricDifference(a, b) + assert c == toSet(["a", "c"]) + assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." result = s1 @@ -515,32 +610,31 @@ proc symmetricDifference*[A](s1, s2: HashSet[A]): HashSet[A] = if containsOrIncl(result, item): excl(result, item) proc `+`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `union(s1, s2) <#union>`_. + ## Alias for `union(s1, s2) <#union,HashSet[A],HashSet[A]>`_. result = union(s1, s2) proc `*`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `intersection(s1, s2) <#intersection>`_. + ## Alias for `intersection(s1, s2) <#intersection,HashSet[A],HashSet[A]>`_. result = intersection(s1, s2) proc `-`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `difference(s1, s2) <#difference>`_. + ## Alias for `difference(s1, s2) <#difference,HashSet[A],HashSet[A]>`_. result = difference(s1, s2) proc `-+-`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `symmetricDifference(s1, s2) <#symmetricDifference>`_. + ## Alias for `symmetricDifference(s1, s2) + ## <#symmetricDifference,HashSet[A],HashSet[A]>`_. result = symmetricDifference(s1, s2) proc disjoint*[A](s1, s2: HashSet[A]): bool = - ## Returns true iff the sets `s1` and `s2` have no items in common. - ## - ## Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## assert disjoint(a, b) == false - ## assert disjoint(a, b - a) == true + ## Returns `true` if the sets `s1` and `s2` have no items in common. + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + assert disjoint(a, b) == false + assert disjoint(a, b - a) == true + assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." for item in s1: @@ -551,30 +645,29 @@ proc `<`*[A](s, t: HashSet[A]): bool = ## Returns true if `s` is a strict or proper subset of `t`. ## ## A strict or proper subset `s` has all of its members in `t` but `t` has - ## more elements than `s`. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c < a and c < b - ## assert((a < a) == false) + ## more elements than `s`. + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = intersection(a, b) + assert c < a and c < b + assert(not (a < a)) s.counter != t.counter and s <= t proc `<=`*[A](s, t: HashSet[A]): bool = - ## Returns true if `s` is subset of `t`. + ## Returns true if `s` is a subset of `t`. ## ## A subset `s` has all of its members in `t` and `t` doesn't necessarily - ## have more members than `s`. That is, `s` can be equal to `t`. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c <= a and c <= b - ## assert((a <= a)) + ## have more members than `s`. That is, `s` can be equal to `t`. + runnableExamples: + let + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = intersection(a, b) + assert c <= a and c <= b + assert a <= a + result = false if s.counter > t.counter: return result = true @@ -585,90 +678,109 @@ proc `<=`*[A](s, t: HashSet[A]): bool = proc `==`*[A](s, t: HashSet[A]): bool = ## Returns true if both `s` and `t` have the same members and set size. - ## - ## Example: - ## - ## .. code-block:: - ## var - ## a = toSet([1, 2]) - ## b = toSet([1]) - ## b.incl(2) - ## assert a == b + runnableExamples: + var + a = toSet([1, 2]) + b = toSet([2, 1]) + assert a == b s.counter == t.counter and s <= t proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = - ## Returns a new set after applying `op` on each of the elements of `data`. + ## Returns a new set after applying `op` pric on each of the elements of + ##`data` set. ## - ## You can use this proc to transform the elements from a set. Example: - ## - ## .. code-block:: - ## var a = toSet([1, 2, 3]) - ## var b = a.map(proc (x: int): string = $x) - ## assert b == toSet(["1", "2", "3"]) + ## You can use this proc to transform the elements from a set. + runnableExamples: + let + a = toSet([1, 2, 3]) + b = a.map(proc (x: int): string = $x) + assert b == toSet(["1", "2", "3"]) + result = initSet[B]() for item in data: result.incl(op(item)) -# ------------------------------ ordered set ------------------------------ +proc hash*[A](s: HashSet[A]): Hash = + ## Hashing of HashSet. + assert s.isValid, "The set needs to be initialized." + for h in 0..high(s.data): + result = result xor s.data[h].hcode + result = !$result + +proc `$`*[A](s: HashSet[A]): string = + ## Converts the set `s` to a string, mostly for logging and printing purposes. + ## + ## Don't use this proc for serialization, the representation may change at + ## any moment and values are not escaped. + ## + ## **Examples:** + ## + ## .. code-block:: + ## echo toSet([2, 4, 5]) + ## # --> {2, 4, 5} + ## echo toSet(["no", "esc'aping", "is \" provided"]) + ## # --> {no, esc'aping, is " provided} + assert s.isValid, "The set needs to be initialized." + dollarImpl() + +proc rightSize*(count: Natural): int {.inline.} = + ## Return the value of `initialSize` to support `count` items. + ## + ## If more items are expected to be added, simply add that + ## expected extra amount to the parameter before calling this. + ## + ## Internally, we want `mustRehash(rightSize(x), x) == false`. + result = nextPowerOfTwo(count * 3 div 2 + 4) + + + +iterator items*[A](s: HashSet[A]): A = + ## Iterates over elements of the set `s`. + ## + ## If you need a sequence with the elelments you can use `sequtils.toSeq + ## template `_. + ## + ## .. code-block:: + ## type + ## pair = tuple[a, b: int] + ## var + ## a, b = initSet[pair]() + ## a.incl((2, 3)) + ## a.incl((3, 2)) + ## a.incl((2, 3)) + ## for x, y in a.items: + ## b.incl((x - 2, y + 1)) + ## assert a.len == 2 + ## echo b + ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + assert s.isValid, "The set needs to be initialized." + for h in 0..high(s.data): + if isFilled(s.data[h].hcode): yield s.data[h].key + + + + + + + + +# --------------------------------------------------------------------- +# --------------------------- OrderedSet ------------------------------ +# --------------------------------------------------------------------- type OrderedKeyValuePair[A] = tuple[ hcode: Hash, next: int, key: A] OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]] - OrderedSet* {.myShallow.}[A] = object ## \ + OrderedSet* {.myShallow.} [A] = object ## \ ## A generic hash set that remembers insertion order. ## - ## Use `init() <#init,OrderedSet[A],int>`_ or `initOrderedSet[type]() - ## <#initOrderedSet>`_ before calling other procs on it. + ## Use `init proc <#init,OrderedSet[A],int>`_ or `initOrderedSet proc + ## <#initOrderedSet,int>`_ before calling other procs on it. data: OrderedKeyValuePairSeq[A] counter, first, last: int -proc clear*[A](s: var OrderedSet[A]) = - ## Clears the OrderedSet back to an empty state, without shrinking - ## any of the existing storage. O(n) where n is the size of the hash bucket. - s.counter = 0 - s.first = -1 - s.last = -1 - for i in 0..`_. - ## - ## Most operations over an uninitialized ordered set will crash at runtime - ## and `assert `_ in debug builds. You can use this proc - ## in your own procs to verify that ordered sets passed to your procs are - ## correctly initialized. Example: - ## - ## .. code-block:: - ## proc saveTarotCards(cards: OrderedSet[int]) = - ## assert cards.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! - result = s.data.len > 0 - -proc len*[A](s: OrderedSet[A]): int {.inline.} = - ## Returns the number of keys in `s`. - ## - ## Due to an implementation detail you can call this proc on variables which - ## have not been initialized yet. The proc will return zero as the length - ## then. Example: - ## - ## .. code-block:: - ## - ## var values: OrderedSet[int] - ## assert(not values.isValid) - ## assert values.len == 0 - result = s.counter - -proc card*[A](s: OrderedSet[A]): int {.inline.} = - ## Alias for `len() <#len,TOrderedSet[A]>`_. - ## - ## Card stands for the `cardinality - ## `_ of a set. - result = s.counter +# ---------------------- helpers ----------------------------------- template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = var h = s.first @@ -680,61 +792,12 @@ template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = inc(idx) h = nxt -iterator items*[A](s: OrderedSet[A]): A = - ## Iterates over keys in the ordered set `s` in insertion order. - ## - ## If you need a sequence with the keys you can use `sequtils.toSeq() - ## `_ on the iterator. Usage example: - ## - ## .. code-block:: - ## var a = initOrderedSet[int]() - ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: - ## a.incl(value) - ## for value in a.items: - ## echo "Got ", value - ## # --> Got 9 - ## # --> Got 2 - ## # --> Got 1 - ## # --> Got 5 - ## # --> Got 8 - ## # --> Got 4 - assert s.isValid, "The set needs to be initialized." - forAllOrderedPairs: - yield s.data[h].key - -proc hash*[A](s: OrderedSet[A]): Hash = - ## hashing of OrderedSet - assert s.isValid, "The set needs to be initialized." - forAllOrderedPairs: - result = result !& s.data[h].hcode - result = !$result - -iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = - assert s.isValid, "The set needs to be initialized" - forAllOrderedPairs: - yield (idx, s.data[h].key) - proc rawGetKnownHC[A](s: OrderedSet[A], key: A, hc: Hash): int {.inline.} = rawGetKnownHCImpl() proc rawGet[A](s: OrderedSet[A], key: A, hc: var Hash): int {.inline.} = rawGetImpl() -proc contains*[A](s: OrderedSet[A], key: A): bool = - ## Returns true iff `key` is in `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## assert(not values.contains(2)) - ## values.incl(2) - ## assert values.contains(2) - assert s.isValid, "The set needs to be initialized." - var hc: Hash - var index = rawGet(s, key, hc) - result = index >= 0 - proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A], key: A, hc: Hash, h: Hash) = rawInsertImpl() @@ -757,33 +820,7 @@ proc enlarge[A](s: var OrderedSet[A]) = rawInsert(s, s.data, n[h].key, n[h].hcode, j) h = nxt -proc incl*[A](s: var OrderedSet[A], key: A) = - ## Includes an element `key` in `s`. - ## - ## This doesn't do anything if `key` is already in `s`. Example: - ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## values.incl(2) - ## values.incl(2) - ## assert values.len == 1 - assert s.isValid, "The set needs to be initialized." - inclImpl() - -proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) = - ## Includes all elements from `other` into `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## values.incl(2) - ## var others = toOrderedSet([6, 7]) - ## values.incl(others) - ## assert values.len == 3 - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: incl(s, item) +proc isValid*[A](s: OrderedSet[A]): bool proc exclImpl[A](s: var OrderedSet[A], key: A) : bool {. inline .} = assert s.isValid, "The set needs to be initialized." @@ -806,65 +843,37 @@ proc exclImpl[A](s: var OrderedSet[A], key: A) : bool {. inline .} = rawInsert(s, s.data, n[h].key, n[h].hcode, j) h = nxt -proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool = - ## Excludes `key` in the set `s` and tells if `key` was removed from `s`. Efficiency: O(n). - ## - ## The difference with regards to the `excl() <#excl,TOrderedSet[A],A>`_ proc is - ## that this proc returns `true` if `key` was not present in `s`. Example: - ## - ## .. code-block:: - ## var s = toOrderedSet([2, 3, 6, 7]) - ## assert s.missingOrExcl(4) == true - ## assert s.missingOrExcl(6) == false - exclImpl(s, key) -proc excl*[A](s: var OrderedSet[A], key: A) = - ## Excludes `key` from the set `s`. Efficiency: O(n). - ## - ## This doesn't do anything if `key` is not found in `s`. Example: - ## - ## .. code-block:: - ## var s = toOrderedSet([2, 3, 6, 7]) - ## s.excl(2) - ## s.excl(2) - ## assert s.len == 3 - discard exclImpl(s, key) +# ----------------------------------------------------------------------- + -proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool = - ## Includes `key` in the set `s` and tells if `key` was added to `s`. - ## - ## The difference with regards to the `incl() <#incl,TOrderedSet[A],A>`_ proc - ## is that this proc returns `true` if `key` was already present in `s`. The - ## proc will return false if `key` was added as a new value to `s` during - ## this call. Example: - ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## assert values.containsOrIncl(2) == false - ## assert values.containsOrIncl(2) == true - assert s.isValid, "The set needs to be initialized." - containsOrInclImpl() proc init*[A](s: var OrderedSet[A], initialSize=64) = ## Initializes an ordered hash set. ## - ## The `initialSize` parameter needs to be a power of two. You can use - ## `math.nextPowerOfTwo() `_ or `rightSize` to - ## guarantee that at runtime. All set variables must be initialized before - ## use with other procs from this module with the exception of `isValid() - ## <#isValid,TOrderedSet[A]>`_ and `len() <#len,TOrderedSet[A]>`_. + ## The `initialSize` parameter needs to be a power of two (default: 64). + ## If you need to accept runtime values for this, you can use + ## `math.nextPowerOfTwo proc `_ or `rightSize proc + ## <#rightSize,Natural>`_ from this module. ## - ## You can call this proc on a previously initialized ordered hash set to - ## discard its values. At the moment this is the only proc to remove elements - ## from an ordered hash set. Example: + ## All set variables must be initialized before + ## use with other procs from this module, with the exception of `isValid proc + ## <#isValid,HashSet[A]>`_ and `len() <#len,HashSet[A]>`_. ## - ## .. code-block :: - ## var a: OrderedSet[int] - ## a.init(4) - ## a.incl(2) - ## a.init - ## assert a.len == 0 and a.isValid + ## You can call this proc on a previously initialized hash set, which will + ## discard all its values. This might be more convenient than iterating over + ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. + ## + ## See also: + ## * `initOrderedSet proc <#initOrderedSet,int>`_ + ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_ + runnableExamples: + var a: OrderedSet[int] + assert(not a.isValid) + init(a) + assert a.isValid + assert isPowerOfTwo(initialSize) s.counter = 0 s.first = -1 @@ -872,47 +881,215 @@ proc init*[A](s: var OrderedSet[A], initialSize=64) = newSeq(s.data, initialSize) proc initOrderedSet*[A](initialSize=64): OrderedSet[A] = - ## Wrapper around `init() <#init,TOrderedSet[A],int>`_ for initialization of + ## Wrapper around `init proc <#init,OrderedSet[A],int>`_ for initialization of ## ordered hash sets. ## - ## Returns an empty ordered hash set you can assign directly in ``var`` - ## blocks in a single line. Example: + ## Returns an empty ordered hash set you can assign directly in ``var`` blocks + ## in a single line. ## - ## .. code-block :: - ## var a = initOrderedSet[int](4) - ## a.incl(2) + ## See also: + ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_ + runnableExamples: + var a = initOrderedSet[int]() + assert a.isValid + a.incl(3) + assert len(a) == 1 result.init(initialSize) proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = - ## Creates a new ordered hash set that contains the given `keys`. + ## Creates a new hash set that contains the members of the given + ## collection (seq, array, or string) `keys`. ## - ## Example: + ## Duplicates are removed. ## - ## .. code-block:: - ## var numbers = toOrderedSet([1, 2, 3, 4, 5]) - ## assert numbers.contains(2) - ## assert numbers.contains(4) + ## See also: + ## * `initOrderedSet proc <#initOrderedSet,int>`_ + runnableExamples: + let + a = toOrderedSet([5, 3, 2]) + b = toOrderedSet("abracadabra") + assert len(a) == 3 + ## a == {5, 3, 2} # different than in HashSet + assert len(b) == 5 + ## b == {'a', 'b', 'r', 'c', 'd'} # different than in HashSet + result = initOrderedSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) -proc `$`*[A](s: OrderedSet[A]): string = - ## Converts the ordered hash set `s` to a string, mostly for logging purposes. +proc isValid*[A](s: OrderedSet[A]): bool = + ## Returns `true` if the set has been initialized (with `initSet proc + ## <#initOrderedSet,int>`_ or `init proc <#init,OrderedSet[A],int>`_). ## - ## Don't use this proc for serialization, the representation may change at - ## any moment and values are not escaped. Example: + ## Most operations over an uninitialized set will crash at runtime and + ## `assert `_ in debug builds. You can use this proc in + ## your own procs to verify that sets passed to your procs are correctly + ## initialized. ## - ## Example: + ## **Examples:** ## - ## .. code-block:: - ## echo toOrderedSet([2, 4, 5]) - ## # --> {2, 4, 5} - ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) - ## # --> {no, esc'aping, is " provided} + ## .. code-block :: + ## proc savePreferences(options: OrderedSet[string]) = + ## assert options.isValid, "Pass an initialized set!" + ## # Do stuff here, may crash in release builds! + result = s.data.len > 0 + +proc contains*[A](s: OrderedSet[A], key: A): bool = + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of `in` operator. + ## + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + assert(not values.contains(2)) + assert 2 notin values + + values.incl(2) + assert values.contains(2) + assert 2 in values + assert s.isValid, "The set needs to be initialized." - dollarImpl() + var hc: Hash + var index = rawGet(s, key, hc) + result = index >= 0 + +proc incl*[A](s: var OrderedSet[A], key: A) = + ## Includes an element `key` in `s`. + ## + ## This doesn't do anything if `key` is already in `s`. + ## + ## See also: + ## * `excl proc <#excl,OrderedSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,HashSet[A],OrderedSet[A]>`_ for including other set + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + values.incl(2) + values.incl(2) + assert values.len == 1 + + assert s.isValid, "The set needs to be initialized." + inclImpl() + +proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) = + ## Includes all elements from the OrderedSet `other` into + ## HashSet `s` (must be declared as `var`). + ## + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var + values = toSet([1, 2, 3]) + others = toOrderedSet([3, 4, 5]) + values.incl(others) + assert values.len == 5 + assert s.isValid, "The set `s` needs to be initialized." + assert other.isValid, "The set `other` needs to be initialized." + for item in other: incl(s, item) + +proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool = + ## Includes `key` in the set `s` and tells if `key` was already in `s`. + ## + ## The difference with regards to the `incl proc <#incl,OrderedSet[A],A>`_ is + ## that this proc returns `true` if `s` already contained `key`. The + ## proc will return false if `key` was added as a new value to `s` during + ## this call. + ## + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + assert values.containsOrIncl(2) == false + assert values.containsOrIncl(2) == true + assert values.containsOrIncl(3) == false + + assert s.isValid, "The set needs to be initialized." + containsOrInclImpl() + +proc excl*[A](s: var OrderedSet[A], key: A) = + ## Excludes `key` from the set `s`. Efficiency: `O(n)`. + ## + ## This doesn't do anything if `key` is not found in `s`. + ## + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,OrderedSet[A],A>`_ + runnableExamples: + var s = toOrderedSet([2, 3, 6, 7]) + s.excl(2) + s.excl(2) + assert s.len == 3 + discard exclImpl(s, key) + +proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool = + ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. + ## Efficiency: O(n). + ## + ## The difference with regards to the `excl proc <#excl,OrderedSet[A],A>`_ is + ## that this proc returns `true` if `key` was missing from `s`. + ## The proc will return `false` if `key` was in `s` and it was removed + ## during this call. + ## + ## See also: + ## * `excl proc <#excl,OrderedSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var s = toOrderedSet([2, 3, 6, 7]) + assert s.missingOrExcl(4) == true + assert s.missingOrExcl(6) == false + assert s.missingOrExcl(6) == true + exclImpl(s, key) + +proc clear*[A](s: var OrderedSet[A]) = + ## Clears the OrderedSet back to an empty state, without shrinking + ## any of the existing storage. + ## + ## `O(n)` operation where `n` is the size of the hash bucket. + runnableExamples: + var s = toOrderedSet([3, 5, 7]) + clear(s) + assert len(s) == 0 + + s.counter = 0 + s.first = -1 + s.last = -1 + for i in 0..`_. + ## + ## Card stands for the `cardinality + ## `_ of a set. + result = s.counter proc `==`*[A](s, t: OrderedSet[A]): bool = ## Equality for ordered sets. + runnableExamples: + let + a = toOrderedSet([1, 2]) + b = toOrderedSet([2, 1]) + assert(not (a == b)) + if s.counter != t.counter: return false var h = s.first var g = t.first @@ -929,6 +1106,74 @@ proc `==`*[A](s, t: OrderedSet[A]): bool = g = nxg result = compared == s.counter +proc hash*[A](s: OrderedSet[A]): Hash = + ## Hashing of OrderedSet. + assert s.isValid, "The set needs to be initialized." + forAllOrderedPairs: + result = result !& s.data[h].hcode + result = !$result + +proc `$`*[A](s: OrderedSet[A]): string = + ## Converts the ordered hash set `s` to a string, mostly for logging and + ## printing purposes. + ## + ## Don't use this proc for serialization, the representation may change at + ## any moment and values are not escaped. + ## + ## **Examples:** + ## + ## .. code-block:: + ## echo toOrderedSet([2, 4, 5]) + ## # --> {2, 4, 5} + ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) + ## # --> {no, esc'aping, is " provided} + assert s.isValid, "The set needs to be initialized." + dollarImpl() + + + +iterator items*[A](s: OrderedSet[A]): A = + ## Iterates over keys in the ordered set `s` in insertion order. + ## + ## If you need a sequence with the elelments you can use `sequtils.toSeq + ## template `_. + ## + ## .. code-block:: + ## var a = initOrderedSet[int]() + ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: + ## a.incl(value) + ## for value in a.items: + ## echo "Got ", value + ## # --> Got 9 + ## # --> Got 2 + ## # --> Got 1 + ## # --> Got 5 + ## # --> Got 8 + ## # --> Got 4 + assert s.isValid, "The set needs to be initialized." + forAllOrderedPairs: + yield s.data[h].key + + +iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = + ## Iterates through (position, value) tuples of OrderedSet `s`. + runnableExamples: + let a = toOrderedSet("abracadabra") + var p = newSeq[(int, char)]() + for x in pairs(a): + p.add(x) + assert p == @[(0, 'a'), (1, 'b'), (2, 'r'), (3, 'c'), (4, 'd')] + + assert s.isValid, "The set needs to be initialized" + forAllOrderedPairs: + yield (idx, s.data[h].key) + + + +# ----------------------------------------------------------------------- + + + when isMainModule and not defined(release): proc testModule() = ## Internal micro test to validate docstrings and such. From 68254308312ce352612b11af164d82d930a1bf68 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Jan 2019 11:17:20 +0100 Subject: [PATCH 185/350] Restrict ptr/ref to ptr/ref implicit conversion (#10411) * Restrict ptr/ref to ptr/ref implicit conversion Fixes #10409 * Make the ptr conversions explicit in db_odbc --- compiler/sigmatch.nim | 2 +- lib/impure/db_odbc.nim | 14 +++++++++----- tests/typerel/tptrs.nim | 8 ++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 tests/typerel/tptrs.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index e08559db61..fa4ab3703f 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1316,7 +1316,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, if typeRel(c, f.sons[i], a.sons[i]) == isNone: return isNone result = typeRel(c, f.lastSon, a.lastSon, flags + {trNoCovariance}) subtypeCheck() - if result <= isConvertible: result = isNone + if result <= isIntConv: result = isNone elif tfNotNil in f.flags and tfNotNil notin a.flags: result = isNilConversion elif a.kind == tyNil: result = f.allowsNil diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index b7af5128a9..a533a28fff 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -128,7 +128,7 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. cast[PSQLCHAR](sqlState.addr), cast[PSQLCHAR](nativeErr.addr), cast[PSQLCHAR](errMsg.addr), - 511.TSqlSmallInt, retSz.addr.PSQLSMALLINT) + 511.TSqlSmallInt, retSz.addr.PSQLINTEGER) except: discard return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg)) @@ -297,7 +297,8 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + cast[cstring](buf.addr), 4095.TSqlSmallInt, + sz.addr.PSQLINTEGER)) rowRes[colId-1] = $(addr buf) cCnt = tempcCnt yield rowRes @@ -332,7 +333,8 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + cast[cstring](buf.addr), 4095.TSqlSmallInt, + sz.addr.PSQLINTEGER)) rowRes[colId-1] = $(addr buf) cCnt = tempcCnt yield (row: rowRes, len: cCnt.int) @@ -374,7 +376,8 @@ proc getRow*(db: var DbConn, query: SqlQuery, for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + cast[cstring](buf.addr), 4095.TSqlSmallInt, + sz.addr.PSQLINTEGER)) rowRes[colId-1] = $(addr buf) cCnt = tempcCnt res = SQLFetch(db.stmt) @@ -409,7 +412,8 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + cast[cstring](buf.addr), 4095.TSqlSmallInt, + sz.addr.PSQLINTEGER)) rowRes[colId-1] = $(addr buf) cCnt = tempcCnt rows.add(rowRes) diff --git a/tests/typerel/tptrs.nim b/tests/typerel/tptrs.nim new file mode 100644 index 0000000000..3505a7736a --- /dev/null +++ b/tests/typerel/tptrs.nim @@ -0,0 +1,8 @@ +discard """ + errormsg: "type mismatch: got but expected 'ptr int'" + line: 8 +""" + +var + n: int16 + p: ptr int = addr n From ced62f330758c3a3a01a613ed9be4023fd65de1d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2019 11:40:22 +0100 Subject: [PATCH 186/350] compiler interface: don't overdocument the on|off switches --- doc/advopt.txt | 5 +++-- doc/basicopt.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 7ab85abfc5..f713839d85 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -97,8 +97,9 @@ Advanced options: symbol matching is fuzzy so that --dynlibOverride:lua matches dynlib: "liblua.so.3" - --dynlibOverrideAll:on|off makes the dynlib pragma have no effect - --listCmd:on|off list the commands used to execute external programs + --dynlibOverrideAll + disables the effects of the dynlib pragma + --listCmd list the commands used to execute external programs --parallelBuild:0|1|... perform a parallel build value = number of processors (0 for auto-detect) --incremental:on|off only recompile the changed modules (experimental!) diff --git a/doc/basicopt.txt b/doc/basicopt.txt index 96c9ced3df..a65b2302e5 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -35,7 +35,7 @@ Options: --debugger:native|endb use native debugger (gdb) | ENDB (experimental) --app:console|gui|lib|staticlib generate a console app|GUI app|DLL|static library - -r, --run:on|off run the compiled program with given arguments + -r, --run run the compiled program with given arguments --fullhelp show all command line switches -h, --help show this help From 458c8270773d7f1bb70797b6bc16df5d54ffb45d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2019 11:45:12 +0100 Subject: [PATCH 187/350] make --define:noSignalHandler compile again --- lib/system.nim | 2 +- lib/system/excpt.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index d4a86e53be..0241b92e08 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3463,7 +3463,7 @@ when not defined(JS): #and not defined(nimscript): ## allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. - when not defined(useNimRtl): + when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook*() ## reverts a call to setControlCHook diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 3efefead4a..e9d251bb4c 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -537,7 +537,7 @@ proc setControlCHook(hook: proc () {.noconv.}) = type SignalHandler = proc (sign: cint) {.noconv, benign.} c_signal(SIGINT, cast[SignalHandler](hook)) -when not defined(useNimRtl): +when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook() = # proc to unset a hook set by setControlCHook c_signal(SIGINT, signalHandler) From a6a014a85982f5eba70862740ca28456b894c554 Mon Sep 17 00:00:00 2001 From: Jjp137 Date: Tue, 22 Jan 2019 05:52:43 -0800 Subject: [PATCH 188/350] better docs: parseopt (#10398) --- lib/pure/parseopt.nim | 315 +++++++++++++++++++++++++++++++++++------- 1 file changed, 266 insertions(+), 49 deletions(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 06f32f0326..0f8f8197c6 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -11,23 +11,141 @@ ## It supports one convenience iterator over all command line options and some ## lower-level features. ## -## Supported syntax with default empty ``shortNoVal``/``longNoVal``: +## Supported Syntax +## ================ ## -## 1. short options - ``-abcd``, where a, b, c, d are names -## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` -## 3. argument - everything else +## The following syntax is supported when arguments for the ``shortNoVal`` and +## ``longNoVal`` parameters, which are +## `described later<#shortnoval-and-longnoval>`_, are not provided: ## -## When ``shortNoVal``/``longNoVal`` are non-empty then the ':' and '=' above -## are still accepted, but become optional. Note that these option key sets -## must be updated along with the set of option keys taking no value, but -## keys which do take values need no special updates as their set evolves. +## 1. Short options: ``-abcd``, ``-e:5``, ``-e=5`` +## 2. Long options: ``--foo:bar``, ``--foo=bar``, ``--foo`` +## 3. Arguments: everything that does not start with a ``-`` ## -## When option values begin with ':' or '=' they need to be doubled up (as in +## These three kinds of tokens are enumerated in the +## `CmdLineKind enum<#CmdLineKind>`_. +## +## When option values begin with ':' or '=', they need to be doubled up (as in ## ``--delim::``) or alternated (as in ``--delim=:``). ## -## The common ``--`` non-option argument delimiter appears as an empty string -## long option key. ``OptParser.cmd``, ``OptParser.pos``, and -## ``os.parseCmdLine`` may be used to complete parsing in that case. +## The ``--`` option, commonly used to denote that every token that follows is +## an argument, is interpreted as a long option, and its name is the empty +## string. +## +## Parsing +## ======= +## +## Use an `OptParser<#OptParser>`_ to parse command line options. It can be +## created with `initOptParser<#initOptParser,string,set[char],seq[string]>`_, +## and `next<#next,OptParser>`_ advances the parser by one token. +## +## For each token, the parser's ``kind``, ``key``, and ``val`` fields give +## information about that token. If the token is a long or short option, ``key`` +## is the option's name, and ``val`` is either the option's value, if provided, +## or the empty string. For arguments, the ``key`` field contains the argument +## itself, and ``val`` is unused. To check if the end of the command line has +## been reached, check if ``kind`` is equal to ``cmdEnd``. +## +## Here is an example: +## +## .. code-block:: +## import parseopt +## +## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") +## while true: +## p.next() +## case p.kind +## of cmdEnd: break +## of cmdShortOption, cmdLongOption: +## if p.val == "": +## echo "Option: ", p.key +## else: +## echo "Option and value: ", p.key, ", ", p.val +## of cmdArgument: +## echo "Argument: ", p.key +## +## # Output: +## # Option: a +## # Option: b +## # Option and value: e, 5 +## # Option: foo +## # Option and value: bar, 20 +## # Argument: file.txt +## +## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for +## convenience, can be used to iterate through all command line options as well. +## +## ``shortNoVal`` and ``longNoVal`` +## ================================ +## +## The optional ``shortNoVal`` and ``longNoVal`` parameters present in +## `initOptParser<#initOptParser,string,set[char],seq[string]>`_ are for +## specifying which short and long options do not accept values. +## +## When ``shortNoVal`` is non-empty, users are not required to separate short +## options and their values with a ':' or '=' since the parser knows which +## options accept values and which ones do not. This behavior also applies for +## long options if ``longNoVal`` is non-empty. For short options, ``-j4`` +## becomes supported syntax, and for long options, ``--foo bar`` becomes +## supported. This is in addition to the `previously mentioned +## syntax<#supported-syntax>`_. Users can still separate options and their +## values with ':' or '=', but that becomes optional. +## +## As more options which do not accept values are added to your program, +## remember to amend ``shortNoVal`` and ``longNoVal`` accordingly. +## +## The following example illustrates the difference between having an empty +## ``shortNoVal`` and ``longNoVal``, which is the default, and providing +## arguments for those two parameters: +## +## .. code-block:: +## import parseopt +## +## proc printToken(kind: CmdLineKind, key: string, val: string) = +## case kind +## of cmdEnd: doAssert(false) # Doesn't happen with getopt() +## of cmdShortOption, cmdLongOption: +## if val == "": +## echo "Option: ", key +## else: +## echo "Option and value: ", key, ", ", val +## of cmdArgument: +## echo "Argument: ", key +## +## let cmdLine = "-j4 --first bar" +## +## var emptyNoVal = initOptParser(cmdLine) +## for kind, key, val in emptyNoVal.getopt(): +## printToken(kind, key, val) +## +## # Output: +## # Option: j +## # Option: 4 +## # Option: first +## # Argument: bar +## +## var withNoVal = initOptParser(cmdLine, shortNoVal = {'c'}, +## longNoVal = @["second"]) +## for kind, key, val in withNoVal.getopt(): +## printToken(kind, key, val) +## +## # Output: +## # Option and value: j, 4 +## # Option and value: first, bar +## +## See also +## ======== +## +## * `os module`_ for lower-level command line parsing procs +## * `parseutils module`_ for helpers that parse tokens, +## numbers, identifiers, etc. +## * `strutils module`_ for common string handling operations +## * `json module`_ for a JSON parser +## * `parsecfg module`_ for a configuration file parser +## * `parsecsv module`_ for a simple CSV (comma separated value) +## parser +## * `parsexml module`_ for a XML / HTML parser +## * `other parsers`_ for more parsers {.push debugger: off.} @@ -37,23 +155,26 @@ import os, strutils type - CmdLineKind* = enum ## the detected command line token - cmdEnd, ## end of command line reached - cmdArgument, ## argument detected - cmdLongOption, ## a long option ``--option`` detected - cmdShortOption ## a short option ``-c`` detected + CmdLineKind* = enum ## The detected command line token. + cmdEnd, ## End of command line reached + cmdArgument, ## An argument such as a filename + cmdLongOption, ## A long option such as --option + cmdShortOption ## A short option such as -c OptParser* = - object of RootObj ## this object implements the command line parser - pos*: int # ..empty key or subcmd cmdArg & handle specially + object of RootObj ## Implementation of the command line parser. + ## + ## To initialize it, use the + ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. + pos*: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] longNoVal: seq[string] cmds: seq[string] idx: int - kind*: CmdLineKind ## the dected command line token - key*, val*: TaintedString ## key and value pair; ``key`` is the option - ## or the argument, ``value`` is not "" if + kind*: CmdLineKind ## The detected command line token + key*, val*: TaintedString ## Key and value pair; the key is the option + ## or the argument, and the value is not "" if ## the option was given a value proc parseWord(s: string, i: int, w: var string, @@ -79,13 +200,24 @@ when declared(os.paramCount): proc initOptParser*(cmdline = "", shortNoVal: set[char]={}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = - ## inits the option parser. If ``cmdline == ""``, the real command line - ## (as provided by the ``OS`` module) is taken. If ``shortNoVal`` is - ## provided command users do not need to delimit short option keys and - ## values with a ':' or '='. If ``longNoVal`` is provided command users do - ## not need to delimit long option keys and values with a ':' or '=' - ## (though they still need at least a space). In both cases, ':' or '=' - ## may still be used if desired. They just become optional. + ## Initializes the command line parser. + ## + ## If ``cmdline == ""``, the real command line as provided by the + ## ``os`` module is retrieved instead. + ## + ## ``shortNoVal`` and ``longNoVal`` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## how this affects parsing. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + result.pos = 0 result.idx = 0 result.inShortState = false @@ -106,9 +238,21 @@ when declared(os.paramCount): proc initOptParser*(cmdline: seq[TaintedString], shortNoVal: set[char]={}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = - ## inits the option parser. If ``cmdline.len == 0``, the real command line - ## (as provided by the ``OS`` module) is taken. ``shortNoVal`` and - ## ``longNoVal`` behavior is the same as for ``initOptParser(string,...)``. + ## Initializes the command line parser. + ## + ## If ``cmdline.len == 0``, the real command line as provided by the + ## ``os`` module is retrieved instead. Behavior of the other parameters + ## remains the same as in `initOptParser(string, ...) + ## <#initOptParser,string,set[char],seq[string]>`_. + ## + ## See also: + ## * `getopt iterator<#getopt.i,seq[TaintedString],set[char],seq[string]>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], + shortNoVal = {'l'}, longNoVal = @["left"]) + result.pos = 0 result.idx = 0 result.inShortState = false @@ -153,8 +297,21 @@ proc handleShortOption(p: var OptParser; cmd: string) = inc p.idx proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = - ## parses the first or next option; ``p.kind`` describes what token has been - ## parsed. ``p.key`` and ``p.val`` are set accordingly. + ## Parses the next token. + ## + ## ``p.kind`` describes what kind of token has been parsed. ``p.key`` and + ## ``p.val`` are set accordingly. + runnableExamples: + var p = initOptParser("--left -r:2 file.txt") + p.next() + doAssert p.kind == cmdLongOption and p.key == "left" + p.next() + doAssert p.kind == cmdShortOption and p.key == "r" and p.val == "2" + p.next() + doAssert p.kind == cmdArgument and p.key == "file.txt" + p.next() + doAssert p.kind == cmdEnd + if p.idx >= p.cmds.len: p.kind = cmdEnd return @@ -209,20 +366,61 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = when declared(os.paramCount): proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = - ## retrieves the rest of the command line that has not been parsed yet. + ## Retrieves the rest of the command line that has not been parsed yet. + ## + ## See also: + ## * `remainingArgs proc<#remainingArgs,OptParser>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") + ## while true: + ## p.next() + ## if p.kind == cmdLongOption and p.key == "": # Look for "--" + ## break + ## else: continue + ## doAssert p.cmdLineRest == "foo.txt bar.txt" result = p.cmds[p.idx .. ^1].quoteShellCommand.TaintedString proc remainingArgs*(p: OptParser): seq[TaintedString] {.rtl, extern: "npo$1".} = - ## retrieves the rest of the command line that has not been parsed yet. + ## Retrieves a sequence of the arguments that have not been parsed yet. + ## + ## See also: + ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") + ## while true: + ## p.next() + ## if p.kind == cmdLongOption and p.key == "": # Look for "--" + ## break + ## else: continue + ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] result = @[] for i in p.idx..`_. ## - ## .. code-block:: nim + ## There is no need to check for ``cmdEnd`` while iterating. + ## + ## See also: + ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## # these are placeholders, of course + ## proc writeHelp() = discard + ## proc writeVersion() = discard + ## + ## var filename: string ## var p = initOptParser("--left --debug:3 -l -r:2") + ## ## for kind, key, val in p.getopt(): ## case kind ## of cmdArgument: @@ -233,7 +431,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedSt ## of "version", "v": writeVersion() ## of cmdEnd: assert(false) # cannot happen ## if filename == "": - ## # no filename has been given, so we show the help: + ## # no filename has been given, so we show the help ## writeHelp() p.pos = 0 p.idx = 0 @@ -246,15 +444,34 @@ when declared(initOptParser): iterator getopt*(cmdline: seq[TaintedString] = commandLineParams(), shortNoVal: set[char]={}, longNoVal: seq[string] = @[]): tuple[kind: CmdLineKind, key, val: TaintedString] = - ## This is an convenience iterator for iterating over command line arguments. - ## This creates a new OptParser. See the above ``getopt(var OptParser)`` - ## example for using default empty ``NoVal`` parameters. This example is - ## for the same option keys as that example but here option key-value - ## separators become optional for command users: + ## Convenience iterator for iterating over command line arguments. ## - ## .. code-block:: nim - ## for kind, key, val in getopt(shortNoVal = { 'l' }, - ## longNoVal = @[ "left" ]): + ## This creates a new `OptParser<#OptParser>`_. If no command line + ## arguments are provided, the real command line as provided by the + ## ``os`` module is retrieved instead. + ## + ## ``shortNoVal`` and ``longNoVal`` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## how this affects parsing. + ## + ## There is no need to check for ``cmdEnd`` while iterating. + ## + ## See also: + ## * `initOptParser proc<#initOptParser,seq[TaintedString],set[char],seq[string]>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## + ## # these are placeholders, of course + ## proc writeHelp() = discard + ## proc writeVersion() = discard + ## + ## var filename: string + ## let params = @["--left", "--debug:3", "-l", "-r:2"] + ## + ## for kind, key, val in getopt(params): ## case kind ## of cmdArgument: ## filename = key @@ -264,8 +481,8 @@ when declared(initOptParser): ## of "version", "v": writeVersion() ## of cmdEnd: assert(false) # cannot happen ## if filename == "": + ## # no filename has been written, so we show the help ## writeHelp() - ## var p = initOptParser(cmdline, shortNoVal=shortNoVal, longNoVal=longNoVal) while true: next(p) From 54fecdb8dfa8a5721cb95b63cc8716283ee754d9 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 22 Jan 2019 14:50:50 +0000 Subject: [PATCH 189/350] Enable log folding in Travis CI (#10414) Prevent log truncation in browsers --- koch.nim | 25 ++++++++++++++----------- tools/kochdocs.nim | 9 +++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/koch.nim b/koch.nim index f70cf21425..d21c5240e5 100644 --- a/koch.nim +++ b/koch.nim @@ -80,6 +80,9 @@ let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch proc kochExec*(cmd: string) = exec kochExe.quoteShell & " " & cmd +proc kochExecFold*(desc, cmd: string) = + execFold(desc, kochExe.quoteShell & " " & cmd) + template withDir(dir, body) = let old = getCurrentDir() try: @@ -453,11 +456,11 @@ proc runCI(cmd: string) = # note(@araq): Do not replace these commands with direct calls (eg boot()) # as that would weaken our testing efforts. when defined(posix): # appveyor (on windows) didn't run this - kochExec "boot" - kochExec "boot -d:release" + kochExecFold("Boot", "boot") + kochExecFold("Boot in release mode", "boot -d:release") ## build nimble early on to enable remainder to depend on it if needed - kochExec "nimble" + kochExecFold("Build Nimble", "nimble") when false: for pkg in "zip opengl sdl1 jester@#head niminst".split: @@ -466,23 +469,23 @@ proc runCI(cmd: string) = buildTools() # altenatively, kochExec "tools --toolsNoNimble" ## run tests - exec "nim e tests/test_nimscript.nims" + execFold("Test nimscript", "nim e tests/test_nimscript.nims") when defined(windows): # note: will be over-written below - exec "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester" + execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester") # main bottleneck here - exec "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines" + execFold("Run tester", "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines") - exec "nim c -r nimdoc/tester" - exec "nim c -r nimpretty/tester.nim" + execFold("Run nimdoc tests", "nim c -r nimdoc/tester") + execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim") when defined(posix): - exec "nim c -r nimsuggest/tester" + execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester") ## remaining actions when defined(posix): - kochExec "docs --git.commit:devel" - kochExec "csource" + kochExecFold("Docs", "docs --git.commit:devel") + kochExecFold("C sources", "csource") elif defined(windows): when false: kochExec "csource" diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 376f33ae50..fc53c1eea0 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -38,6 +38,15 @@ proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) +proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + ## Execute shell command. Add log folding on Travis CI. + # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 + if existsEnv("TRAVIS"): + echo "travis_fold:start:" & desc.replace(" ", "") + exec(cmd, errorcode, additionalPath) + if existsEnv("TRAVIS"): + echo "travis_fold:end:" & desc.replace(" ", "") + proc execCleanPath*(cmd: string, additionalPath = ""; errorcode: int = QuitFailure) = # simulate a poor man's virtual environment From 0ebfcd4c0f6a5c287c23c188c28df242e2349747 Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 22 Jan 2019 16:06:44 +0100 Subject: [PATCH 190/350] Remove deprecated modules (asyncio, sockets, ftpclient) (#10401) --- lib/deprecated/pure/asyncio.nim | 711 ------- lib/deprecated/pure/ftpclient.nim | 669 ------- lib/deprecated/pure/sockets.nim | 1748 ----------------- lib/pure/asyncdispatch.nim | 43 +- lib/pure/asyncftpclient.nim | 52 +- tests/async/tasyncawait.nim | 14 +- tests/bind/tnicerrorforsymchoice.nim | 2 +- tests/closure/texplicit_dummy_closure.nim | 3 + .../manyloc/keineschweine/lib/sg_packets.nim | 2 +- tests/types/tillegaltyperecursion.nim | 2 +- tools/kochdocs.nim | 2 - 11 files changed, 61 insertions(+), 3187 deletions(-) delete mode 100644 lib/deprecated/pure/asyncio.nim delete mode 100644 lib/deprecated/pure/ftpclient.nim delete mode 100644 lib/deprecated/pure/sockets.nim diff --git a/lib/deprecated/pure/asyncio.nim b/lib/deprecated/pure/asyncio.nim deleted file mode 100644 index 2dd08024da..0000000000 --- a/lib/deprecated/pure/asyncio.nim +++ /dev/null @@ -1,711 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf, Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import sockets, os - -## -## **Warning:** This module is deprecated since version 0.10.2. -## Use the brand new `asyncdispatch `_ module together -## with the `asyncnet `_ module. - -## This module implements an asynchronous event loop together with asynchronous -## sockets which use this event loop. -## It is akin to Python's asyncore module. Many modules that use sockets -## have an implementation for this module, those modules should all have a -## ``register`` function which you should use to add the desired objects to a -## dispatcher which you created so -## that you can receive the events associated with that module's object. -## -## Once everything is registered in a dispatcher, you need to call the ``poll`` -## function in a while loop. -## -## **Note:** Most modules have tasks which need to be ran regularly, this is -## why you should not call ``poll`` with a infinite timeout, or even a -## very long one. In most cases the default timeout is fine. -## -## **Note:** This module currently only supports select(), this is limited by -## FD_SETSIZE, which is usually 1024. So you may only be able to use 1024 -## sockets at a time. -## -## Most (if not all) modules that use asyncio provide a userArg which is passed -## on with the events. The type that you set userArg to must be inheriting from -## ``RootObj``! -## -## **Note:** If you want to provide async ability to your module please do not -## use the ``Delegate`` object, instead use ``AsyncSocket``. It is possible -## that in the future this type's fields will not be exported therefore breaking -## your code. -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. -## -## Asynchronous sockets -## ==================== -## -## For most purposes you do not need to worry about the ``Delegate`` type. The -## ``AsyncSocket`` is what you are after. It's a reference to -## the ``AsyncSocketObj`` object. This object defines events which you should -## overwrite by your own procedures. -## -## For server sockets the only event you need to worry about is the ``handleAccept`` -## event, in your handleAccept proc you should call ``accept`` on the server -## socket which will give you the client which is connecting. You should then -## set any events that you want to use on that client and add it to your dispatcher -## using the ``register`` procedure. -## -## An example ``handleAccept`` follows: -## -## .. code-block:: nim -## -## var disp = newDispatcher() -## ... -## proc handleAccept(s: AsyncSocket) = -## echo("Accepted client.") -## var client: AsyncSocket -## new(client) -## s.accept(client) -## client.handleRead = ... -## disp.register(client) -## ... -## -## For client sockets you should only be interested in the ``handleRead`` and -## ``handleConnect`` events. The former gets called whenever the socket has -## received messages and can be read from and the latter gets called whenever -## the socket has established a connection to a server socket; from that point -## it can be safely written to. -## -## Getting a blocking client from an AsyncSocket -## ============================================= -## -## If you need a asynchronous server socket but you wish to process the clients -## synchronously then you can use the ``getSocket`` converter to get -## a ``Socket`` from the ``AsyncSocket`` object, this can then be combined -## with ``accept`` like so: -## -## .. code-block:: nim -## -## proc handleAccept(s: AsyncSocket) = -## var client: Socket -## getSocket(s).accept(client) - -{.deprecated.} - -when defined(windows): - from winlean import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet, - FD_ISSET, select -else: - from posix import TimeVal, Time, Suseconds, SocketHandle, FD_SET, FD_ZERO, - TFdSet, FD_ISSET, select - -type - DelegateObj* = object - fd*: SocketHandle - deleVal*: RootRef - - handleRead*: proc (h: RootRef) {.nimcall, gcsafe.} - handleWrite*: proc (h: RootRef) {.nimcall, gcsafe.} - handleError*: proc (h: RootRef) {.nimcall, gcsafe.} - hasDataBuffered*: proc (h: RootRef): bool {.nimcall, gcsafe.} - - open*: bool - task*: proc (h: RootRef) {.nimcall, gcsafe.} - mode*: FileMode - - Delegate* = ref DelegateObj - - Dispatcher* = ref DispatcherObj - DispatcherObj = object - delegates: seq[Delegate] - - AsyncSocket* = ref AsyncSocketObj - AsyncSocketObj* = object of RootObj - socket: Socket - info: SocketStatus - - handleRead*: proc (s: AsyncSocket) {.closure, gcsafe.} - handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.} - handleConnect*: proc (s: AsyncSocket) {.closure, gcsafe.} - - handleAccept*: proc (s: AsyncSocket) {.closure, gcsafe.} - - handleTask*: proc (s: AsyncSocket) {.closure, gcsafe.} - - lineBuffer: TaintedString ## Temporary storage for ``readLine`` - sendBuffer: string ## Temporary storage for ``send`` - sslNeedAccept: bool - proto: Protocol - deleg: Delegate - - SocketStatus* = enum - SockIdle, SockConnecting, SockConnected, SockListening, SockClosed, - SockUDPBound - - -proc newDelegate*(): Delegate = - ## Creates a new delegate. - new(result) - result.handleRead = (proc (h: RootRef) = discard) - result.handleWrite = (proc (h: RootRef) = discard) - result.handleError = (proc (h: RootRef) = discard) - result.hasDataBuffered = (proc (h: RootRef): bool = return false) - result.task = (proc (h: RootRef) = discard) - result.mode = fmRead - -proc newAsyncSocket(): AsyncSocket = - new(result) - result.info = SockIdle - - result.handleRead = (proc (s: AsyncSocket) = discard) - result.handleWrite = nil - result.handleConnect = (proc (s: AsyncSocket) = discard) - result.handleAccept = (proc (s: AsyncSocket) = discard) - result.handleTask = (proc (s: AsyncSocket) = discard) - - result.lineBuffer = "".TaintedString - result.sendBuffer = "" - -proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, - buffered = true): AsyncSocket = - ## Initialises an AsyncSocket object. If a socket cannot be initialised - ## OSError is raised. - result = newAsyncSocket() - result.socket = socket(domain, typ, protocol, buffered) - result.proto = protocol - if result.socket == invalidSocket: raiseOSError(osLastError()) - result.socket.setBlocking(false) - -proc toAsyncSocket*(sock: Socket, state: SocketStatus = SockConnected): AsyncSocket = - ## Wraps an already initialized ``Socket`` into a AsyncSocket. - ## This is useful if you want to use an already connected Socket as an - ## asynchronous AsyncSocket in asyncio's event loop. - ## - ## ``state`` may be overriden, i.e. if ``sock`` is not connected it should be - ## adjusted properly. By default it will be assumed that the socket is - ## connected. Please note this is only applicable to TCP client sockets, if - ## ``sock`` is a different type of socket ``state`` needs to be adjusted!!! - ## - ## ================ ================================================================ - ## Value Meaning - ## ================ ================================================================ - ## SockIdle Socket has only just been initialised, not connected or closed. - ## SockConnected Socket is connected to a server. - ## SockConnecting Socket is in the process of connecting to a server. - ## SockListening Socket is a server socket and is listening for connections. - ## SockClosed Socket has been closed. - ## SockUDPBound Socket is a UDP socket which is listening for data. - ## ================ ================================================================ - ## - ## **Warning**: If ``state`` is set incorrectly the resulting ``AsyncSocket`` - ## object may not work properly. - ## - ## **Note**: This will set ``sock`` to be non-blocking. - result = newAsyncSocket() - result.socket = sock - result.proto = if state == SockUDPBound: IPPROTO_UDP else: IPPROTO_TCP - result.socket.setBlocking(false) - result.info = state - -proc asyncSockHandleRead(h: RootRef) = - when defined(ssl): - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - return - - if AsyncSocket(h).info != SockListening: - if AsyncSocket(h).info != SockConnecting: - AsyncSocket(h).handleRead(AsyncSocket(h)) - else: - AsyncSocket(h).handleAccept(AsyncSocket(h)) - -proc close*(sock: AsyncSocket) {.gcsafe.} -proc asyncSockHandleWrite(h: RootRef) = - when defined(ssl): - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - return - - if AsyncSocket(h).info == SockConnecting: - AsyncSocket(h).handleConnect(AsyncSocket(h)) - AsyncSocket(h).info = SockConnected - # Stop receiving write events if there is no handleWrite event. - if AsyncSocket(h).handleWrite == nil: - AsyncSocket(h).deleg.mode = fmRead - else: - AsyncSocket(h).deleg.mode = fmReadWrite - else: - if AsyncSocket(h).sendBuffer != "": - let sock = AsyncSocket(h) - try: - let bytesSent = sock.socket.sendAsync(sock.sendBuffer) - if bytesSent == 0: - # Apparently the socket cannot be written to. Even though select - # just told us that it can be... This used to be an assert. Just - # do nothing instead. - discard - elif bytesSent != sock.sendBuffer.len: - sock.sendBuffer = sock.sendBuffer[bytesSent .. ^1] - elif bytesSent == sock.sendBuffer.len: - sock.sendBuffer = "" - - if AsyncSocket(h).handleWrite != nil: - AsyncSocket(h).handleWrite(AsyncSocket(h)) - except OSError: - # Most likely the socket closed before the full buffer could be sent to it. - sock.close() # TODO: Provide a handleError for users? - else: - if AsyncSocket(h).handleWrite != nil: - AsyncSocket(h).handleWrite(AsyncSocket(h)) - else: - AsyncSocket(h).deleg.mode = fmRead - -when defined(ssl): - proc asyncSockDoHandshake(h: RootRef) {.gcsafe.} = - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - if AsyncSocket(h).sslNeedAccept: - var d = "" - let ret = AsyncSocket(h).socket.acceptAddrSSL(AsyncSocket(h).socket, d) - assert ret != AcceptNoClient - if ret == AcceptSuccess: - AsyncSocket(h).info = SockConnected - else: - # handshake will set socket's ``sslNoHandshake`` field. - discard AsyncSocket(h).socket.handshake() - - -proc asyncSockTask(h: RootRef) = - when defined(ssl): - h.asyncSockDoHandshake() - - AsyncSocket(h).handleTask(AsyncSocket(h)) - -proc toDelegate(sock: AsyncSocket): Delegate = - result = newDelegate() - result.deleVal = sock - result.fd = getFD(sock.socket) - # We need this to get write events, just to know when the socket connects. - result.mode = fmReadWrite - result.handleRead = asyncSockHandleRead - result.handleWrite = asyncSockHandleWrite - result.task = asyncSockTask - # TODO: Errors? - #result.handleError = (proc (h: PObject) = assert(false)) - - result.hasDataBuffered = - proc (h: RootRef): bool {.nimcall.} = - return AsyncSocket(h).socket.hasDataBuffered() - - sock.deleg = result - if sock.info notin {SockIdle, SockClosed}: - sock.deleg.open = true - else: - sock.deleg.open = false - -proc connect*(sock: AsyncSocket, name: string, port = Port(0), - af: Domain = AF_INET) = - ## Begins connecting ``sock`` to ``name``:``port``. - sock.socket.connectAsync(name, port, af) - sock.info = SockConnecting - if sock.deleg != nil: - sock.deleg.open = true - -proc close*(sock: AsyncSocket) = - ## Closes ``sock``. Terminates any current connections. - sock.socket.close() - sock.info = SockClosed - if sock.deleg != nil: - sock.deleg.open = false - -proc bindAddr*(sock: AsyncSocket, port = Port(0), address = "") = - ## Equivalent to ``sockets.bindAddr``. - sock.socket.bindAddr(port, address) - if sock.proto == IPPROTO_UDP: - sock.info = SockUDPBound - if sock.deleg != nil: - sock.deleg.open = true - -proc listen*(sock: AsyncSocket) = - ## Equivalent to ``sockets.listen``. - sock.socket.listen() - sock.info = SockListening - if sock.deleg != nil: - sock.deleg.open = true - -proc acceptAddr*(server: AsyncSocket, client: var AsyncSocket, - address: var string) = - ## Equivalent to ``sockets.acceptAddr``. This procedure should be called in - ## a ``handleAccept`` event handler **only** once. - ## - ## **Note**: ``client`` needs to be initialised. - assert(client != nil) - client = newAsyncSocket() - var c: Socket - new(c) - when defined(ssl): - if server.socket.isSSL: - var ret = server.socket.acceptAddrSSL(c, address) - # The following shouldn't happen because when this function is called - # it is guaranteed that there is a client waiting. - # (This should be called in handleAccept) - assert(ret != AcceptNoClient) - if ret == AcceptNoHandshake: - client.sslNeedAccept = true - else: - client.sslNeedAccept = false - client.info = SockConnected - else: - server.socket.acceptAddr(c, address) - client.sslNeedAccept = false - client.info = SockConnected - else: - server.socket.acceptAddr(c, address) - client.sslNeedAccept = false - client.info = SockConnected - - if c == invalidSocket: raiseSocketError(server.socket) - c.setBlocking(false) # TODO: Needs to be tested. - - # deleg.open is set in ``toDelegate``. - - client.socket = c - client.lineBuffer = "".TaintedString - client.sendBuffer = "" - client.info = SockConnected - -proc accept*(server: AsyncSocket, client: var AsyncSocket) = - ## Equivalent to ``sockets.accept``. - var dummyAddr = "" - server.acceptAddr(client, dummyAddr) - -proc acceptAddr*(server: AsyncSocket): tuple[sock: AsyncSocket, - address: string] {.deprecated.} = - ## Equivalent to ``sockets.acceptAddr``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - var client = newAsyncSocket() - var address: string = "" - acceptAddr(server, client, address) - return (client, address) - -proc accept*(server: AsyncSocket): AsyncSocket {.deprecated.} = - ## Equivalent to ``sockets.accept``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - new(result) - var address = "" - server.acceptAddr(result, address) - -proc newDispatcher*(): Dispatcher = - new(result) - result.delegates = @[] - -proc register*(d: Dispatcher, deleg: Delegate) = - ## Registers delegate ``deleg`` with dispatcher ``d``. - d.delegates.add(deleg) - -proc register*(d: Dispatcher, sock: AsyncSocket): Delegate {.discardable.} = - ## Registers async socket ``sock`` with dispatcher ``d``. - result = sock.toDelegate() - d.register(result) - -proc unregister*(d: Dispatcher, deleg: Delegate) = - ## Unregisters deleg ``deleg`` from dispatcher ``d``. - for i in 0..len(d.delegates)-1: - if d.delegates[i] == deleg: - d.delegates.del(i) - return - raise newException(IndexError, "Could not find delegate.") - -proc isWriteable*(s: AsyncSocket): bool = - ## Determines whether socket ``s`` is ready to be written to. - var writeSock = @[s.socket] - return selectWrite(writeSock, 1) != 0 and s.socket notin writeSock - -converter getSocket*(s: AsyncSocket): Socket = - return s.socket - -proc isConnected*(s: AsyncSocket): bool = - ## Determines whether ``s`` is connected. - return s.info == SockConnected -proc isListening*(s: AsyncSocket): bool = - ## Determines whether ``s`` is listening for incoming connections. - return s.info == SockListening -proc isConnecting*(s: AsyncSocket): bool = - ## Determines whether ``s`` is connecting. - return s.info == SockConnecting -proc isClosed*(s: AsyncSocket): bool = - ## Determines whether ``s`` has been closed. - return s.info == SockClosed -proc isSendDataBuffered*(s: AsyncSocket): bool = - ## Determines whether ``s`` has data waiting to be sent, i.e. whether this - ## socket's sendBuffer contains data. - return s.sendBuffer.len != 0 - -proc setHandleWrite*(s: AsyncSocket, - handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.}) = - ## Setter for the ``handleWrite`` event. - ## - ## To remove this event you should use the ``delHandleWrite`` function. - ## It is advised to use that function instead of just setting the event to - ## ``proc (s: AsyncSocket) = nil`` as that would mean that that function - ## would be called constantly. - s.deleg.mode = fmReadWrite - s.handleWrite = handleWrite - -proc delHandleWrite*(s: AsyncSocket) = - ## Removes the ``handleWrite`` event handler on ``s``. - s.handleWrite = nil - -{.push warning[deprecated]: off.} -proc recvLine*(s: AsyncSocket, line: var TaintedString): bool {.deprecated.} = - ## Behaves similar to ``sockets.recvLine``, however it handles non-blocking - ## sockets properly. This function guarantees that ``line`` is a full line, - ## if this function can only retrieve some data; it will save this data and - ## add it to the result when a full line is retrieved. - ## - ## Unlike ``sockets.recvLine`` this function will raise an OSError or SslError - ## exception if an error occurs. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLine. - setLen(line.string, 0) - var dataReceived = "".TaintedString - var ret = s.socket.recvLineAsync(dataReceived) - case ret - of RecvFullLine: - if s.lineBuffer.len > 0: - string(line).add(s.lineBuffer.string) - setLen(s.lineBuffer.string, 0) - string(line).add(dataReceived.string) - if string(line) == "": - line = "\c\L".TaintedString - result = true - of RecvPartialLine: - string(s.lineBuffer).add(dataReceived.string) - result = false - of RecvDisconnected: - result = true - of RecvFail: - s.raiseSocketError(async = true) - result = false -{.pop.} - -proc readLine*(s: AsyncSocket, line: var TaintedString): bool = - ## Behaves similar to ``sockets.readLine``, however it handles non-blocking - ## sockets properly. This function guarantees that ``line`` is a full line, - ## if this function can only retrieve some data; it will save this data and - ## add it to the result when a full line is retrieved, when this happens - ## False will be returned. True will only be returned if a full line has been - ## retrieved or the socket has been disconnected in which case ``line`` will - ## be set to "". - ## - ## This function will raise an OSError exception when a socket error occurs. - setLen(line.string, 0) - var dataReceived = "".TaintedString - var ret = s.socket.readLineAsync(dataReceived) - case ret - of ReadFullLine: - if s.lineBuffer.len > 0: - string(line).add(s.lineBuffer.string) - setLen(s.lineBuffer.string, 0) - string(line).add(dataReceived.string) - if string(line) == "": - line = "\c\L".TaintedString - result = true - of ReadPartialLine: - string(s.lineBuffer).add(dataReceived.string) - result = false - of ReadNone: - result = false - of ReadDisconnected: - result = true - -proc send*(sock: AsyncSocket, data: string) = - ## Sends ``data`` to socket ``sock``. This is basically a nicer implementation - ## of ``sockets.sendAsync``. - ## - ## If ``data`` cannot be sent immediately it will be buffered and sent - ## when ``sock`` becomes writeable (during the ``handleWrite`` event). - ## It's possible that only a part of ``data`` will be sent immediately, while - ## the rest of it will be buffered and sent later. - if sock.sendBuffer.len != 0: - sock.sendBuffer.add(data) - return - let bytesSent = sock.socket.sendAsync(data) - assert bytesSent >= 0 - if bytesSent == 0: - sock.sendBuffer.add(data) - sock.deleg.mode = fmReadWrite - elif bytesSent != data.len: - sock.sendBuffer.add(data[bytesSent .. ^1]) - sock.deleg.mode = fmReadWrite - -proc timeValFromMilliseconds(timeout = 500): Timeval = - if timeout != -1: - var seconds = timeout div 1000 - when defined(posix): - result.tv_sec = seconds.Time - result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds - else: - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(fd: var TFdSet, s: seq[Delegate], m: var int) = - FD_ZERO(fd) - for i in items(s): - m = max(m, int(i.fd)) - FD_SET(i.fd, fd) - -proc pruneSocketSet(s: var seq[Delegate], fd: var TFdSet) = - var i = 0 - var L = s.len - while i < L: - if FD_ISSET(s[i].fd, fd) != 0'i32: - s[i] = s[L-1] - dec(L) - else: - inc(i) - setLen(s, L) - -proc select(readfds, writefds, exceptfds: var seq[Delegate], - timeout = 500): int = - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr, ex: TFdSet - var m = 0 - createFdSet(rd, readfds, m) - createFdSet(wr, writefds, m) - createFdSet(ex, exceptfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - pruneSocketSet(exceptfds, (ex)) - -proc poll*(d: Dispatcher, timeout: int = 500): bool = - ## This function checks for events on all the delegates in the `PDispatcher`. - ## It then proceeds to call the correct event handler. - ## - ## This function returns ``True`` if there are file descriptors that are still - ## open, otherwise ``False``. File descriptors that have been - ## closed are immediately removed from the dispatcher automatically. - ## - ## **Note:** Each delegate has a task associated with it. This gets called - ## after each select() call, if you set timeout to ``-1`` the tasks will - ## only be executed after one or more file descriptors becomes readable or - ## writeable. - result = true - var readDg, writeDg, errorDg: seq[Delegate] = @[] - var len = d.delegates.len - var dc = 0 - - while dc < len: - let deleg = d.delegates[dc] - if (deleg.mode != fmWrite or deleg.mode != fmAppend) and deleg.open: - readDg.add(deleg) - if (deleg.mode != fmRead) and deleg.open: - writeDg.add(deleg) - if deleg.open: - errorDg.add(deleg) - inc dc - else: - # File/socket has been closed. Remove it from dispatcher. - d.delegates[dc] = d.delegates[len-1] - dec len - - d.delegates.setLen(len) - - var hasDataBufferedCount = 0 - for d in d.delegates: - if d.hasDataBuffered(d.deleVal): - hasDataBufferedCount.inc() - d.handleRead(d.deleVal) - if hasDataBufferedCount > 0: return true - - if readDg.len() == 0 and writeDg.len() == 0: - ## TODO: Perhaps this shouldn't return if errorDg has something? - return false - - if select(readDg, writeDg, errorDg, timeout) != 0: - for i in 0..len(d.delegates)-1: - if i > len(d.delegates)-1: break # One delegate might've been removed. - let deleg = d.delegates[i] - if not deleg.open: continue # This delegate might've been closed. - if (deleg.mode != fmWrite or deleg.mode != fmAppend) and - deleg notin readDg: - deleg.handleRead(deleg.deleVal) - if (deleg.mode != fmRead) and deleg notin writeDg: - deleg.handleWrite(deleg.deleVal) - if deleg notin errorDg: - deleg.handleError(deleg.deleVal) - - # Execute tasks - for i in items(d.delegates): - i.task(i.deleVal) - -proc len*(disp: Dispatcher): int = - ## Retrieves the amount of delegates in ``disp``. - return disp.delegates.len - -when not defined(testing) and isMainModule: - - proc testConnect(s: AsyncSocket, no: int) = - echo("Connected! " & $no) - - proc testRead(s: AsyncSocket, no: int) = - echo("Reading! " & $no) - var data = "" - if not s.readLine(data): return - if data == "": - echo("Closing connection. " & $no) - s.close() - echo(data) - echo("Finished reading! " & $no) - - proc testAccept(s: AsyncSocket, disp: Dispatcher, no: int) = - echo("Accepting client! " & $no) - var client: AsyncSocket - new(client) - var address = "" - s.acceptAddr(client, address) - echo("Accepted ", address) - client.handleRead = - proc (s: AsyncSocket) = - testRead(s, 2) - disp.register(client) - - proc main = - var d = newDispatcher() - - var s = asyncSocket() - s.connect("amber.tenthbit.net", Port(6667)) - s.handleConnect = - proc (s: AsyncSocket) = - testConnect(s, 1) - s.handleRead = - proc (s: AsyncSocket) = - testRead(s, 1) - d.register(s) - - var server = asyncSocket() - server.handleAccept = - proc (s: AsyncSocket) = - testAccept(s, d, 78) - server.bindAddr(Port(5555)) - server.listen() - d.register(server) - - while d.poll(-1): discard - main() diff --git a/lib/deprecated/pure/ftpclient.nim b/lib/deprecated/pure/ftpclient.nim deleted file mode 100644 index 206c21f27d..0000000000 --- a/lib/deprecated/pure/ftpclient.nim +++ /dev/null @@ -1,669 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import sockets, strutils, parseutils, times, os, asyncio - -from asyncnet import nil -from nativesockets import nil -from asyncdispatch import Future -## **Note**: This module is deprecated since version 0.11.3. -## You should use the async version of this module -## `asyncftpclient `_. -## -## ---- -## -## This module **partially** implements an FTP client as specified -## by `RFC 959 `_. -## -## This module provides both a synchronous and asynchronous implementation. -## The asynchronous implementation requires you to use the ``asyncFTPClient`` -## function. You are then required to register the ``AsyncFTPClient`` with a -## asyncio dispatcher using the ``register`` function. Take a look at the -## asyncio module documentation for more information. -## -## **Note**: The asynchronous implementation is only asynchronous for long -## file transfers, calls to functions which use the command socket will block. -## -## Here is some example usage of this module: -## -## .. code-block:: Nim -## var ftp = ftpClient("example.org", user = "user", pass = "pass") -## ftp.connect() -## ftp.retrFile("file.ext", "file.ext") -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. - -{.deprecated.} - -type - FtpBase*[SockType] = ref FtpBaseObj[SockType] - FtpBaseObj*[SockType] = object - csock*: SockType - dsock*: SockType - when SockType is asyncio.AsyncSocket: - handleEvent*: proc (ftp: AsyncFTPClient, ev: FTPEvent){.closure,gcsafe.} - disp: Dispatcher - asyncDSockID: Delegate - user*, pass*: string - address*: string - when SockType is asyncnet.AsyncSocket: - port*: nativesockets.Port - else: - port*: Port - - jobInProgress*: bool - job*: FTPJob[SockType] - - dsockConnected*: bool - - FTPJobType* = enum - JRetrText, JRetr, JStore - - FtpJob[T] = ref FtpJobObj[T] - FTPJobObj[T] = object - prc: proc (ftp: FTPBase[T], async: bool): bool {.nimcall, gcsafe.} - case typ*: FTPJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - - FtpClientObj* = FtpBaseObj[Socket] - FtpClient* = ref FtpClientObj - - AsyncFtpClient* = ref AsyncFtpClientObj ## Async alternative to TFTPClient. - AsyncFtpClientObj* = FtpBaseObj[asyncio.AsyncSocket] - - FTPEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FTPEvent* = object ## Event - filename*: string - case typ*: FTPEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FTPJobType ## The current job being performed. - - ReplyError* = object of IOError - FTPError* = object of IOError - - -const multiLineLimit = 10000 - -proc ftpClient*(address: string, port = Port(21), - user, pass = ""): FtpClient = - ## Create a ``FtpClient`` object. - new(result) - result.user = user - result.pass = pass - result.address = address - result.port = port - - result.dsockConnected = false - result.csock = socket() - if result.csock == invalidSocket: raiseOSError(osLastError()) - -template blockingOperation(sock: Socket, body: untyped) = - body - -template blockingOperation(sock: asyncio.AsyncSocket, body: untyped) = - sock.setBlocking(true) - body - sock.setBlocking(false) - -proc expectReply[T](ftp: FtpBase[T]): TaintedString = - result = TaintedString"" - blockingOperation(ftp.csock): - when T is Socket: - ftp.csock.readLine(result) - else: - discard ftp.csock.readLine(result) - var count = 0 - while result[3] == '-': - ## Multi-line reply. - var line = TaintedString"" - when T is Socket: - ftp.csock.readLine(line) - else: - discard ftp.csock.readLine(line) - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*[T](ftp: FtpBase[T], m: string): TaintedString = - ## Send a message to the server, and wait for a primary reply. - ## ``\c\L`` is added for you. - ## - ## **Note:** The server may return multiple lines of coded replies. - blockingOperation(ftp.csock): - ftp.csock.send(m & "\c\L") - return ftp.expectReply() - -proc assertReply(received: TaintedString, expected: string) = - if not received.string.startsWith(expected): - raise newException(ReplyError, - "Expected reply '$1' got: $2" % [ - expected, received.string]) - -proc assertReply(received: TaintedString, expected: varargs[string]) = - for i in items(expected): - if received.string.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received.string]) - -proc createJob[T](ftp: FtpBase[T], - prc: proc (ftp: FtpBase[T], async: bool): bool {. - nimcall,gcsafe.}, - cmd: FTPJobType) = - if ftp.jobInProgress: - raise newException(FTPError, "Unable to do two jobs at once.") - ftp.jobInProgress = true - new(ftp.job) - ftp.job.prc = prc - ftp.job.typ = cmd - case cmd - of JRetrText: - ftp.job.lines = "" - of JRetr, JStore: - ftp.job.toStore = "" - -proc deleteJob[T](ftp: FtpBase[T]) = - assert ftp.jobInProgress - ftp.jobInProgress = false - case ftp.job.typ - of JRetrText: - ftp.job.lines = "" - of JRetr, JStore: - ftp.job.file.close() - ftp.dsock.close() - -proc handleTask(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - if ftp.job.typ in {JRetr, JStore}: - if epochTime() - ftp.job.lastProgressReport >= 1.0: - var r: FTPEvent - ftp.job.lastProgressReport = epochTime() - r.typ = EvTransferProgress - r.bytesTotal = ftp.job.total - r.bytesFinished = ftp.job.progress - r.speed = ftp.job.oneSecond - r.filename = ftp.job.filename - r.currentJob = ftp.job.typ - ftp.job.oneSecond = 0 - ftp.handleEvent(ftp, r) - -proc handleWrite(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - if ftp.job.typ == JStore: - assert (not ftp.job.prc(ftp, true)) - -proc handleConnect(s: AsyncSocket, ftp: AsyncFTPClient) = - ftp.dsockConnected = true - assert(ftp.jobInProgress) - if ftp.job.typ == JStore: - s.setHandleWrite(proc (s: AsyncSocket) = handleWrite(s, ftp)) - else: - s.delHandleWrite() - -proc handleRead(s: AsyncSocket, ftp: AsyncFTPClient) = - assert ftp.jobInProgress - assert ftp.job.typ != JStore - # This can never return true, because it shouldn't check for code - # 226 from csock. - assert(not ftp.job.prc(ftp, true)) - -proc pasv[T](ftp: FtpBase[T]) = - ## Negotiate a data connection. - when T is Socket: - ftp.dsock = socket() - if ftp.dsock == invalidSocket: raiseOSError(osLastError()) - elif T is AsyncSocket: - ftp.dsock = asyncSocket() - ftp.dsock.handleRead = - proc (s: AsyncSocket) = - handleRead(s, ftp) - ftp.dsock.handleConnect = - proc (s: AsyncSocket) = - handleConnect(s, ftp) - ftp.dsock.handleTask = - proc (s: AsyncSocket) = - handleTask(s, ftp) - ftp.disp.register(ftp.dsock) - else: - {.fatal: "Incorrect socket instantiation".} - - var pasvMsg = ftp.send("PASV").string.strip.TaintedString - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg.string, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0.. ^3] - var port = nums[^2.. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - ftp.dsock.connect(ip.join("."), Port(properPort.toU16)) - when T is AsyncSocket: - ftp.dsockConnected = false - else: - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*[T](ftp: FtpBase[T]) = - ## Connect to the FTP server specified by ``ftp``. - when T is AsyncSocket: - blockingOperation(ftp.csock): - ftp.csock.connect(ftp.address, ftp.port) - elif T is Socket: - ftp.csock.connect(ftp.address, ftp.port) - else: - {.fatal: "Incorrect socket instantiation".} - - var reply = ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = ftp.expectReply() - - # Handle 220 messages from the server - assertReply ftp.expectReply(), "220" - - if ftp.user != "": - assertReply(ftp.send("USER " & ftp.user), "230", "331") - - if ftp.pass != "": - assertReply ftp.send("PASS " & ftp.pass), "230" - -proc pwd*[T](ftp: FtpBase[T]): string = - ## Returns the current working directory. - var wd = ftp.send("PWD") - assertReply wd, "257" - return wd.string.captureBetween('"') # " - -proc cd*[T](ftp: FtpBase[T], dir: string) = - ## Changes the current directory on the remote FTP server to ``dir``. - assertReply ftp.send("CWD " & dir.normalizePathSep), "250" - -proc cdup*[T](ftp: FtpBase[T]) = - ## Changes the current directory to the parent of the current directory. - assertReply ftp.send("CDUP"), "200" - -proc getLines[T](ftp: FtpBase[T], async: bool = false): bool = - ## Downloads text data in ASCII mode - ## Returns true if the download is complete. - ## It doesn't if `async` is true, because it doesn't check for 226 then. - if ftp.dsockConnected: - var r = TaintedString"" - when T is AsyncSocket: - if ftp.asyncDSock.readLine(r): - if r.string == "": - ftp.dsockConnected = false - else: - ftp.job.lines.add(r.string & "\n") - elif T is Socket: - assert(not async) - ftp.dsock.readLine(r) - if r.string == "": - ftp.dsockConnected = false - else: - ftp.job.lines.add(r.string & "\n") - else: - {.fatal: "Incorrect socket instantiation".} - - if not async: - var readSocks: seq[Socket] = @[ftp.csock] - # This is only needed here. Asyncio gets this socket... - blockingOperation(ftp.csock): - if readSocks.select(1) != 0 and ftp.csock in readSocks: - assertReply ftp.expectReply(), "226" - return true - -proc listDirs*[T](ftp: FtpBase[T], dir: string = "", - async = false): seq[string] = - ## Returns a list of filenames in the given directory. If ``dir`` is "", - ## the current directory is used. If ``async`` is true, this - ## function will return immediately and it will be your job to - ## use asyncio's ``poll`` to progress this operation. - - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - - assertReply ftp.send("NLST " & dir.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - result = splitLines(ftp.job.lines) - ftp.deleteJob() - else: return @[] - -proc fileExists*(ftp: FtpClient, file: string): bool {.deprecated.} = - ## **Deprecated since version 0.9.0:** Please use ``existsFile``. - ## - ## Determines whether ``file`` exists. - ## - ## Warning: This function may block. Especially on directories with many - ## files, because a full list of file names must be retrieved. - var files = ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc existsFile*(ftp: FtpClient, file: string): bool = - ## Determines whether ``file`` exists. - ## - ## Warning: This function may block. Especially on directories with many - ## files, because a full list of file names must be retrieved. - var files = ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*[T](ftp: FtpBase[T], dir: string, recursive: bool = false) = - ## Creates a directory ``dir``. If ``recursive`` is true, the topmost - ## subdirectory of ``dir`` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the ``dir`` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply ftp.send("MKD " & dir.normalizePathSep), "257" - else: - var reply = TaintedString"" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*[T](ftp: FtpBase[T], path: string, - permissions: set[FilePermission]) = - ## Changes permission of ``path`` to ``permissions``. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep), "200" - -proc list*[T](ftp: FtpBase[T], dir: string = "", async = false): string = - ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current - ## working directory. If ``async`` is true, this function will return - ## immediately and it will be your job to call asyncio's - ## ``poll`` to progress this operation. - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - - assertReply(ftp.send("LIST" & " " & dir.normalizePathSep), ["125", "150"]) - - if not async: - while not ftp.job.prc(ftp, false): discard - result = ftp.job.lines - ftp.deleteJob() - else: - return "" - -proc retrText*[T](ftp: FtpBase[T], file: string, async = false): string = - ## Retrieves ``file``. File must be ASCII text. - ## If ``async`` is true, this function will return immediately and - ## it will be your job to call asyncio's ``poll`` to progress this operation. - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - result = ftp.job.lines - ftp.deleteJob() - else: - return "" - -proc getFile[T](ftp: FtpBase[T], async = false): bool = - if ftp.dsockConnected: - var r = "".TaintedString - var bytesRead = 0 - var returned = false - if async: - when T is Socket: - raise newException(FTPError, "FTPClient must be async.") - else: - bytesRead = ftp.dsock.recvAsync(r, BufferSize) - returned = bytesRead != -1 - else: - bytesRead = ftp.dsock.recv(r, BufferSize) - returned = true - let r2 = r.string - if r2 != "": - ftp.job.progress.inc(r2.len) - ftp.job.oneSecond.inc(r2.len) - ftp.job.file.write(r2) - elif returned and r2 == "": - ftp.dsockConnected = false - - when T is Socket: - if not async: - var readSocks: seq[Socket] = @[ftp.csock] - blockingOperation(ftp.csock): - if readSocks.select(1) != 0 and ftp.csock in readSocks: - assertReply ftp.expectReply(), "226" - return true - -proc retrFile*[T](ftp: FtpBase[T], file, dest: string, async = false) = - ## Downloads ``file`` and saves it to ``dest``. Usage of this function - ## asynchronously is recommended to view the progress of the download. - ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function - ## when the download is finished, and the ``filename`` field will be equal - ## to ``file``. - ftp.createJob(getFile[T], JRetr) - ftp.job.file = open(dest, mode = fmWrite) - ftp.pasv() - var reply = ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply.string: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - ftp.job.total = fileSize - ftp.job.lastProgressReport = epochTime() - ftp.job.filename = file.normalizePathSep - - if not async: - while not ftp.job.prc(ftp, false): discard - ftp.deleteJob() - -proc doUpload[T](ftp: FtpBase[T], async = false): bool = - if ftp.dsockConnected: - if ftp.job.toStore.len() > 0: - assert(async) - let bytesSent = ftp.dsock.sendAsync(ftp.job.toStore) - if bytesSent == ftp.job.toStore.len: - ftp.job.toStore = "" - elif bytesSent != ftp.job.toStore.len and bytesSent != 0: - ftp.job.toStore = ftp.job.toStore[bytesSent .. ^1] - ftp.job.progress.inc(bytesSent) - ftp.job.oneSecond.inc(bytesSent) - else: - var s = newStringOfCap(4000) - var len = ftp.job.file.readBuffer(addr(s[0]), 4000) - setLen(s, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - if not async: - assertReply ftp.expectReply(), "226" - return true - return false - - if not async: - ftp.dsock.send(s) - else: - let bytesSent = ftp.dsock.sendAsync(s) - if bytesSent == 0: - ftp.job.toStore.add(s) - elif bytesSent != s.len: - ftp.job.toStore.add(s[bytesSent .. ^1]) - len = bytesSent - - ftp.job.progress.inc(len) - ftp.job.oneSecond.inc(len) - -proc store*[T](ftp: FtpBase[T], file, dest: string, async = false) = - ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The ``EvStore`` event is passed to the specified ``handleEvent`` function - ## when the upload is finished, and the ``filename`` field will be - ## equal to ``file``. - ftp.createJob(doUpload[T], JStore) - ftp.job.file = open(file) - ftp.job.total = ftp.job.file.getFileSize() - ftp.job.lastProgressReport = epochTime() - ftp.job.filename = file - ftp.pasv() - - assertReply ftp.send("STOR " & dest.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - ftp.deleteJob() - -proc close*[T](ftp: FtpBase[T]) = - ## Terminates the connection to the server. - assertReply ftp.send("QUIT"), "221" - if ftp.jobInProgress: ftp.deleteJob() - ftp.csock.close() - ftp.dsock.close() - -proc csockHandleRead(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - assertReply ftp.expectReply(), "226" # Make sure the transfer completed. - var r: FTPEvent - case ftp.job.typ - of JRetrText: - r.typ = EvLines - r.lines = ftp.job.lines - of JRetr: - r.typ = EvRetr - r.filename = ftp.job.filename - if ftp.job.progress != ftp.job.total: - raise newException(FTPError, "Didn't download full file.") - of JStore: - r.typ = EvStore - r.filename = ftp.job.filename - if ftp.job.progress != ftp.job.total: - raise newException(FTPError, "Didn't upload full file.") - ftp.deleteJob() - - ftp.handleEvent(ftp, r) - -proc asyncFTPClient*(address: string, port = Port(21), - user, pass = "", - handleEvent: proc (ftp: AsyncFTPClient, ev: FTPEvent) {.closure,gcsafe.} = - (proc (ftp: AsyncFTPClient, ev: FTPEvent) = discard)): AsyncFTPClient = - ## Create a ``AsyncFTPClient`` object. - ## - ## Use this if you want to use asyncio's dispatcher. - var dres: AsyncFtpClient - new(dres) - dres.user = user - dres.pass = pass - dres.address = address - dres.port = port - dres.dsockConnected = false - dres.handleEvent = handleEvent - dres.csock = asyncSocket() - dres.csock.handleRead = - proc (s: AsyncSocket) = - csockHandleRead(s, dres) - result = dres - -proc register*(d: Dispatcher, ftp: AsyncFTPClient): Delegate {.discardable.} = - ## Registers ``ftp`` with dispatcher ``d``. - ftp.disp = d - return ftp.disp.register(ftp.csock) - -when not defined(testing) and isMainModule: - proc main = - var d = newDispatcher() - let hev = - proc (ftp: AsyncFTPClient, event: FTPEvent) = - case event.typ - of EvStore: - echo("Upload finished!") - ftp.retrFile("payload.jpg", "payload2.jpg", async = true) - of EvTransferProgress: - var time: int64 = -1 - if event.speed != 0: - time = (event.bytesTotal - event.bytesFinished) div event.speed - echo(event.currentJob) - echo(event.speed div 1000, " kb/s. - ", - event.bytesFinished, "/", event.bytesTotal, - " - ", time, " seconds") - echo(d.len) - of EvRetr: - echo("Download finished!") - ftp.close() - echo d.len - else: assert(false) - var ftp = asyncFTPClient("example.com", user = "foo", pass = "bar", handleEvent = hev) - - d.register(ftp) - d.len.echo() - ftp.connect() - echo "connected" - ftp.store("payload.jpg", "payload.jpg", async = true) - d.len.echo() - echo "uploading..." - while true: - if not d.poll(): break - main() - -when not defined(testing) and isMainModule: - var ftp = ftpClient("example.com", user = "foo", pass = "bar") - ftp.connect() - echo ftp.pwd() - echo ftp.list() - echo("uploading") - ftp.store("payload.jpg", "payload.jpg", async = false) - - echo("Upload complete") - ftp.retrFile("payload.jpg", "payload2.jpg", async = false) - - echo("Download complete") - sleep(5000) - ftp.close() - sleep(200) diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim deleted file mode 100644 index cc1b6039b4..0000000000 --- a/lib/deprecated/pure/sockets.nim +++ /dev/null @@ -1,1748 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** Since version 0.10.2 this module is deprecated. -## Use the `net `_ or the -## `nativesockets `_ module instead. -## -## This module implements portable sockets, it supports a mix of different types -## of sockets. Sockets are buffered by default meaning that data will be -## received in ``BufferSize`` (4000) sized chunks, buffering -## behaviour can be disabled by setting the ``buffered`` parameter when calling -## the ``socket`` function to `false`. Be aware that some functions may not yet -## support buffered sockets (mainly the recvFrom function). -## -## Most procedures raise OSError on error, but some may return ``-1`` or a -## boolean ``false``. -## -## SSL is supported through the OpenSSL library. This support can be activated -## by compiling with the ``-d:ssl`` switch. When an SSL socket is used it will -## raise SslError exceptions when SSL errors occur. -## -## Asynchronous sockets are supported, however a better alternative is to use -## the `asyncio `_ module. - -{.deprecated.} - -include "system/inclrtl" - -{.deadCodeElim: on.} # dce option deprecated - -when hostOS == "solaris": - {.passl: "-lsocket -lnsl".} -elif hostOS == "haiku": - {.passl: "-lnetwork".} - -import os, parseutils -from times import epochTime - -when defined(ssl): - import openssl -else: - type SSLAcceptResult = int - -when defined(Windows): - import winlean -else: - import posix - -# Note: The enumerations are mapped to Window's constants. - -when defined(ssl): - - type - SSLError* = object of Exception - - SSLCVerifyMode* = enum - CVerifyNone, CVerifyPeer - - SSLProtVersion* = enum - protSSLv2, protSSLv3, protTLSv1, protSSLv23 - - SSLContext* = distinct SSLCTX - - SSLAcceptResult* = enum - AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess - - -const - BufferSize*: int = 4000 ## size of a buffered socket's buffer - -type - SocketImpl = object ## socket type - fd: SocketHandle - case isBuffered: bool # determines whether this socket is buffered. - of true: - buffer: array[0..BufferSize, char] - currPos: int # current index in buffer - bufLen: int # current length of buffer - of false: nil - when defined(ssl): - case isSsl: bool - of true: - sslHandle: SSLPtr - sslContext: SSLContext - sslNoHandshake: bool # True if needs handshake. - sslHasPeekChar: bool - sslPeekChar: char - of false: nil - nonblocking: bool - - Socket* = ref SocketImpl - - Port* = distinct uint16 ## port type - - Domain* = enum ## domain, which specifies the protocol family of the - ## created socket. Other domains than those that are listed - ## here are unsupported. - AF_UNIX, ## for local socket (using a file). Unsupported on Windows. - AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = 23 ## for network protocol IPv6. - - SockType* = enum ## second argument to `socket` proc - SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets - SOCK_DGRAM = 2, ## datagram service or Datagram Sockets - SOCK_RAW = 3, ## raw protocols atop the network layer. - SOCK_SEQPACKET = 5 ## reliable sequenced packet service - - Protocol* = enum ## third argument to `socket` proc - IPPROTO_TCP = 6, ## Transmission control protocol. - IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. - IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. - - Servent* = object ## information about a service - name*: string - aliases*: seq[string] - port*: Port - proto*: string - - Hostent* = object ## information about a given host - name*: string - aliases*: seq[string] - addrtype*: Domain - length*: int - addrList*: seq[string] - - SOBool* = enum ## Boolean socket options. - OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr - - RecvLineResult* = enum ## result for recvLineAsync - RecvFullLine, RecvPartialLine, RecvDisconnected, RecvFail - - ReadLineResult* = enum ## result for readLineAsync - ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone - - TimeoutError* = object of Exception - -when defined(booting): - let invalidSocket*: Socket = nil ## invalid socket -else: - const invalidSocket*: Socket = nil ## invalid socket - -when defined(windows): - let - osInvalidSocket = winlean.INVALID_SOCKET -else: - let - osInvalidSocket = posix.INVALID_SOCKET - -proc newTSocket(fd: SocketHandle, isBuff: bool): Socket = - if fd == osInvalidSocket: - return nil - new(result) - result.fd = fd - result.isBuffered = isBuff - if isBuff: - result.currPos = 0 - result.nonblocking = false - -proc `==`*(a, b: Port): bool {.borrow.} - ## ``==`` for ports. - -proc `$`*(p: Port): string {.borrow.} - ## returns the port number as a string - -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. - ## On machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 4-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) - -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -proc htonl*(x: int32): int32 = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. - result = sockets.ntohl(x) - -proc htons*(x: int16): int16 = - ## Converts 16-bit positive integers from host to network byte order. - ## On machines where the host byte order is the same as network byte - ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. - result = sockets.ntohs(x) - -template ntohl(x: uint32): uint32 = - cast[uint32](sockets.ntohl(cast[int32](x))) - -template ntohs(x: uint16): uint16 = - cast[uint16](sockets.ntohs(cast[int16](x))) - -template htonl(x: uint32): uint32 = - sockets.ntohl(x) - -template htons(x: uint16): uint16 = - sockets.ntohs(x) - -when defined(Posix): - proc toInt(domain: Domain): cint = - case domain - of AF_UNIX: result = posix.AF_UNIX - of AF_INET: result = posix.AF_INET - of AF_INET6: result = posix.AF_INET6 - - proc toInt(typ: SockType): cint = - case typ - of SOCK_STREAM: result = posix.SOCK_STREAM - of SOCK_DGRAM: result = posix.SOCK_DGRAM - of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET - of SOCK_RAW: result = posix.SOCK_RAW - - proc toInt(p: Protocol): cint = - case p - of IPPROTO_TCP: result = posix.IPPROTO_TCP - of IPPROTO_UDP: result = posix.IPPROTO_UDP - of IPPROTO_IP: result = posix.IPPROTO_IP - of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 - of IPPROTO_RAW: result = posix.IPPROTO_RAW - of IPPROTO_ICMP: result = posix.IPPROTO_ICMP - -else: - proc toInt(domain: Domain): cint = - result = toU16(ord(domain)) - - proc toInt(typ: SockType): cint = - result = cint(ord(typ)) - - proc toInt(p: Protocol): cint = - result = cint(ord(p)) - -proc socket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - - # TODO: Perhaps this should just raise OSError when an error occurs. - when defined(Windows): - result = newTSocket(winlean.socket(cint(domain), cint(typ), cint(protocol)), buffered) - else: - result = newTSocket(posix.socket(toInt(domain), toInt(typ), toInt(protocol)), buffered) - -when defined(ssl): - CRYPTO_malloc_init() - SslLibraryInit() - SslLoadErrorStrings() - ErrLoadBioStrings() - OpenSSL_add_all_algorithms() - - proc raiseSSLError(s = "") = - if s != "": - raise newException(SSLError, s) - let err = ErrPeekLastError() - if err == 0: - raise newException(SSLError, "No error reported.") - if err == -1: - raiseOSError(osLastError()) - var errStr = ErrErrorString(err, nil) - raise newException(SSLError, $errStr) - - # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html - proc loadCertificates(ctx: SSL_CTX, certFile, keyFile: string) = - if certFile != "" and not existsFile(certFile): - raise newException(system.IOError, "Certificate file could not be found: " & certFile) - if keyFile != "" and not existsFile(keyFile): - raise newException(system.IOError, "Key file could not be found: " & keyFile) - - if certFile != "": - var ret = SSLCTXUseCertificateChainFile(ctx, certFile) - if ret != 1: - raiseSslError() - - # TODO: Password? www.rtfm.com/openssl-examples/part1.pdf - if keyFile != "": - if SSL_CTX_use_PrivateKey_file(ctx, keyFile, - SSL_FILETYPE_PEM) != 1: - raiseSslError() - - if SSL_CTX_check_private_key(ctx) != 1: - raiseSslError("Verification of private key file failed.") - - proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = ""): SSLContext = - ## Creates an SSL context. - ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 are - ## are available with the addition of ``ProtSSLv23`` which allows for - ## compatibility with all of them. - ## - ## There are currently only two options for verify mode; - ## one is ``CVerifyNone`` and with it certificates will not be verified - ## the other is ``CVerifyPeer`` and certificates will be verified for - ## it, ``CVerifyPeer`` is the safest choice. - ## - ## The last two parameters specify the certificate file path and the key file - ## path, a server socket will most likely not work without these. - ## Certificates can be generated using the following command: - ## ``openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem``. - var newCTX: SSL_CTX - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") - of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) - - if newCTX.SSLCTXSetCipherList("ALL") != 1: - raiseSslError() - case verifyMode - of CVerifyPeer: - newCTX.SSLCTXSetVerify(SSLVerifyPeer, nil) - of CVerifyNone: - newCTX.SSLCTXSetVerify(SSLVerifyNone, nil) - if newCTX == nil: - raiseSslError() - - discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) - newCTX.loadCertificates(certFile, keyFile) - return SSLContext(newCTX) - - proc wrapSocket*(ctx: SSLContext, socket: Socket) = - ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. - ## - ## **Disclaimer**: This code is not well tested, may be very unsafe and - ## prone to security vulnerabilities. - - socket.isSSL = true - socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) - socket.sslNoHandshake = false - socket.sslHasPeekChar = false - if socket.sslHandle == nil: - raiseSslError() - - if SSLSetFd(socket.sslHandle, socket.fd) != 1: - raiseSslError() - -proc raiseSocketError*(socket: Socket, err: int = -1, async = false) = - ## Raises proper errors based on return values of ``recv`` functions. - ## - ## If ``async`` is ``True`` no error will be thrown in the case when the - ## error was caused by no data being available to be read. - ## - ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): - if socket.isSSL: - if err <= 0: - var ret = SSLGetError(socket.sslHandle, err.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - if async: - return - else: raiseSslError("Not enough data on socket.") - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - if async: - return - else: raiseSslError("Not enough data on socket.") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - - if err == -1 and not (when defined(ssl): socket.isSSL else: false): - let lastError = osLastError() - if async: - when defined(windows): - if lastError.int32 == WSAEWOULDBLOCK: - return - else: raiseOSError(lastError) - else: - if lastError.int32 == EAGAIN or lastError.int32 == EWOULDBLOCK: - return - else: raiseOSError(lastError) - else: raiseOSError(lastError) - -proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the - ## queue of pending connections. - if listen(socket.fd, cint(backlog)) < 0'i32: raiseOSError(osLastError()) - -proc invalidIp4(s: string) {.noreturn, noinline.} = - raise newException(ValueError, "invalid ip4 address: " & s) - -proc parseIp4*(s: string): BiggestInt = - ## parses an IP version 4 in dotted decimal form like "a.b.c.d". - ## - ## This is equivalent to `inet_ntoa`:idx:. - ## - ## Raises ValueError in case of an error. - var a, b, c, d: int - var i = 0 - var j = parseInt(s, a, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, b, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, c, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, d, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] != '\0': invalidIp4(s) - result = BiggestInt(a shl 24 or b shl 16 or c shl 8 or d) - -template gaiNim(a, p, h, list: untyped): untyped = - var gaiResult = getaddrinfo(a, $p, addr(h), list) - if gaiResult != 0'i32: - when defined(windows): - raiseOSError(osLastError()) - else: - raiseOSError(osLastError(), $gai_strerror(gaiResult)) - -proc bindAddr*(socket: Socket, port = Port(0), address = "") {. - tags: [ReadIOEffect].} = - ## binds an address/port number to a socket. - ## Use address string in dotted decimal form like "a.b.c.d" - ## or leave "" for any address. - - if address == "": - var name: Sockaddr_in - when defined(Windows): - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = uint16(posix.AF_INET) - name.sin_port = sockets.htons(uint16(port)) - name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) - if bindSocket(socket.fd, cast[ptr SockAddr](addr(name)), - sizeof(name).SockLen) < 0'i32: - raiseOSError(osLastError()) - else: - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(AF_INET) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - if bindSocket(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc getSockName*(socket: Socket): Port = - ## returns the socket's associated port number. - var name: Sockaddr_in - when defined(Windows): - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = uint16(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket.fd, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(sockets.ntohs(name.sin_port)) - -template acceptAddrPlain(noClientRet, successRet: SSLAcceptResult or int, - sslImplementation: untyped): untyped = - assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) - - if sock == osInvalidSocket: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEINPROGRESS: - client = invalidSocket - address = "" - when noClientRet.int == -1: - return - else: - return noClientRet - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - client = invalidSocket - address = "" - when noClientRet.int == -1: - return - else: - return noClientRet - else: raiseOSError(err) - else: - client.fd = sock - client.isBuffered = server.isBuffered - sslImplementation - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - when successRet.int == -1: - return - else: - return successRet - -proc acceptAddr*(server: Socket, client: var Socket, address: var string) {. - tags: [ReadIOEffect].} = - ## Blocks until a connection is being made from a client. When a connection - ## is made sets ``client`` to the client socket and ``address`` to the address - ## of the connecting client. - ## If ``server`` is non-blocking then this function returns immediately, and - ## if there are no connections queued the returned socket will be - ## ``InvalidSocket``. - ## This function will raise OSError if an error occurs. - ## - ## The resulting client will inherit any properties of the server socket. For - ## example: whether the socket is buffered or not. - ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - ## - ## **Warning:** When using SSL with non-blocking sockets, it is best to use - ## the acceptAddrSSL procedure as this procedure will most likely block. - acceptAddrPlain(SSLAcceptResult(-1), SSLAcceptResult(-1)): - when defined(ssl): - if server.isSSL: - # We must wrap the client sock in a ssl context. - - server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) - while ret <= 0: - let err = SSLGetError(client.sslHandle, ret) - if err != SSL_ERROR_WANT_ACCEPT: - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, - SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("acceptAddrSSL should be used for non-blocking SSL sockets.") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - -proc setBlocking*(s: Socket, blocking: bool) {.tags: [], gcsafe.} - ## Sets blocking mode on socket - -when defined(ssl): - proc acceptAddrSSL*(server: Socket, client: var Socket, - address: var string): SSLAcceptResult {. - tags: [ReadIOEffect].} = - ## This procedure should only be used for non-blocking **SSL** sockets. - ## It will immediately return with one of the following values: - ## - ## ``AcceptSuccess`` will be returned when a client has been successfully - ## accepted and the handshake has been successfully performed between - ## ``server`` and the newly connected client. - ## - ## ``AcceptNoHandshake`` will be returned when a client has been accepted - ## but no handshake could be performed. This can happen when the client - ## connects but does not yet initiate a handshake. In this case - ## ``acceptAddrSSL`` should be called again with the same parameters. - ## - ## ``AcceptNoClient`` will be returned when no client is currently attempting - ## to connect. - template doHandshake(): untyped = - when defined(ssl): - if server.isSSL: - client.setBlocking(false) - # We must wrap the client sock in a ssl context. - - if not client.isSSL or client.sslHandle == nil: - server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) - while ret <= 0: - let err = SSLGetError(client.sslHandle, ret) - if err != SSL_ERROR_WANT_ACCEPT: - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, - SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - client.sslNoHandshake = true - return AcceptNoHandshake - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - client.sslNoHandshake = false - - if client.isSSL and client.sslNoHandshake: - doHandshake() - return AcceptSuccess - else: - acceptAddrPlain(AcceptNoClient, AcceptSuccess): - doHandshake() - -proc accept*(server: Socket, client: var Socket) {.tags: [ReadIOEffect].} = - ## Equivalent to ``acceptAddr`` but doesn't return the address, only the - ## socket. - ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - - var addrDummy = "" - acceptAddr(server, client, addrDummy) - -proc acceptAddr*(server: Socket): tuple[client: Socket, address: string] {. - deprecated, tags: [ReadIOEffect].} = - ## Slightly different version of ``acceptAddr``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - var client: Socket - new(client) - var address = "" - acceptAddr(server, client, address) - return (client, address) - -proc accept*(server: Socket): Socket {.deprecated, tags: [ReadIOEffect].} = - ## **Deprecated since version 0.9.0:** Please use the function above. - new(result) - var address = "" - acceptAddr(server, result, address) - -proc close*(socket: Socket) = - ## closes a socket. - when defined(windows): - discard winlean.closesocket(socket.fd) - else: - discard posix.close(socket.fd) - # TODO: These values should not be discarded. An OSError should be raised. - # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times - when defined(ssl): - if socket.isSSL: - discard SSLShutdown(socket.sslHandle) - SSLFree(socket.sslHandle) - socket.sslHandle = nil - -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by ``name`` matches the s_name member - ## and the protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when defined(Windows): - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by ``port`` matches the s_port member and the - ## protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when defined(Windows): - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when defined(windows): - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(sockets.AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when defined(windows): - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype.cint == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype.cint == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when defined(Windows): - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when defined(windows): - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype.cint == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype.cint == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getSockOptInt*(socket: Socket, level, optname: int): int {. - tags: [ReadIOEffect].} = - ## getsockopt for integer options. - var res: cint - var size = sizeof(res).SockLen - if getsockopt(socket.fd, cint(level), cint(optname), - addr(res), addr(size)) < 0'i32: - raiseOSError(osLastError()) - result = int(res) - -proc setSockOptInt*(socket: Socket, level, optname, optval: int) {. - tags: [WriteIOEffect].} = - ## setsockopt for integer options. - var value = cint(optval) - if setsockopt(socket.fd, cint(level), cint(optname), addr(value), - sizeof(value).SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc toCInt(opt: SOBool): cint = - case opt - of OptAcceptConn: SO_ACCEPTCONN - of OptBroadcast: SO_BROADCAST - of OptDebug: SO_DEBUG - of OptDontRoute: SO_DONTROUTE - of OptKeepAlive: SO_KEEPALIVE - of OptOOBInline: SO_OOBINLINE - of OptReuseAddr: SO_REUSEADDR - -proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. - tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. - var res: cint - var size = sizeof(res).SockLen - if getsockopt(socket.fd, cint(level), toCInt(opt), - addr(res), addr(size)) < 0'i32: - raiseOSError(osLastError()) - result = res != 0 - -proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. - tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. - var valuei = cint(if value: 1 else: 0) - if setsockopt(socket.fd, cint(level), toCInt(opt), addr(valuei), - sizeof(valuei).SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc connect*(socket: Socket, address: string, port = Port(0), - af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## Connects socket to ``address``:``port``. ``Address`` can be an IP address or a - ## host name. If ``address`` is a host name, this function will try each IP - ## of that host name. ``htons`` is already performed on ``port`` so you must - ## not do it. - ## - ## If ``socket`` is an SSL socket a handshake will be automatically performed. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - if connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) == 0'i32: - success = true - break - else: lastError = osLastError() - it = it.ai_next - - freeaddrinfo(aiList) - if not success: raiseOSError(lastError) - - when defined(ssl): - if socket.isSSL: - let ret = SSLConnect(socket.sslHandle) - if ret <= 0: - let err = SSLGetError(socket.sslHandle, ret) - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_CONNECT, - SSL_ERROR_WANT_ACCEPT: - raiseSslError("The operation did not complete. Perhaps you should use connectAsync?") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - - when false: - var s: TSockAddrIn - s.sin_addr.s_addr = inet_addr(address) - s.sin_port = sockets.htons(uint16(port)) - when defined(windows): - s.sin_family = toU16(ord(af)) - else: - case af - of AF_UNIX: s.sin_family = posix.AF_UNIX - of AF_INET: s.sin_family = posix.AF_INET - of AF_INET6: s.sin_family = posix.AF_INET6 - if connect(socket.fd, cast[ptr TSockAddr](addr(s)), sizeof(s).cint) < 0'i32: - OSError() - -proc connectAsync*(socket: Socket, name: string, port = Port(0), - af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## A variant of ``connect`` for non-blocking sockets. - ## - ## This procedure will immediately return, it will not block until a connection - ## is made. It is up to the caller to make sure the connection has been established - ## by checking (using ``select``) whether the socket is writeable. - ## - ## **Note**: For SSL sockets, the ``handshake`` procedure must be called - ## whenever the socket successfully connects to a server. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(name, port, hints, aiList) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - var ret = connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) - if ret == 0'i32: - success = true - break - else: - lastError = osLastError() - when defined(windows): - # Windows EINTR doesn't behave same as POSIX. - if lastError.int32 == WSAEWOULDBLOCK: - success = true - break - else: - if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: - success = true - break - - it = it.ai_next - - freeaddrinfo(aiList) - if not success: raiseOSError(lastError) - when defined(ssl): - if socket.isSSL: - socket.sslNoHandshake = true - -when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = - ## This proc needs to be called on a socket after it connects. This is - ## only applicable when using ``connectAsync``. - ## This proc performs the SSL handshake. - ## - ## Returns ``False`` whenever the socket is not yet ready for a handshake, - ## ``True`` whenever handshake completed successfully. - ## - ## A SslError error is raised on any other errors. - result = true - if socket.isSSL: - var ret = SSLConnect(socket.sslHandle) - if ret <= 0: - var errret = SSLGetError(socket.sslHandle, ret) - case errret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT, - SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown Error") - socket.sslNoHandshake = false - else: - raiseSslError("Socket is not an SSL socket.") - - proc gotHandshake*(socket: Socket): bool = - ## Determines whether a handshake has occurred between a client (``socket``) - ## and the server that ``socket`` is connected to. - ## - ## Throws SslError if ``socket`` is not an SSL socket. - if socket.isSSL: - return not socket.sslNoHandshake - else: - raiseSslError("Socket is not an SSL socket.") - -proc timeValFromMilliseconds(timeout = 500): Timeval = - if timeout != -1: - var seconds = timeout div 1000 - when defined(posix): - result.tv_sec = seconds.Time - result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds - else: - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(fd: var TFdSet, s: seq[Socket], m: var int) = - FD_ZERO(fd) - for i in items(s): - m = max(m, int(i.fd)) - FD_SET(i.fd, fd) - -proc pruneSocketSet(s: var seq[Socket], fd: var TFdSet) = - var i = 0 - var L = s.len - while i < L: - if FD_ISSET(s[i].fd, fd) == 0'i32: - # not set. - s[i] = s[L-1] - dec(L) - else: - inc(i) - setLen(s, L) - -proc hasDataBuffered*(s: Socket): bool = - ## Determines whether a socket has data buffered. - result = false - if s.isBuffered: - result = s.bufLen > 0 and s.currPos != s.bufLen - - when defined(ssl): - if s.isSSL and not result: - result = s.sslHasPeekChar - -proc checkBuffer(readfds: var seq[Socket]): int = - ## Checks the buffer of each socket in ``readfds`` to see whether there is data. - ## Removes the sockets from ``readfds`` and returns the count of removed sockets. - var res: seq[Socket] = @[] - result = 0 - for s in readfds: - if hasDataBuffered(s): - inc(result) - res.add(s) - if result > 0: - readfds = res - -proc select*(readfds, writefds, exceptfds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## Traditional select function. This function will return the number of - ## sockets that are ready to be read from, written to, or which have errors. - ## If there are none; 0 is returned. - ## ``Timeout`` is in milliseconds and -1 can be specified for no timeout. - ## - ## Sockets which are **not** ready for reading, writing or which don't have - ## errors waiting on them are removed from the ``readfds``, ``writefds``, - ## ``exceptfds`` sequences respectively. - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr, ex: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - createFdSet((wr), writefds, m) - createFdSet((ex), exceptfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - pruneSocketSet(exceptfds, (ex)) - -proc select*(readfds, writefds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## Variant of select with only a read and write list. - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - createFdSet((wr), writefds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - -proc selectWrite*(writefds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## When a socket in ``writefds`` is ready to be written to then a non-zero - ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which **cannot** be written to will also be removed - ## from ``writefds``. - ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for - ## an unlimited time. - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var wr: TFdSet - var m = 0 - createFdSet((wr), writefds, m) - - if timeout != -1: - result = int(select(cint(m+1), nil, addr(wr), nil, addr(tv))) - else: - result = int(select(cint(m+1), nil, addr(wr), nil, nil)) - - pruneSocketSet(writefds, (wr)) - -proc select*(readfds: var seq[Socket], timeout = 500): int = - ## variant of select with a read list only - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), nil, nil, nil)) - - pruneSocketSet(readfds, (rd)) - -proc readIntoBuf(socket: Socket, flags: int32): int = - result = 0 - when defined(ssl): - if socket.isSSL: - result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - if result <= 0: - socket.bufLen = 0 - socket.currPos = 0 - return result - socket.bufLen = result - socket.currPos = 0 - -template retRead(flags, readBytes: int) {.dirty.} = - let res = socket.readIntoBuf(flags.int32) - if res <= 0: - if readBytes > 0: - return readBytes - else: - return res - -proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect].} = - ## Receives data from a socket. - ## - ## **Note**: This is a low-level function, you may be interested in the higher - ## level versions of this function which are also named ``recv``. - if size == 0: return - if socket.isBuffered: - if socket.bufLen == 0: - retRead(0'i32, 0) - - var read = 0 - while read < size: - if socket.currPos >= socket.bufLen: - retRead(0'i32, read) - - let chunk = min(socket.bufLen-socket.currPos, size-read) - var d = cast[cstring](data) - copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk) - read.inc(chunk) - socket.currPos.inc(chunk) - - result = read - else: - when defined(ssl): - if socket.isSSL: - if socket.sslHasPeekChar: - copyMem(data, addr(socket.sslPeekChar), 1) - socket.sslHasPeekChar = false - if size-1 > 0: - var d = cast[cstring](data) - result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 - else: - result = 1 - else: - result = SSLRead(socket.sslHandle, data, size) - else: - result = recv(socket.fd, data, size.cint, 0'i32) - else: - result = recv(socket.fd, data, size.cint, 0'i32) - -proc waitFor(socket: Socket, waited: var float, timeout, size: int, - funcName: string): int {.tags: [TimeEffect].} = - ## determines the amount of characters that can be read. Result will never - ## be larger than ``size``. For unbuffered sockets this will be ``1``. - ## For buffered sockets it can be as big as ``BufferSize``. - ## - ## If this function does not determine that there is data on the socket - ## within ``timeout`` ms, an ETimeout error will be raised. - result = 1 - if size <= 0: assert false - if timeout == -1: return size - if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos: - result = socket.bufLen - socket.currPos - result = min(result, size) - else: - if timeout - int(waited * 1000.0) < 1: - raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - - when defined(ssl): - if socket.isSSL: - if socket.hasDataBuffered: - # sslPeekChar is present. - return 1 - let sslPending = SSLPending(socket.sslHandle) - if sslPending != 0: - return sslPending - - var s = @[socket] - var startTime = epochTime() - let selRet = select(s, timeout - int(waited * 1000.0)) - if selRet < 0: raiseOSError(osLastError()) - if selRet != 1: - raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - waited += (epochTime() - startTime) - -proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. - tags: [ReadIOEffect, TimeEffect].} = - ## overload with a ``timeout`` parameter in milliseconds. - var waited = 0.0 # number of seconds already waited - - var read = 0 - while read < size: - let avail = waitFor(socket, waited, timeout, size-read, "recv") - var d = cast[cstring](data) - result = recv(socket, addr(d[read]), avail) - if result == 0: break - if result < 0: - return result - inc(read, result) - - result = read - -proc recv*(socket: Socket, data: var string, size: int, timeout = -1): int = - ## Higher-level version of ``recv``. - ## - ## When 0 is returned the socket's connection has been closed. - ## - ## This function will throw an OSError exception when an error occurs. A value - ## lower than 0 is never returned. - ## - ## A timeout may be specified in milliseconds, if enough data is not received - ## within the time specified an ETimeout exception will be raised. - ## - ## **Note**: ``data`` must be initialised. - data.setLen(size) - result = recv(socket, cstring(data), size, timeout) - if result < 0: - data.setLen(0) - socket.raiseSocketError(result) - data.setLen(result) - -proc recvAsync*(socket: Socket, data: var string, size: int): int = - ## Async version of ``recv``. - ## - ## When socket is non-blocking and no data is available on the socket, - ## ``-1`` will be returned and ``data`` will be ``""``. - ## - ## **Note**: ``data`` must be initialised. - data.setLen(size) - result = recv(socket, cstring(data), size) - if result < 0: - data.setLen(0) - socket.raiseSocketError(async = true) - result = -1 - data.setLen(result) - -proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = - if socket.isBuffered: - result = 1 - if socket.bufLen == 0 or socket.currPos > socket.bufLen-1: - var res = socket.readIntoBuf(0'i32) - if res <= 0: - result = res - - c = socket.buffer[socket.currPos] - else: - when defined(ssl): - if socket.isSSL: - if not socket.sslHasPeekChar: - result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) - socket.sslHasPeekChar = true - - c = socket.sslPeekChar - return - result = recv(socket.fd, addr(c), 1, MSG_PEEK) - -proc recvLine*(socket: Socket, line: var TaintedString, timeout = -1): bool {. - tags: [ReadIOEffect, TimeEffect], deprecated.} = - ## Receive a line of data from ``socket``. - ## - ## If a full line is received ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is received then ``line`` - ## will be set to it. - ## - ## ``True`` is returned if data is available. ``False`` suggests an - ## error, OSError exceptions are not raised and ``False`` is simply returned - ## instead. - ## - ## If the socket is disconnected, ``line`` will be set to ``""`` and ``True`` - ## will be returned. - ## - ## A timeout can be specified in milliseconds, if data is not received within - ## the specified time an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLine. - - template addNLIfEmpty(): untyped = - if line.len == 0: - line.add("\c\L") - - var waited = 0.0 - - setLen(line.string, 0) - while true: - var c: char - discard waitFor(socket, waited, timeout, 1, "recvLine") - var n = recv(socket, addr(c), 1) - if n < 0: return - elif n == 0: return true - if c == '\r': - discard waitFor(socket, waited, timeout, 1, "recvLine") - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: return false - addNLIfEmpty() - return true - elif c == '\L': - addNLIfEmpty() - return true - add(line.string, c) - -proc readLine*(socket: Socket, line: var TaintedString, timeout = -1) {. - tags: [ReadIOEffect, TimeEffect].} = - ## Reads a line of data from ``socket``. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## An OSError exception will be raised in the case of a socket error. - ## - ## A timeout can be specified in milliseconds, if data is not received within - ## the specified time an ETimeout exception will be raised. - - template addNLIfEmpty(): untyped = - if line.len == 0: - line.add("\c\L") - - var waited = 0.0 - - setLen(line.string, 0) - while true: - var c: char - discard waitFor(socket, waited, timeout, 1, "readLine") - var n = recv(socket, addr(c), 1) - if n < 0: socket.raiseSocketError() - elif n == 0: return - if c == '\r': - discard waitFor(socket, waited, timeout, 1, "readLine") - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: socket.raiseSocketError() - addNLIfEmpty() - return - elif c == '\L': - addNLIfEmpty() - return - add(line.string, c) - -proc recvLineAsync*(socket: Socket, - line: var TaintedString): RecvLineResult {.tags: [ReadIOEffect], deprecated.} = - ## Similar to ``recvLine`` but designed for non-blocking sockets. - ## - ## The values of the returned enum should be pretty self explanatory: - ## - ## * If a full line has been retrieved; ``RecvFullLine`` is returned. - ## * If some data has been retrieved; ``RecvPartialLine`` is returned. - ## * If the socket has been disconnected; ``RecvDisconnected`` is returned. - ## * If call to ``recv`` failed; ``RecvFail`` is returned. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLineAsync. - - setLen(line.string, 0) - while true: - var c: char - var n = recv(socket, addr(c), 1) - if n < 0: - return (if line.len == 0: RecvFail else: RecvPartialLine) - elif n == 0: - return (if line.len == 0: RecvDisconnected else: RecvPartialLine) - if c == '\r': - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: - return (if line.len == 0: RecvFail else: RecvPartialLine) - return RecvFullLine - elif c == '\L': return RecvFullLine - add(line.string, c) - -proc readLineAsync*(socket: Socket, - line: var TaintedString): ReadLineResult {.tags: [ReadIOEffect].} = - ## Similar to ``recvLine`` but designed for non-blocking sockets. - ## - ## The values of the returned enum should be pretty self explanatory: - ## - ## * If a full line has been retrieved; ``ReadFullLine`` is returned. - ## * If some data has been retrieved; ``ReadPartialLine`` is returned. - ## * If the socket has been disconnected; ``ReadDisconnected`` is returned. - ## * If no data could be retrieved; ``ReadNone`` is returned. - ## * If call to ``recv`` failed; **an OSError exception is raised.** - setLen(line.string, 0) - - template errorOrNone = - socket.raiseSocketError(async = true) - return ReadNone - - while true: - var c: char - var n = recv(socket, addr(c), 1) - #echo(n) - if n < 0: - if line.len == 0: errorOrNone else: return ReadPartialLine - elif n == 0: - return (if line.len == 0: ReadDisconnected else: ReadPartialLine) - if c == '\r': - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: - if line.len == 0: errorOrNone else: return ReadPartialLine - return ReadFullLine - elif c == '\L': return ReadFullLine - add(line.string, c) - -proc recv*(socket: Socket): TaintedString {.tags: [ReadIOEffect], deprecated.} = - ## receives all the available data from the socket. - ## Socket errors will result in an ``OSError`` error. - ## If socket is not a connectionless socket and socket is not connected - ## ``""`` will be returned. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 4000 - result = newStringOfCap(bufSize).TaintedString - var pos = 0 - while true: - var bytesRead = recv(socket, addr(string(result)[pos]), bufSize-1) - if bytesRead == -1: raiseOSError(osLastError()) - setLen(result.string, pos + bytesRead) - if bytesRead != bufSize-1: break - # increase capacity: - setLen(result.string, result.string.len + bufSize) - inc(pos, bytesRead) - when false: - var buf = newString(bufSize) - result = TaintedString"" - while true: - var bytesRead = recv(socket, cstring(buf), bufSize-1) - # Error - if bytesRead == -1: OSError(osLastError()) - - buf[bytesRead] = '\0' # might not be necessary - setLen(buf, bytesRead) - add(result.string, buf) - if bytesRead != bufSize-1: break - -{.push warning[deprecated]: off.} -proc recvTimeout*(socket: Socket, timeout: int): TaintedString {. - tags: [ReadIOEffect], deprecated.} = - ## overloaded variant to support a ``timeout`` parameter, the ``timeout`` - ## parameter specifies the amount of milliseconds to wait for data on the - ## socket. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - if socket.bufLen == 0: - var s = @[socket] - if s.select(timeout) != 1: - raise newException(TimeoutError, "Call to recv() timed out.") - - return socket.recv -{.pop.} - -proc recvAsync*(socket: Socket, s: var TaintedString): bool {. - tags: [ReadIOEffect], deprecated.} = - ## receives all the data from a non-blocking socket. If socket is non-blocking - ## and there are no messages available, `False` will be returned. - ## Other socket errors will result in an ``OSError`` error. - ## If socket is not a connectionless socket and socket is not connected - ## ``s`` will be set to ``""``. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 1000 - # ensure bufSize capacity: - setLen(s.string, bufSize) - setLen(s.string, 0) - var pos = 0 - while true: - var bytesRead = recv(socket, addr(string(s)[pos]), bufSize-1) - when defined(ssl): - if socket.isSSL: - if bytesRead <= 0: - var ret = SSLGetError(socket.sslHandle, bytesRead.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occurred.") # This should just not happen. - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - - if bytesRead == -1 and not (when defined(ssl): socket.isSSL else: false): - let err = osLastError() - when defined(windows): - if err.int32 == WSAEWOULDBLOCK: - return false - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return false - else: raiseOSError(err) - - setLen(s.string, pos + bytesRead) - if bytesRead != bufSize-1: break - # increase capacity: - setLen(s.string, s.string.len + bufSize) - inc(pos, bytesRead) - result = true - -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. - tags: [ReadIOEffect].} = - ## Receives data from ``socket``. This function should normally be used with - ## connection-less sockets (UDP sockets). - ## - ## If an error occurs the return value will be ``-1``. Otherwise the return - ## value will be the length of data received. - ## - ## **Warning:** This function does not yet have a buffered implementation, - ## so when ``socket`` is buffered the non-buffered implementation will be - ## used. Therefore if ``socket`` contains something in its buffer this - ## function will make no effort to return it. - - # TODO: Buffered sockets - data.setLen(length) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - - if result != -1: - data.setLen(result) - address = $inet_ntoa(sockAddress.sin_addr) - port = ntohs(sockAddress.sin_port).Port - -proc recvFromAsync*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, - flags = 0'i32): bool {.tags: [ReadIOEffect].} = - ## Variant of ``recvFrom`` for non-blocking sockets. Unlike ``recvFrom``, - ## this function will raise an OSError error whenever a socket error occurs. - ## - ## If there is no data to be read from the socket ``False`` will be returned. - result = true - var callRes = recvFrom(socket, data, length, address, port, flags) - if callRes < 0: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEWOULDBLOCK: - return false - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return false - else: raiseOSError(err) - -proc skip*(socket: Socket) {.tags: [ReadIOEffect], deprecated.} = - ## skips all the data that is pending for the socket - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 1000 - var buf = alloc(bufSize) - while recv(socket, buf, bufSize) == bufSize: discard - dealloc(buf) - -proc skip*(socket: Socket, size: int, timeout = -1) = - ## Skips ``size`` amount of bytes. - ## - ## An optional timeout can be specified in milliseconds, if skipping the - ## bytes takes longer than specified an ETimeout exception will be raised. - ## - ## Returns the number of skipped bytes. - var waited = 0.0 - var dummy = alloc(size) - var bytesSkipped = 0 - while bytesSkipped != size: - let avail = waitFor(socket, waited, timeout, size-bytesSkipped, "skip") - bytesSkipped += recv(socket, dummy, avail) - dealloc(dummy) - -proc send*(socket: Socket, data: pointer, size: int): int {. - tags: [WriteIOEffect].} = - ## sends data to a socket. - when defined(ssl): - if socket.isSSL: - return SSLWrite(socket.sslHandle, cast[cstring](data), size) - - when defined(windows) or defined(macosx): - result = send(socket.fd, data, size.cint, 0'i32) - else: - when defined(solaris): - const MSG_NOSIGNAL = 0 - result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) - -proc send*(socket: Socket, data: string) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - if socket.nonblocking: - raise newException(ValueError, "This function cannot be used on non-blocking sockets.") - let sent = send(socket, cstring(data), data.len) - if sent < 0: - when defined(ssl): - if socket.isSSL: - raiseSslError() - - raiseOSError(osLastError()) - - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") - -proc sendAsync*(socket: Socket, data: string): int {.tags: [WriteIOEffect].} = - ## sends data to a non-blocking socket. - ## Returns ``0`` if no data could be sent, if data has been sent - ## returns the amount of bytes of ``data`` that was successfully sent. This - ## number may not always be the length of ``data`` but typically is. - ## - ## An OSError (or SslError if socket is an SSL socket) exception is raised if an error - ## occurs. - result = send(socket, cstring(data), data.len) - when defined(ssl): - if socket.isSSL: - if result <= 0: - let ret = SSLGetError(socket.sslHandle, result.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occurred.") # This should just not happen. - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - return 0 - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - else: - return - if result == -1: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEINPROGRESS: - return 0 - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return 0 - else: raiseOSError(err) - - -proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} = - ## safe alternative to ``send``. Does not raise an OSError when an error occurs, - ## and instead returns ``false`` on failure. - result = send(socket, cstring(data), data.len) == data.len - -proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, - size: int, af: Domain = AF_INET, flags = 0'i32): int {. - tags: [WriteIOEffect].} = - ## low-level sendTo proc. This proc sends ``data`` to the specified ``address``, - ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. - ## - ## **Note:** This proc is not available for SSL sockets. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - - # try all possibilities: - var success = false - var it = aiList - while it != nil: - result = sendto(socket.fd, data, size.cint, flags.cint, it.ai_addr, - it.ai_addrlen.SockLen) - if result != -1'i32: - success = true - break - it = it.ai_next - - freeaddrinfo(aiList) - -proc sendTo*(socket: Socket, address: string, port: Port, - data: string): int {.tags: [WriteIOEffect].} = - ## Friendlier version of the low-level ``sendTo``. - result = socket.sendTo(address, port, cstring(data), data.len) - -when defined(Windows): - const - IOCPARM_MASK = 127 - IOC_IN = int(-2147483648) - FIONBIO = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or - (102 shl 8) or 126 - - proc ioctlsocket(s: SocketHandle, cmd: clong, - argptr: ptr clong): cint {. - stdcall, importc:"ioctlsocket", dynlib: "ws2_32.dll".} - -proc setBlocking(s: Socket, blocking: bool) = - when defined(Windows): - var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking - if ioctlsocket(s.fd, FIONBIO, addr(mode)) == -1: - raiseOSError(osLastError()) - else: # BSD sockets - var x: int = fcntl(s.fd, F_GETFL, 0) - if x == -1: - raiseOSError(osLastError()) - else: - var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK - if fcntl(s.fd, F_SETFL, mode) == -1: - raiseOSError(osLastError()) - s.nonblocking = not blocking - -discard """ proc setReuseAddr*(s: Socket) = - var blah: int = 1 - var mode = SO_REUSEADDR - if setsockopt(s.fd, SOL_SOCKET, mode, addr blah, TSOcklen(sizeof(int))) == -1: - raiseOSError(osLastError()) """ - -proc connect*(socket: Socket, address: string, port = Port(0), timeout: int, - af: Domain = AF_INET) {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Connects to server as specified by ``address`` on port specified by ``port``. - ## - ## The ``timeout`` paremeter specifies the time in milliseconds to allow for - ## the connection to the server to be made. - let originalStatus = not socket.nonblocking - socket.setBlocking(false) - - socket.connectAsync(address, port, af) - var s: seq[Socket] = @[socket] - if selectWrite(s, timeout) != 1: - raise newException(TimeoutError, "Call to 'connect' timed out.") - else: - when defined(ssl): - if socket.isSSL: - socket.setBlocking(true) - doAssert socket.handshake() - socket.setBlocking(originalStatus) - -proc isSSL*(socket: Socket): bool = return socket.isSSL - ## Determines whether ``socket`` is a SSL socket. - -proc getFD*(socket: Socket): SocketHandle = return socket.fd - ## Returns the socket's file descriptor - -proc isBlocking*(socket: Socket): bool = not socket.nonblocking - ## Determines whether ``socket`` is blocking. - -when defined(Windows): - var wsa: WSAData - if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) - - diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 36319a3179..5953ed9758 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1674,7 +1674,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curFd = fdPerDomain[ord(domain)] if curFd == osInvalidSocket.AsyncFD: try: - curFd = newAsyncNativeSocket(domain, sockType, protocol) + curFd = createAsyncNativeSocket(domain, sockType, protocol) except: freeAddrInfo(addrInfo) closeUnusedFds() @@ -1806,47 +1806,6 @@ proc readAll*(future: FutureStream[string]): Future[string] {.async.} = else: break -proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. - ## The partial line **will be lost**. - ## - ## **Warning**: This assumes that lines are delimited by ``\r\L``. - ## - ## **Note**: This procedure is mostly used for testing. You likely want to - ## use ``asyncnet.recvLine`` instead. - ## - ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead. - - template addNLIfEmpty(): typed = - if result.len == 0: - result.add("\c\L") - - result = "" - var c = "" - while true: - c = await recv(socket, 1) - if c.len == 0: - return "" - if c == "\r": - c = await recv(socket, 1) - assert c == "\l" - addNLIfEmpty() - return - elif c == "\L": - addNLIfEmpty() - return - add(result, c) - proc callSoon*(cbproc: proc ()) = ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index 3d6a9a015a..d28e9fb577 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -75,14 +75,54 @@ ## waitFor(main()) -import asyncdispatch, asyncnet, strutils, parseutils, os, times - -from ftpclient import FtpBaseObj, ReplyError, FtpEvent +import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times from net import BufferSize type - AsyncFtpClientObj* = FtpBaseObj[AsyncSocket] - AsyncFtpClient* = ref AsyncFtpClientObj + AsyncFtpClient* = ref object + csock*: AsyncSocket + dsock*: AsyncSocket + user*, pass*: string + address*: string + port*: Port + jobInProgress*: bool + job*: FTPJob + dsockConnected*: bool + + FTPJobType* = enum + JRetrText, JRetr, JStore + + FtpJob = ref object + prc: proc (ftp: AsyncFtpClient, async: bool): bool {.nimcall, gcsafe.} + case typ*: FTPJobType + of JRetrText: + lines: string + of JRetr, JStore: + file: File + filename: string + total: BiggestInt # In bytes. + progress: BiggestInt # In bytes. + oneSecond: BiggestInt # Bytes transferred in one second. + lastProgressReport: float # Time + toStore: string # Data left to upload (Only used with async) + + FTPEventType* = enum + EvTransferProgress, EvLines, EvRetr, EvStore + + FTPEvent* = object ## Event + filename*: string + case typ*: FTPEventType + of EvLines: + lines*: string ## Lines that have been transferred. + of EvRetr, EvStore: ## Retr/Store operation finished. + nil + of EvTransferProgress: + bytesTotal*: BiggestInt ## Bytes total. + bytesFinished*: BiggestInt ## Bytes transferred. + speed*: BiggestInt ## Speed in bytes/s + currentJob*: FTPJobType ## The current job being performed. + + ReplyError* = object of IOError ProgressChangedProc* = proc (total, progress: BiggestInt, speed: float): @@ -183,7 +223,7 @@ proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = ## Returns a list of filenames in the given directory. If ``dir`` is "", ## the current directory is used. If ``async`` is true, this ## function will return immediately and it will be your job to - ## use asyncio's ``poll`` to progress this operation. + ## use asyncdispatch's ``poll`` to progress this operation. await ftp.pasv() assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index fcb48a1f52..1e6cf3761a 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -1,7 +1,7 @@ discard """ output: "5000" """ -import asyncdispatch, nativesockets, net, strutils, os +import asyncdispatch, asyncnet, nativesockets, net, strutils, os var msgCount = 0 @@ -12,20 +12,22 @@ const var clientCount = 0 proc sendMessages(client: AsyncFD) {.async.} = - for i in 0 .. Date: Tue, 22 Jan 2019 22:38:52 +0100 Subject: [PATCH 191/350] Restore compatibility with old vcc versions [backport] (#10415) Local variables are declared before anything else. Fixes #10352 --- compiler/cgen.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e4f16f4ed3..190814b79e 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1313,9 +1313,19 @@ proc genInitCode(m: BModule) = appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", [m.nimTypesName, rope(m.nimTypes)]) - # Give this small function its own scope - addf(prc, "{$N", []) - block: + if m.initProc.gcFrameId > 0: + moduleInitRequired = true + add(prc, initGCFrame(m.initProc)) + + if m.initProc.s(cpsLocals).len > 0: + moduleInitRequired = true + add(prc, genSectionStart(cpsLocals, m.config)) + add(prc, m.initProc.s(cpsLocals)) + add(prc, genSectionEnd(cpsLocals, m.config)) + + if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: + # Give this small function its own scope + addf(prc, "{$N", []) # Keep a bogus frame in case the code needs one add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") @@ -1336,17 +1346,7 @@ proc genInitCode(m: BModule) = add(prc, genSectionStart(cpsStmts, m.config)) add(prc, m.preInitProc.s(cpsStmts)) add(prc, genSectionEnd(cpsStmts, m.config)) - addf(prc, "}$N", []) - - if m.initProc.gcFrameId > 0: - moduleInitRequired = true - add(prc, initGCFrame(m.initProc)) - - if m.initProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.initProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) + addf(prc, "}$N", []) if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: moduleInitRequired = true From a0c07ef863ffe31cb7f71472d22cefa713edc61a Mon Sep 17 00:00:00 2001 From: pgkos Date: Tue, 22 Jan 2019 22:39:53 +0100 Subject: [PATCH 192/350] Fix wrong integer types in odbcsql and db_odbc (#10419) * Fix wrong parameter type in SQLErr * Fix wrong types of integers passed to SQLGetData --- lib/impure/db_odbc.nim | 64 +++++++++++----------------------------- lib/wrappers/odbcsql.nim | 2 +- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index a533a28fff..b621d652d0 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -128,7 +128,7 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. cast[PSQLCHAR](sqlState.addr), cast[PSQLCHAR](nativeErr.addr), cast[PSQLCHAR](errMsg.addr), - 511.TSqlSmallInt, retSz.addr.PSQLINTEGER) + 511.TSqlSmallInt, retSz.addr) except: discard return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg)) @@ -277,14 +277,9 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, ## Rows are retrieved from the server at each iteration. var rowRes: Row - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, - # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. - # After every cycle and specified to cCnt. To ensure the traversal of all fields. + sz: TSqlInteger = 0 + cCnt: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 res = db.prepareFetch(query, args) if res == SQL_NO_DATA: discard @@ -292,15 +287,13 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, res = SQLNumResultCols(db.stmt, cCnt) rowRes = newRow(cCnt) rowRes.setLen(max(cCnt,0)) - tempcCnt = cCnt while res == SQL_SUCCESS: for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr.PSQLINTEGER)) + cast[cstring](buf.addr), 4095.TSqlSmallInt, + sz.addr)) rowRes[colId-1] = $(addr buf) - cCnt = tempcCnt yield rowRes res = SQLFetch(db.stmt) properFreeResult(SQL_HANDLE_STMT, db.stmt) @@ -313,14 +306,9 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, ## on demand using []. Returned handle is valid only within the interator body. var rowRes: Row = @[] - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, - # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. - # After every cycle and specified to cCnt. To ensure the traversal of all fields. + sz: TSqlInteger = 0 + cCnt: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 res = db.prepareFetch(query, args) if res == SQL_NO_DATA: discard @@ -328,15 +316,13 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, res = SQLNumResultCols(db.stmt, cCnt) rowRes = newRow(cCnt) rowRes.setLen(max(cCnt,0)) - tempcCnt = cCnt while res == SQL_SUCCESS: for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr.PSQLINTEGER)) + sz.addr)) rowRes[colId-1] = $(addr buf) - cCnt = tempcCnt yield (row: rowRes, len: cCnt.int) res = SQLFetch(db.stmt) properFreeResult(SQL_HANDLE_STMT, db.stmt) @@ -357,14 +343,9 @@ proc getRow*(db: var DbConn, query: SqlQuery, ## will return a Row with empty strings for each column. var rowRes: Row - sz: TSqlSmallInt = 0.TSqlSmallInt - cCnt: TSqlSmallInt = 0.TSqlSmallInt - res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, - ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. - ## After every cycle and specified to cCnt. To ensure the traversal of all fields. + sz: TSqlInteger = 0 + cCnt: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 res = db.prepareFetch(query, args) if res == SQL_NO_DATA: result = @[] @@ -372,14 +353,12 @@ proc getRow*(db: var DbConn, query: SqlQuery, res = SQLNumResultCols(db.stmt, cCnt) rowRes = newRow(cCnt) rowRes.setLen(max(cCnt,0)) - tempcCnt = cCnt for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr.PSQLINTEGER)) + sz.addr)) rowRes[colId-1] = $(addr buf) - cCnt = tempcCnt res = SQLFetch(db.stmt) result = rowRes properFreeResult(SQL_HANDLE_STMT, db.stmt) @@ -392,14 +371,9 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, var rows: seq[Row] = @[] rowRes: Row - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, - ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. - ## After every cycle and specified to cCnt. To ensure the traversal of all fields. + sz: TSqlInteger = 0 + cCnt: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 res = db.prepareFetch(query, args) if res == SQL_NO_DATA: result = @[] @@ -407,15 +381,13 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, res = SQLNumResultCols(db.stmt, cCnt) rowRes = newRow(cCnt) rowRes.setLen(max(cCnt,0)) - tempcCnt = cCnt while res == SQL_SUCCESS: for colId in 1..cCnt: buf[0] = '\0' db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr.PSQLINTEGER)) + sz.addr)) rowRes[colId-1] = $(addr buf) - cCnt = tempcCnt rows.add(rowRes) res = SQLFetch(db.stmt) result = rows diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim index 3b022290bb..6a27d8355f 100644 --- a/lib/wrappers/odbcsql.nim +++ b/lib/wrappers/odbcsql.nim @@ -830,7 +830,7 @@ proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, cbErrorMsgMax: TSqlSmallInt, - pcbErrorMsg: PSQLINTEGER): TSqlSmallInt {. + pcbErrorMsg: PSQLSMALLINT): TSqlSmallInt {. dynlib: odbclib, importc: "SQLError".} {.pop.} From 94f6a6b29447caa1814016c61f9dd30b82d83eab Mon Sep 17 00:00:00 2001 From: Ray Imber Date: Tue, 22 Jan 2019 17:05:26 -0800 Subject: [PATCH 193/350] Fix for issue #10342. better message for generic subclass instantiation (#10354) * Fix for issue #10342. better message for generic subclass instantiation errors. --- compiler/semtypinst.nim | 15 ++++++++++++++- compiler/types.nim | 9 +++++++-- tests/generics/tsubclassgenericerror.nim | 11 +++++++++++ tests/generics/twrong_generic_object.nim | 2 +- 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/generics/tsubclassgenericerror.nim diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index f3c12e5571..027ffd4aa8 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -490,7 +490,12 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = result.kind = tyUserTypeClassInst of tyGenericBody: - localError(cl.c.config, cl.info, "cannot instantiate: '" & typeToString(t) & "'") + localError( + cl.c.config, + cl.info, + "cannot instantiate: '" & + typeToString(t, preferDesc) & + "'; Maybe generic arguments are missing?") result = errorType(cl.c) #result = replaceTypeVarsT(cl, lastSon(t)) @@ -555,6 +560,14 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = for i in countup(0, sonsLen(result) - 1): if result.sons[i] != nil: + if result.sons[i].kind == tyGenericBody: + localError( + cl.c.config, + t.sym.info, + "cannot instantiate '" & + typeToString(result.sons[i], preferDesc) & + "' inside of type definition: '" & + t.owner.name.s & "'; Maybe generic arguments are missing?") var r = replaceTypeVarsT(cl, result.sons[i]) if result.kind == tyObject: # carefully coded to not skip the precious tyGenericInst: diff --git a/compiler/types.nim b/compiler/types.nim index 797336ddf7..1d6e71f143 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -456,12 +456,18 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = $t.n.intVal else: result = "int literal(" & $t.n.intVal & ")" - of tyGenericBody, tyGenericInst, tyGenericInvocation: + of tyGenericInst, tyGenericInvocation: result = typeToString(t.sons[0]) & '[' for i in countup(1, sonsLen(t)-1-ord(t.kind != tyGenericInvocation)): if i > 1: add(result, ", ") add(result, typeToString(t.sons[i], preferGenericArg)) add(result, ']') + of tyGenericBody: + result = typeToString(t.lastSon) & '[' + for i in countup(0, sonsLen(t)-2): + if i > 0: add(result, ", ") + add(result, typeToString(t.sons[i], preferTypeName)) + add(result, ']') of tyTypeDesc: if t.sons[0].kind == tyNone: result = "typedesc" else: result = "type " & typeToString(t.sons[0]) @@ -612,7 +618,6 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = typeToStr[t.kind] result.addTypeFlags(t) - proc firstOrd*(conf: ConfigRef; t: PType): BiggestInt = case t.kind of tyBool, tyChar, tySequence, tyOpenArray, tyString, tyVarargs, tyProxy: diff --git a/tests/generics/tsubclassgenericerror.nim b/tests/generics/tsubclassgenericerror.nim new file mode 100644 index 0000000000..87f8a8e64d --- /dev/null +++ b/tests/generics/tsubclassgenericerror.nim @@ -0,0 +1,11 @@ +discard """ + errormsg: "cannot instantiate 'GenericParentType[T]' inside of type definition: 'GenericChildType'; Maybe generic arguments are missing?" + line: 8 +""" + +type + GenericParentType[T] = ref object of RootObj + GenericChildType[T] = ref object of GenericParentType # missing the [T] + val: T + +var instance : GenericChildType[int] = nil diff --git a/tests/generics/twrong_generic_object.nim b/tests/generics/twrong_generic_object.nim index 00d90c55e1..442b89ea1e 100644 --- a/tests/generics/twrong_generic_object.nim +++ b/tests/generics/twrong_generic_object.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "cannot instantiate: 'GenericNodeObj'" + errormsg: "cannot instantiate: 'GenericNodeObj[T]'; Maybe generic arguments are missing?" line: 21 """ # bug #2509 From eee9729f536fecd94565e879f28edcb73bcf3861 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Wed, 23 Jan 2019 07:30:49 +0100 Subject: [PATCH 194/350] Fix semantic analysis with noReturn proc in tail pos (#10422) Fixes #10417 --- compiler/semstmts.nim | 3 ++- tests/exception/texceptions.nim | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 17f88d0393..cac24d33eb 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -279,7 +279,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode = for i in 1..last: var it = n.sons[i] let j = it.len-1 - it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) + if not endsInNoReturn(it.sons[j]): + it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info) result.typ = typ proc fitRemoveHiddenConv(c: PContext, typ: PType, n: PNode): PNode = diff --git a/tests/exception/texceptions.nim b/tests/exception/texceptions.nim index b30b3874b2..d63187b0ed 100644 --- a/tests/exception/texceptions.nim +++ b/tests/exception/texceptions.nim @@ -64,3 +64,13 @@ proc return_in_except = try: return_in_except() except: echo "RECOVER" +block: #10417 + proc moo() {.noreturn.} = discard + + let bar = + try: + 1 + except: + moo() + + doAssert(bar == 1) From 36e34d9aedd5d6f39dcc78a2239752c94227c542 Mon Sep 17 00:00:00 2001 From: Vindaar Date: Wed, 23 Jan 2019 08:44:19 +0100 Subject: [PATCH 195/350] close #3899 by adding test case (#10424) --- tests/array/tarray.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index f7c1dbf7f0..b40c8757c5 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -27,6 +27,7 @@ dflfdjkl__abcdefgasfsgdfgsgdfggsdfasdfsafewfkljdsfajs dflfdjkl__abcdefgasfsgdfgsgdfggsdfasdfsafewfkljdsfajsdf kgdchlfniambejop fjpmholcibdgeakn +2.0 ''' joinable: false """ @@ -538,3 +539,12 @@ block trelaxedindextyp: proc foo(x: seq[int]; idx: uint64) = echo x[idx] proc foo(x: string|cstring; idx: uint64) = echo x[idx] proc foo(x: openArray[int]; idx: uint64) = echo x[idx] + +block t3899: + # https://github.com/nim-lang/Nim/issues/3899 + type O = object + a: array[1..2,float] + template `[]`(x: O, i: int): float = + x.a[i] + const c = O(a: [1.0,2.0]) + echo c[2] From 0d480bfe22651edc9bdf7c5976668b47eb61b0a5 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Wed, 23 Jan 2019 09:16:14 +0100 Subject: [PATCH 196/350] Added basic bit manipulation procs to bitops (#10338) --- lib/pure/bitops.nim | 60 +++++++++++++++++++++++++++++++++++++++- tests/stdlib/tbitops.nim | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 3f213c5ea3..d258a42de0 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -8,7 +8,8 @@ # ## This module implements a series of low level methods for bit manipulation. -## By default, this module use compiler intrinsics to improve performance + +## By default, this module use compiler intrinsics where possible to improve performance ## on supported compilers: ``GCC``, ``LLVM_GCC``, ``CLANG``, ``VCC``, ``ICC``. ## ## The module will fallback to pure nim procs incase the backend is not supported. @@ -32,6 +33,63 @@ const useICC_builtins = defined(icc) and useBuiltins const useVCC_builtins = defined(vcc) and useBuiltins const arch64 = sizeof(int) == 8 +when defined(nimHasalignOf): + + import macros + + type BitsRange*[T] = range[0..sizeof(T)*8-1] + ## Returns a range with all bit positions for type ``T`` + + proc setMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = + ## Returns ``v``, with all the ``1`` bits from ``mask`` set to 1 + v = v or mask + + proc clearMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = + ## Returns ``v``, with all the ``1`` bits from ``mask`` set to 0 + v = v and not mask + + proc flipMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = + ## Returns ``v``, with all the ``1`` bits from ``mask`` flipped + v = v xor mask + + proc setBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = + ## Returns ``v``, with the bit at position ``bit`` set to 1 + v.setMask(1.T shl bit) + + proc clearBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = + ## Returns ``v``, with the bit at position ``bit`` set to 0 + v.clearMask(1.T shl bit) + + proc flipBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = + ## Returns ``v``, with the bit at position ``bit`` flipped + v.flipMask(1.T shl bit) + + macro setBits*(v: typed, bits: varargs[typed]): untyped = + ## Returns ``v``, with the bits at positions ``bits`` set to 1 + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("setBit", v, bit) + + macro clearBits*(v: typed, bits: varargs[typed]): untyped = + ## Returns ``v``, with the bits at positions ``bits`` set to 0 + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("clearBit", v, bit) + + macro flipBits*(v: typed, bits: varargs[typed]): untyped = + ## Returns ``v``, with the bits at positions ``bits`` set to 0 + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("flipBit", v, bit) + + proc testBit*[T: SomeInteger](v: var T, bit: BitsRange[T]): bool {.inline.} = + ## Returns true if the bit in ``v`` at positions ``bit`` is set to 1 + let mask = 1.T shl bit + return (v and mask) == mask + # #### Pure Nim version #### proc firstSetBit_nim(x: uint32): int {.inline, nosideeffect.} = diff --git a/tests/stdlib/tbitops.nim b/tests/stdlib/tbitops.nim index d8c6da1d42..b8b44703c4 100644 --- a/tests/stdlib/tbitops.nim +++ b/tests/stdlib/tbitops.nim @@ -162,6 +162,63 @@ proc main() = doAssert( U64A.rotateLeftBits(64) == U64A) doAssert( U64A.rotateRightBits(64) == U64A) + block: + # mask operations + var v: uint8 + v.setMask(0b1100_0000) + v.setMask(0b0000_1100) + doAssert(v == 0b1100_1100) + v.flipMask(0b0101_0101) + doAssert(v == 0b1001_1001) + v.clearMask(0b1000_1000) + doAssert(v == 0b0001_0001) + v.clearMask(0b0001_0001) + doAssert(v == 0b0000_0000) + block: + # single bit operations + var v: uint8 + v.setBit(0) + doAssert v == 0x0000_0001 + v.setBit(1) + doAssert v == 0b0000_0011 + v.flipBit(7) + doAssert v == 0b1000_0011 + v.clearBit(0) + doAssert v == 0b1000_0010 + v.flipBit(1) + doAssert v == 0b1000_0000 + doAssert v.testbit(7) + doAssert not v.testbit(6) + block: + # multi bit operations + var v: uint8 + v.setBits(0, 1, 7) + doAssert v == 0b1000_0011 + v.flipBits(2, 3) + doAssert v == 0b1000_1111 + v.clearBits(7, 0, 1) + doAssert v == 0b0000_1100 + block: + # signed + var v: int8 + v.setBit(7) + doAssert v == -128 + block: + var v: uint64 + v.setBit(63) + doAssert v == 0b1000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000'u64 + block: + # Test if RangeError is thrown if indexing out of range + try: + var v: uint32 + var i = 32 + v.setBit(i) + doAssert false + except RangeError: + discard + except: + doAssert false + echo "OK" main() From f1a841c605870711c4b046ee8f8c0a5bbf479bf2 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 23 Jan 2019 09:19:42 +0100 Subject: [PATCH 197/350] remove `tdont_be_stupid`, fixes #10386 --- tests/parallel/tdont_be_stupid.nim | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 tests/parallel/tdont_be_stupid.nim diff --git a/tests/parallel/tdont_be_stupid.nim b/tests/parallel/tdont_be_stupid.nim deleted file mode 100644 index d765c11a90..0000000000 --- a/tests/parallel/tdont_be_stupid.nim +++ /dev/null @@ -1,23 +0,0 @@ -discard """ -output: ''' -100 -200 -300 -400 -''' -""" - -import threadpool, os - -proc single(time: int) = - sleep time - echo time - -proc sleepsort(nums: openArray[int]) = - parallel: - var i = 0 - while i <= len(nums) + -1: - spawn single(nums[i]) - i += 1 - -sleepsort([400,100,300,200]) From e962be8981ff6ef09625b3cc89e0c0aa1f07b35a Mon Sep 17 00:00:00 2001 From: alaviss Date: Wed, 23 Jan 2019 15:24:21 +0700 Subject: [PATCH 198/350] compiler/sem*: improve lineinfo for qualified and generic procs (#10427) Previously the compiler will believe these are where `newSeq` symbol starts: newSeq[int]() ^ system.newSeq[int]() ^ This commit moves them back to: newSeq[int]() ^ system.newSeq[int]() ^ --- compiler/semcall.nim | 28 +++++++++++++++-------- compiler/semexprs.nim | 7 +++--- compiler/semtempl.nim | 13 ++++++----- nimsuggest/tests/tgeneric_highlight.nim | 18 +++++++++++++++ nimsuggest/tests/tqualified_highlight.nim | 9 ++++++++ 5 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 nimsuggest/tests/tgeneric_highlight.nim create mode 100644 nimsuggest/tests/tqualified_highlight.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 3723d3fc1c..0613a41452 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -462,16 +462,23 @@ proc updateDefaultParams(call: PNode) = if nfDefaultRefsParam in def.flags: call.flags.incl nfDefaultRefsParam call[i] = def +proc getCallLineInfo(n: PNode): TLineInfo = + case n.kind + of nkBracketExpr, nkCall, nkCommand: getCallLineInfo(n.sons[0]) + of nkDotExpr: getCallLineInfo(n.sons[1]) + else: n.info + proc semResolvedCall(c: PContext, x: TCandidate, n: PNode, flags: TExprFlags): PNode = assert x.state == csMatch var finalCallee = x.calleeSym - markUsed(c.config, n.sons[0].info, finalCallee, c.graph.usageSym) - onUse(n.sons[0].info, finalCallee) + let info = getCallLineInfo(n) + markUsed(c.config, info, finalCallee, c.graph.usageSym) + onUse(info, finalCallee) assert finalCallee.ast != nil if x.hasFauxMatch: result = x.call - result.sons[0] = newSymNode(finalCallee, result.sons[0].info) + result.sons[0] = newSymNode(finalCallee, getCallLineInfo(result.sons[0])) if containsGenericType(result.typ) or x.fauxMatch == tyUnknown: result.typ = newTypeS(x.fauxMatch, c) return @@ -496,7 +503,7 @@ proc semResolvedCall(c: PContext, x: TCandidate, result = x.call instGenericConvertersSons(c, result, x) - result[0] = newSymNode(finalCallee, result[0].info) + result[0] = newSymNode(finalCallee, getCallLineInfo(result[0])) result.typ = finalCallee.typ.sons[0] updateDefaultParams(result) @@ -551,7 +558,7 @@ proc semOverloadedCall(c: PContext, n, nOrig: PNode, notFoundError(c, n, errors) proc explicitGenericInstError(c: PContext; n: PNode): PNode = - localError(c.config, n.info, errCannotInstantiateX % renderTree(n)) + localError(c.config, getCallLineInfo(n), errCannotInstantiateX % renderTree(n)) result = n proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode = @@ -574,9 +581,10 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode = if tm in {isNone, isConvertible}: return nil var newInst = generateInstance(c, s, m.bindings, n.info) newInst.typ.flags.excl tfUnresolved - markUsed(c.config, n.info, s, c.graph.usageSym) - onUse(n.info, s) - result = newSymNode(newInst, n.info) + let info = getCallLineInfo(n) + markUsed(c.config, info, s, c.graph.usageSym) + onUse(info, s) + result = newSymNode(newInst, info) proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = assert n.kind == nkBracketExpr @@ -593,7 +601,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = # number of generic type parameters: if safeLen(s.ast.sons[genericParamsPos]) != n.len-1: let expected = safeLen(s.ast.sons[genericParamsPos]) - localError(c.config, n.info, errGenerated, "cannot instantiate: '" & renderTree(n) & + localError(c.config, getCallLineInfo(n), errGenerated, "cannot instantiate: '" & renderTree(n) & "'; got " & $(n.len-1) & " type(s) but expected " & $expected) return n result = explicitGenericSym(c, n, s) @@ -602,7 +610,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = # choose the generic proc with the proper number of type parameters. # XXX I think this could be improved by reusing sigmatch.paramTypesMatch. # It's good enough for now. - result = newNodeI(a.kind, n.info) + result = newNodeI(a.kind, getCallLineInfo(n)) for i in countup(0, len(a)-1): var candidate = a.sons[i].sym if candidate.kind in {skProc, skMethod, skConverter, diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 82f948492c..96fefa4b89 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1171,9 +1171,10 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = onUse(n.info, s) result = newSymNode(s, n.info) else: - markUsed(c.config, n.info, s, c.graph.usageSym) - onUse(n.info, s) - result = newSymNode(s, n.info) + let info = getCallLineInfo(n) + markUsed(c.config, info, s, c.graph.usageSym) + onUse(info, s) + result = newSymNode(s, info) proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = ## returns nil if it's not a built-in field access diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 14507cf9d9..d920a54b8a 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -58,25 +58,26 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode = inc(i) if i > 1: break a = nextOverloadIter(o, c, n) + let info = getCallLineInfo(n) if i <= 1 and r != scForceOpen: # XXX this makes more sense but breaks bootstrapping for now: # (s.kind notin routineKinds or s.magic != mNone): # for instance 'nextTry' is both in tables.nim and astalgo.nim ... - result = newSymNode(s, n.info) - markUsed(c.config, n.info, s, c.graph.usageSym) - onUse(n.info, s) + result = newSymNode(s, info) + markUsed(c.config, info, s, c.graph.usageSym) + onUse(info, s) else: # semantic checking requires a type; ``fitNode`` deals with it # appropriately let kind = if r == scClosed or n.kind == nkDotExpr: nkClosedSymChoice else: nkOpenSymChoice - result = newNodeIT(kind, n.info, newTypeS(tyNone, c)) + result = newNodeIT(kind, info, newTypeS(tyNone, c)) a = initOverloadIter(o, c, n) while a != nil: if a.kind != skModule: incl(a.flags, sfUsed) - addSon(result, newSymNode(a, n.info)) - onUse(n.info, a) + addSon(result, newSymNode(a, info)) + onUse(info, a) a = nextOverloadIter(o, c, n) proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode = diff --git a/nimsuggest/tests/tgeneric_highlight.nim b/nimsuggest/tests/tgeneric_highlight.nim new file mode 100644 index 0000000000..85bef7c43f --- /dev/null +++ b/nimsuggest/tests/tgeneric_highlight.nim @@ -0,0 +1,18 @@ +newSeq[int]() +system.newSeq[int]()#[!]# + +discard """ +disabled:true +$nimsuggest --tester $file +>highlight $1 +highlight;;skType;;1;;7;;3 +highlight;;skProc;;1;;0;;6 +highlight;;skProc;;1;;0;;6 +highlight;;skType;;1;;7;;3 +highlight;;skProc;;1;;0;;6 +highlight;;skType;;2;;14;;3 +highlight;;skProc;;2;;7;;6 +highlight;;skProc;;2;;7;;6 +highlight;;skType;;2;;14;;3 +highlight;;skProc;;2;;7;;6 +""" diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim new file mode 100644 index 0000000000..fe7c889d64 --- /dev/null +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -0,0 +1,9 @@ +system.echo#[!]# + +discard """ +disabled:true +$nimsuggest --tester $file +>highlight $1 +highlight;;skProc;;1;;7;;4 +highlight;;skProc;;1;;7;;4 +""" From 2fb8b1d1323ce2b683933e0a7d59044bf26caf99 Mon Sep 17 00:00:00 2001 From: alaviss Date: Wed, 23 Jan 2019 15:43:11 +0700 Subject: [PATCH 199/350] nimsuggest/tester: disable highlight tests for epc (#10426) The EPC backend of nimsuggest currently doesn't support nimsuggest highlighter, see nim-lang/nim-mode#140 --- nimsuggest/tester.nim | 4 ++++ nimsuggest/tests/taccent_highlight.nim | 1 - nimsuggest/tests/ttype_highlight.nim | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 63095d4901..1074419a58 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -240,6 +240,10 @@ proc skipDisabledTest(test: Test): bool = proc runEpcTest(filename: string): int = let s = parseTest(filename, true) if s.skipDisabledTest: return 0 + for req, _ in items(s.script): + if req.startsWith("highlight"): + echo "disabled epc: " & s.filename + return 0 for cmd in s.startup: if not runCmd(cmd, s.dest): quit "invalid command: " & cmd diff --git a/nimsuggest/tests/taccent_highlight.nim b/nimsuggest/tests/taccent_highlight.nim index 2f03d7c860..52ac2fc629 100644 --- a/nimsuggest/tests/taccent_highlight.nim +++ b/nimsuggest/tests/taccent_highlight.nim @@ -1,7 +1,6 @@ proc `$$$`#[!]# discard """ -disabled:true $nimsuggest --tester $file >highlight $1 highlight;;skProc;;1;;6;;3 diff --git a/nimsuggest/tests/ttype_highlight.nim b/nimsuggest/tests/ttype_highlight.nim index e4189a0158..a324215fe5 100644 --- a/nimsuggest/tests/ttype_highlight.nim +++ b/nimsuggest/tests/ttype_highlight.nim @@ -6,7 +6,6 @@ type TypeE* {.unchecked.} = array[0, int]#[!]# discard """ -disabled:true $nimsuggest --tester $file >highlight $1 highlight;;skType;;2;;2;;5 From 11022fea1b530aceced4c3f24d9295898d2eaf44 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 21 Jan 2019 14:35:49 +0100 Subject: [PATCH 200/350] control flow graphs: introduce 'join' points for easy analyses based on abstract interpretation --- compiler/destroyer.nim | 4 +- compiler/dfa.nim | 543 +++++++++++++++++++++++++++++------------ compiler/sempass2.nim | 2 +- 3 files changed, 395 insertions(+), 154 deletions(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 03ce9a5cf7..ca2aa8558b 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -184,7 +184,7 @@ proc isHarmlessVar*(s: PSym; c: Con): bool = inc usages #of useWithinCall: # if c.g[i].sym == s: return false - of goto, fork: + of goto, fork, InstrKind.join: discard "we do not perform an abstract interpretation yet" result = usages <= 1 @@ -246,6 +246,8 @@ proc isLastRead(n: PNode; c: var Con): bool = if not takenForks.containsOrIncl(pc): pcs.add pc + c.g[pc].dest inc pc + of InstrKind.join: + inc pc #echo c.graph.config $ n.info, " last read here!" return true diff --git a/compiler/dfa.nim b/compiler/dfa.nim index df9584576a..f349810006 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -34,12 +34,12 @@ import ast, astalgo, types, intsets, tables, msgs, options, lineinfos type InstrKind* = enum - goto, fork, def, use + goto, fork, join, def, use Instr* = object n*: PNode case kind*: InstrKind of def, use: sym*: PSym - of goto, fork: dest*: int + of goto, fork, join: dest*: int ControlFlowGraph* = seq[Instr] @@ -56,6 +56,7 @@ type inCall, inTryStmt: int blocks: seq[TBlock] tryStmtFixups: seq[TPosition] + forks: seq[TPosition] owner: PSym proc debugInfo(info: TLineInfo): string = @@ -67,7 +68,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) = var jumpTargets = initIntSet() let last = if last < 0: c.len-1 else: min(last, c.len-1) for i in start..last: - if c[i].kind in {goto, fork}: + if c[i].kind in {goto, fork, join}: jumpTargets.incl(i+c[i].dest) var i = start while i <= last: @@ -78,7 +79,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) = case c[i].kind of def, use: result.add c[i].sym.name.s - of goto, fork: + of goto, fork, join: result.add "L" result.add c[i].dest+i result.add("\t#") @@ -98,11 +99,166 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} = proc forkI(c: var Con; n: PNode): TPosition = result = TPosition(c.code.len) c.code.add Instr(n: n, kind: fork, dest: 0) + c.forks.add result proc gotoI(c: var Con; n: PNode): TPosition = result = TPosition(c.code.len) c.code.add Instr(n: n, kind: goto, dest: 0) +#[ + +Design of join +============== + +block: + if cond: break + def(x) + +use(x) + +Generates: + +L0: fork L1 + join L0 # patched. + goto Louter +L1: + def x + join L0 +Louter: + use x + + +block outer: + while a: + while b: + if foo: + if bar: + break outer # --> we need to 'join' every pushed 'fork' here + + +This works and then our abstract interpretation needs to deal with 'fork' +differently. It really causes a split in execution. Two threads are +"spawned" and both need to reach the 'join L' instruction. Afterwards +the abstract interpretations are joined and execution resumes single +threaded. + + +Abstract Interpretation +----------------------- + +proc interpret(pc, state, comesFrom): state = + result = state + # we need an explicit 'create' instruction (an explicit heap), in order + # to deal with 'var x = create(); var y = x; var z = y; destroy(z)' + while true: + case pc + of fork: + let a = interpret(pc+1, result, pc) + let b = interpret(forkTarget, result, pc) + result = a ++ b # ++ is a union operation + inc pc + of join: + if joinTarget == comesFrom: return result + else: inc pc + of use X: + if not result.contains(x): + error "variable not initialized " & x + inc pc + of def X: + if not result.contains(x): + result.incl X + else: + error "overwrite of variable causes memory leak " & x + inc pc + of destroy X: + result.excl X + +This is correct but still can lead to false positives: + +proc p(cond: bool) = + if cond: + new(x) + otherThings() + if cond: + destroy x + +Is not a leak. We should find a way to model *data* flow, not just +control flow. One solution is to rewrite the 'if' without a fork +instruction. The unstructured aspect can now be easily dealt with +the 'goto' and 'join' instructions. + +proc p(cond: bool) = + L0: fork Lend + new(x) + # do not 'join' here! + + Lend: + otherThings() + join L0 # SKIP THIS FOR new(x) SOMEHOW + destroy x + join L0 # but here. + + + +But if we follow 'goto Louter' we will never come to the join point. +We restore the bindings after popping pc from the stack then there +"no" problem?! + + +while cond: + prelude() + if not condB: break + postlude() + +---> +var setFlag = true +while cond and not setFlag: + prelude() + if not condB: + setFlag = true # BUT: Dependency + if not setFlag: # HERE + postlude() + +---> +var setFlag = true +while cond and not setFlag: + prelude() + if not condB: + postlude() + setFlag = true + + +------------------------------------------------- + +while cond: + prelude() + if more: + if not condB: break + stuffHere() + postlude() + +--> +var setFlag = true +while cond and not setFlag: + prelude() + if more: + if not condB: + setFlag = false + else: + stuffHere() + postlude() + else: + postlude() + +This is getting complicated. Instead we keep the whole 'join' idea but +duplicate the 'join' instructions on breaks and return exits! + +]# + +proc joinI(c: var Con; fromFork: TPosition; n: PNode) = + let dist = fromFork.int - c.code.len + c.code.add Instr(n: n, kind: join, dest: dist) + proc genLabel(c: Con): TPosition = result = TPosition(c.code.len) @@ -135,30 +291,97 @@ proc isTrue(n: PNode): bool = proc gen(c: var Con; n: PNode) # {.noSideEffect.} -proc genWhile(c: var Con; n: PNode) = - # L1: - # cond, tmp - # fork tmp, L2 - # body - # jmp L1 - # L2: - let L1 = c.genLabel - withBlock(nil): +when true: + proc genWhile(c: var Con; n: PNode) = + # We unroll every loop 3 times. We emulate 0, 1, 2 iterations + # through the loop. We need to prove this is correct for our + # purposes. But Herb Sutter claims it is. (Proof by authority.) + #[ + while cond: + body + + Becomes: + + if cond: + body + if cond: + body + if cond: + body + + We still need to ensure 'break' resolves properly, so an AST to AST + translation is impossible. + + So the code to generate is: + + cond + fork L4 # F1 + body + cond + fork L5 # F2 + body + cond + fork L6 # F3 + body + L6: + join F3 + L5: + join F2 + L4: + join F1 + ]# if isTrue(n.sons[0]): - c.gen(n.sons[1]) - c.jmpBack(n, L1) + # 'while true' is an idiom in Nim and so we produce + # better code for it: + for i in 0..2: + withBlock(nil): + c.gen(n.sons[1]) else: - c.gen(n.sons[0]) - let L2 = c.forkI(n) - c.gen(n.sons[1]) - c.jmpBack(n, L1) - c.patch(L2) + let oldForksLen = c.forks.len + var endings: array[3, TPosition] + for i in 0..2: + withBlock(nil): + c.gen(n.sons[0]) + endings[i] = c.forkI(n) + c.gen(n.sons[1]) + for i in countdown(endings.high, 0): + let endPos = endings[i] + c.patch(endPos) + c.joinI(c.forks.pop(), n) + doAssert(c.forks.len == oldForksLen) + +else: + + proc genWhile(c: var Con; n: PNode) = + # L1: + # cond, tmp + # fork tmp, L2 + # body + # jmp L1 + # L2: + let oldForksLen = c.forks.len + let L1 = c.genLabel + withBlock(nil): + if isTrue(n.sons[0]): + c.gen(n.sons[1]) + c.jmpBack(n, L1) + else: + c.gen(n.sons[0]) + let L2 = c.forkI(n) + c.gen(n.sons[1]) + c.jmpBack(n, L1) + c.patch(L2) + setLen(c.forks, oldForksLen) proc genBlock(c: var Con; n: PNode) = withBlock(n.sons[0].sym): c.gen(n.sons[1]) +proc genJoins(c: var Con; n: PNode) = + for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n) + proc genBreak(c: var Con; n: PNode) = + genJoins(c, n) let L1 = c.gotoI(n) if n.sons[0].kind == nkSym: #echo cast[int](n.sons[0].sym) @@ -170,28 +393,76 @@ proc genBreak(c: var Con; n: PNode) = else: c.blocks[c.blocks.high].fixups.add L1 +template forkT(n, body) = + let oldLen = c.forks.len + let L1 = c.forkI(n) + body + c.patch(L1) + c.joinI(L1, n) + setLen(c.forks, oldLen) + proc genIf(c: var Con, n: PNode) = + #[ + + if cond: + A + elif condB: + B + elif condC: + C + else: + D + + cond + fork L1 + A + goto Lend + L1: + condB + fork L2 + B + goto Lend2 + L2: + condC + fork L3 + C + goto Lend3 + L3: + D + goto Lend3 # not eliminated to simplify the join generation + Lend3: + join F3 + Lend2: + join F2 + Lend: + join F1 + + ]# + let oldLen = c.forks.len var endings: seq[TPosition] = @[] for i in countup(0, len(n) - 1): var it = n.sons[i] c.gen(it.sons[0]) if it.len == 2: - let elsePos = c.forkI(it.sons[1]) + let elsePos = forkI(c, it[1]) c.gen(it.sons[1]) - if i < sonsLen(n)-1: - endings.add(c.gotoI(it.sons[1])) + endings.add(c.gotoI(it.sons[1])) c.patch(elsePos) - for endPos in endings: c.patch(endPos) + for i in countdown(endings.high, 0): + let endPos = endings[i] + c.patch(endPos) + c.joinI(c.forks.pop(), n) + doAssert(c.forks.len == oldLen) proc genAndOr(c: var Con; n: PNode) = # asgn dest, a # fork L1 # asgn dest, b # L1: + # join F1 c.gen(n.sons[1]) - let L1 = c.forkI(n) - c.gen(n.sons[2]) - c.patch(L1) + forkT(n): + c.gen(n.sons[2]) proc genCase(c: var Con; n: PNode) = # if (!expr1) goto L1; @@ -204,56 +475,73 @@ proc genCase(c: var Con; n: PNode) = # L2: # elsePart # Lend: - when false: - # XXX Exhaustiveness is not yet mapped to the control flow graph as - # it seems to offer no benefits for the 'last read of' question. - let isExhaustive = skipTypes(n.sons[0].typ, - abstractVarRange-{tyTypeDesc}).kind in {tyFloat..tyFloat128, tyString} or - lastSon(n).kind == nkElse + let isExhaustive = skipTypes(n.sons[0].typ, + abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString} var endings: seq[TPosition] = @[] + let oldLen = c.forks.len c.gen(n.sons[0]) for i in 1 ..< n.len: let it = n.sons[i] if it.len == 1: c.gen(it.sons[0]) + elif i == n.len-1 and isExhaustive: + # treat the last branch as 'else' if this is an exhaustive case statement. + c.gen(it.lastSon) else: let elsePos = c.forkI(it.lastSon) c.gen(it.lastSon) - if i < sonsLen(n)-1: - endings.add(c.gotoI(it.lastSon)) + endings.add(c.gotoI(it.lastSon)) c.patch(elsePos) - for endPos in endings: c.patch(endPos) + for i in countdown(endings.high, 0): + let endPos = endings[i] + c.patch(endPos) + c.joinI(c.forks.pop(), n) + doAssert(c.forks.len == oldLen) proc genTry(c: var Con; n: PNode) = + let oldLen = c.forks.len var endings: seq[TPosition] = @[] inc c.inTryStmt - var newFixups: seq[TPosition] - swap(newFixups, c.tryStmtFixups) + let oldFixups = c.tryStmtFixups.len - let elsePos = c.forkI(n) + #let elsePos = c.forkI(n) c.gen(n.sons[0]) dec c.inTryStmt - for f in newFixups: + for i in oldFixups..c.tryStmtFixups.high: + let f = c.tryStmtFixups[i] c.patch(f) - swap(newFixups, c.tryStmtFixups) + # we also need to produce join instructions + # for the 'fork' that might preceed the goto instruction + if f.int-1 >= 0 and c.code[f.int-1].kind == fork: + c.joinI(TPosition(f.int-1), n) - c.patch(elsePos) + setLen(c.tryStmtFixups, oldFixups) + + #c.patch(elsePos) for i in 1 ..< n.len: let it = n.sons[i] if it.kind != nkFinally: var blen = len(it) let endExcept = c.forkI(it) c.gen(it.lastSon) - if i < sonsLen(n)-1: - endings.add(c.gotoI(it)) + endings.add(c.gotoI(it)) c.patch(endExcept) - for endPos in endings: c.patch(endPos) + for i in countdown(endings.high, 0): + let endPos = endings[i] + c.patch(endPos) + c.joinI(c.forks.pop(), n) + + # join the 'elsePos' forkI instruction: + #c.joinI(c.forks.pop(), n) + let fin = lastSon(n) if fin.kind == nkFinally: c.gen(fin.sons[0]) + doAssert(c.forks.len == oldLen) proc genRaise(c: var Con; n: PNode) = + genJoins(c, n) gen(c, n.sons[0]) if c.inTryStmt > 0: c.tryStmtFixups.add c.gotoI(n) @@ -265,6 +553,7 @@ proc genImplicitReturn(c: var Con) = gen(c, c.owner.ast.sons[resultPos]) proc genReturn(c: var Con; n: PNode) = + genJoins(c, n) if n.sons[0].kind != nkEmpty: gen(c, n.sons[0]) else: @@ -287,6 +576,14 @@ proc genDef(c: var Con; n: PNode) = if n.kind == nkSym and n.sym.kind in InterestingSyms: c.code.add Instr(n: n, kind: def, sym: n.sym) +proc canRaise(fn: PNode): bool = + const magicsThatCanRaise = { + mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst} + if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise: + result = false + else: + result = true + proc genCall(c: var Con; n: PNode) = gen(c, n[0]) var t = n[0].typ @@ -297,8 +594,16 @@ proc genCall(c: var Con; n: PNode) = if t != nil and i < t.len and t.sons[i].kind == tyVar: genDef(c, n[i]) # every call can potentially raise: - if c.inTryStmt > 0: - c.tryStmtFixups.add c.forkI(n) + if c.inTryStmt > 0 and canRaise(n[0]): + # we generate the instruction sequence: + # fork L1 + # goto exceptionHandler (except or finally) + # L1: + # join F1 + let endGoto = c.forkI(n) + c.tryStmtFixups.add c.gotoI(n) + c.patch(endGoto) + c.joinI(c.forks.pop(), n) dec c.inCall proc genMagic(c: var Con; n: PNode; m: TMagic) = @@ -368,114 +673,48 @@ proc gen(c: var Con; n: PNode) = doAssert false, "dfa construction pass requires the elimination of 'defer'" else: discard -proc dfa(code: seq[Instr]; conf: ConfigRef) = - var u = newSeq[IntSet](code.len) # usages - var d = newSeq[IntSet](code.len) # defs - var c = newSeq[IntSet](code.len) # consumed - var backrefs = initTable[int, int]() - for i in 0.. 0 and maxIters > 0: # and someChange: - dec maxIters - var pc = w.pop() # w[^1] - var prevPc = -1 - # this simulates a single linear control flow execution: - while pc < code.len: - if prevPc >= 0: - someChange = false - # merge step and test for changes (we compute the fixpoints here): - # 'u' needs to be the union of prevPc, pc - # 'd' needs to be the intersection of 'pc' - for id in u[prevPc]: - if not u[pc].containsOrIncl(id): - someChange = true - # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection - # is only interesting on merge points: - for id in d[prevPc]: - if not d[pc].containsOrIncl(id): - someChange = true - # if this is a merge point, we take the intersection of the 'd' sets: - if backrefs.hasKey(pc): - var intersect = initIntSet() - assign(intersect, d[pc]) - var first = true - for prevPc in backrefs.allValues(pc): - for def in d[pc]: - if def notin d[prevPc]: - excl(intersect, def) - someChange = true - when defined(debugDfa): - echo "Excluding ", pc, " prev ", prevPc - assign d[pc], intersect - if consuming >= 0: - if not c[pc].containsOrIncl(consuming): - someChange = true - consuming = -1 - - # our interpretation ![I!]: - prevPc = pc - case code[pc].kind - of goto: - # we must leave endless loops eventually: - if not takenGotos.containsOrIncl(pc) or someChange: - pc = pc + code[pc].dest - else: - inc pc - of fork: - # we follow the next instruction but push the dest onto our "work" stack: - #if someChange: - w.add pc + code[pc].dest - inc pc - of use: - #if not d[prevPc].missingOrExcl(): - # someChange = true - consuming = code[pc].sym.id - inc pc - of def: - if not d[pc].containsOrIncl(code[pc].sym.id): - someChange = true - inc pc - - when defined(useDfa) and defined(debugDfa): - for i in 0.. Date: Mon, 21 Jan 2019 19:46:06 +0100 Subject: [PATCH 201/350] make tests green again --- compiler/destroyer.nim | 129 +++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 49 deletions(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index ca2aa8558b..1881730e88 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -188,6 +188,35 @@ proc isHarmlessVar*(s: PSym; c: Con): bool = discard "we do not perform an abstract interpretation yet" result = usages <= 1 +proc isLastRead(s: PSym; c: var Con; pc, comesFrom: int): int = + var pc = pc + while pc < c.g.len: + case c.g[pc].kind + of def: + if c.g[pc].sym == s: + # the path lead to a redefinition of 's' --> abandon it. + return pc + inc pc + of use: + if c.g[pc].sym == s: + c.otherRead = c.g[pc].n + return -1 + inc pc + of goto: + pc = pc + c.g[pc].dest + of fork: + # every branch must lead to the last read of the location: + let variantA = isLastRead(s, c, pc+1, pc) + if variantA < 0: return -1 + let variantB = isLastRead(s, c, pc + c.g[pc].dest, pc) + if variantB < 0: return -1 + pc = variantA+1 + of InstrKind.join: + let dest = pc + c.g[pc].dest + if dest == comesFrom: return pc + inc pc + return pc + proc isLastRead(n: PNode; c: var Con): bool = # first we need to search for the instruction that belongs to 'n': doAssert n.kind == nkSym @@ -195,61 +224,63 @@ proc isLastRead(n: PNode; c: var Con): bool = var instr = -1 for i in 0..= c.g.len: return true - let s = n.sym - var pcs: seq[int] = @[instr+1] - var takenGotos: IntSet - var takenForks = initIntSet() - while pcs.len > 0: - var pc = pcs.pop + when false: + result = isLastRead(n.sym, c, 0, -1) >= 0 + else: + let s = n.sym + var pcs: seq[int] = @[instr+1] + var takenGotos: IntSet + var takenForks = initIntSet() + while pcs.len > 0: + var pc = pcs.pop - takenGotos = initIntSet() - while pc < c.g.len: - case c.g[pc].kind - of def: - if c.g[pc].sym == s: - # the path lead to a redefinition of 's' --> abandon it. - when false: - # Too complex thinking ahead: In reality it is enough to find - # the 'def x' here on the current path to make the 'use x' valid. - # but for this the definition needs to dominate the usage: - var dominates = true - for j in pc+1 .. instr: - # not within the same basic block? - if c.g[j].kind in {goto, fork} and (j + c.g[j].dest) in (pc+1 .. instr): - #if j in c.jumpTargets: - dominates = false - if dominates: break - break - inc pc - of use: - if c.g[pc].sym == s: - c.otherRead = c.g[pc].n - return false - inc pc - of goto: - # we must leave endless loops eventually: - if not takenGotos.containsOrIncl(pc): - pc = pc + c.g[pc].dest - else: + takenGotos = initIntSet() + while pc < c.g.len: + case c.g[pc].kind + of def: + if c.g[pc].sym == s: + # the path lead to a redefinition of 's' --> abandon it. + when false: + # Too complex thinking ahead: In reality it is enough to find + # the 'def x' here on the current path to make the 'use x' valid. + # but for this the definition needs to dominate the usage: + var dominates = true + for j in pc+1 .. instr: + # not within the same basic block? + if c.g[j].kind in {goto, fork} and (j + c.g[j].dest) in (pc+1 .. instr): + #if j in c.jumpTargets: + dominates = false + if dominates: break + break inc pc - of fork: - # we follow the next instruction but push the dest onto our "work" stack: - if not takenForks.containsOrIncl(pc): - pcs.add pc + c.g[pc].dest - inc pc - of InstrKind.join: - inc pc - #echo c.graph.config $ n.info, " last read here!" - return true + of use: + if c.g[pc].sym == s: + c.otherRead = c.g[pc].n + return false + inc pc + of goto: + # we must leave endless loops eventually: + if not takenGotos.containsOrIncl(pc): + pc = pc + c.g[pc].dest + else: + inc pc + of fork: + # we follow the next instruction but push the dest onto our "work" stack: + if not takenForks.containsOrIncl(pc): + pcs.add pc + c.g[pc].dest + inc pc + of InstrKind.join: + inc pc + #echo c.graph.config $ n.info, " last read here!" + return true template interestingSym(s: PSym): bool = s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ) From 112f7e650aad3200b123174d8d835f989e707fcb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 21 Jan 2019 19:56:58 +0100 Subject: [PATCH 202/350] destroyer.nim: code cleanup --- compiler/destroyer.nim | 67 ------------------------------------------ 1 file changed, 67 deletions(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 1881730e88..a97b49dacf 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -132,62 +132,6 @@ type emptyNode: PNode otherRead: PNode - -proc isHarmlessVar*(s: PSym; c: Con): bool = - # 's' is harmless if it used only once and its - # definition/usage are not split by any labels: - # - # let s = foo() - # while true: - # a[i] = s - # - # produces: - # - # def s - # L1: - # use s - # goto L1 - # - # let s = foo() - # if cond: - # a[i] = s - # else: - # a[j] = s - # - # produces: - # - # def s - # fork L2 - # use s - # goto L3 - # L2: - # use s - # L3 - # - # So this analysis is for now overly conservative, but correct. - var defsite = -1 - var usages = 0 - for i in 0.. 1: return false - inc usages - #of useWithinCall: - # if c.g[i].sym == s: return false - of goto, fork, InstrKind.join: - discard "we do not perform an abstract interpretation yet" - result = usages <= 1 - proc isLastRead(s: PSym; c: var Con; pc, comesFrom: int): int = var pc = pc while pc < c.g.len: @@ -248,17 +192,6 @@ proc isLastRead(n: PNode; c: var Con): bool = of def: if c.g[pc].sym == s: # the path lead to a redefinition of 's' --> abandon it. - when false: - # Too complex thinking ahead: In reality it is enough to find - # the 'def x' here on the current path to make the 'use x' valid. - # but for this the definition needs to dominate the usage: - var dominates = true - for j in pc+1 .. instr: - # not within the same basic block? - if c.g[j].kind in {goto, fork} and (j + c.g[j].dest) in (pc+1 .. instr): - #if j in c.jumpTargets: - dominates = false - if dominates: break break inc pc of use: From 2655f242a838a5845a71bc4857cbc4a7df297eb6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 21 Jan 2019 20:11:42 +0100 Subject: [PATCH 203/350] use the lastRead analysis; it's correct by construction, not a mess of special cases and seems to be just as fast as the old algorithm --- compiler/destroyer.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index a97b49dacf..06d4dcbefe 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -176,8 +176,8 @@ proc isLastRead(n: PNode; c: var Con): bool = # we go through all paths beginning from 'instr+1' and need to # ensure that we don't find another 'use X' instruction. if instr+1 >= c.g.len: return true - when false: - result = isLastRead(n.sym, c, 0, -1) >= 0 + when true: + result = isLastRead(n.sym, c, instr+1, -1) >= 0 else: let s = n.sym var pcs: seq[int] = @[instr+1] From 7fcf51248bb5c37cf49765bac020fa7bc86dea34 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Mon, 21 Jan 2019 22:56:40 +0100 Subject: [PATCH 204/350] Added --docInternal option to allow 'nim doc' to include non-exported symbols --- compiler/commands.nim | 2 ++ compiler/docgen.nim | 5 ++++- compiler/options.nim | 1 + doc/advopt.txt | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 30521f9ca8..af775f5cd2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -746,6 +746,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; else: conf.cppCustomNamespace = "Nim" defineSymbol(conf.symbols, "cppCompileToNamespace", conf.cppCustomNamespace) + of "docinternal": + processOnOffSwitchG(conf, {optDocInternal}, arg, pass, info) else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 33cd98f38f..4271a116ef 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -464,7 +464,10 @@ proc isVisible(d: PDoc; n: PNode): bool = # we cannot generate code for forwarded symbols here as we have no # exception tracking information here. Instead we copy over the comment # from the proc header. - result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported} + if optDocInternal in d.conf.globalOptions: + result = {sfFromGeneric, sfForward}*n.sym.flags == {} + else: + result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported} if result and containsOrIncl(d.emitted, n.sym.id): result = false elif n.kind == nkPragmaExpr: diff --git a/compiler/options.nim b/compiler/options.nim index d39b0a2682..54276f99d9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -77,6 +77,7 @@ type # please make sure we have under 32 options optExcessiveStackTrace # fully qualified module filenames optShowAllMismatches # show all overloading resolution candidates optWholeProject # for 'doc2': output any dependency + optDocInternal # generate documentation for non-exported symbols optMixedMode # true if some module triggered C++ codegen optListFullPaths # use full paths in toMsgFilename, toFilename optNoNimblePath diff --git a/doc/advopt.txt b/doc/advopt.txt index f713839d85..58ed9edd01 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -58,6 +58,7 @@ Advanced options: --project document the whole project (doc2) --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) + --docInternal also generate documentation for non-exported symbols --lineDir:on|off generation of #line directive on|off --embedsrc:on|off embeds the original source code as comments in the generated output From 9c0e5c4c074ac9d4db7b215ba202ad9e09f18254 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Wed, 23 Jan 2019 11:10:51 +0100 Subject: [PATCH 205/350] Harmonize the var/let and const handling (#10410) Fixes #10333 --- compiler/semstmts.nim | 18 ++++++++++++++---- tests/vm/tvmmisc.nim | 11 +++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index cac24d33eb..5e9d5d9c53 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -440,11 +440,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if a.kind notin {nkIdentDefs, nkVarTuple, nkConstDef}: illFormedAst(a, c.config) checkMinSonsLen(a, 3, c.config) var length = sonsLen(a) - var typ: PType + + var typ: PType = nil if a.sons[length-2].kind != nkEmpty: typ = semTypeNode(c, a.sons[length-2], nil) - else: - typ = nil + var def: PNode = c.graph.emptyNode if a.sons[length-1].kind != nkEmpty: def = semExprWithType(c, a.sons[length-1], {efAllowDestructor}) @@ -582,9 +582,19 @@ proc semConst(c: PContext, n: PNode): PNode = if def == nil: localError(c.config, a.sons[length-1].info, errConstExprExpected) continue + + if def.typ.kind == tyTypeDesc and c.p.owner.kind != skMacro: + # prevent the all too common 'const x = int' bug: + localError(c.config, def.info, "'typedesc' metatype is not valid here; typed '=' instead of ':'?") + def.typ = errorType(c) + # check type compatibility between def.typ and typ: if typ != nil: - def = fitRemoveHiddenConv(c, typ, def) + if typ.isMetaType: + def = inferWithMetatype(c, typ, def) + typ = def.typ + else: + def = fitRemoveHiddenConv(c, typ, def) else: typ = def.typ if typ == nil: diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index bd3aa2fcd7..78871d1036 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -149,3 +149,14 @@ static: static: doAssert foo().i == 1 + +# #10333 +block: + const + encoding: auto = [ + ["", "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX"], + ["", "X", "XX", "XXX", "XL", "L", "LX", "LXX", "LXXX", "XC"], + ["", "C", "CC", "CCC", "CD", "D", "DC", "DCC", "DCCC", "CM"], + ["", "M", "MM", "MMM", "--", "-", "--", "---", "----", "--"], + ] + doAssert encoding.len == 4 From c3229dc1c3af3c1766bba79d013e0f90b02d19d5 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 23 Jan 2019 10:12:46 +0000 Subject: [PATCH 206/350] Add packaging guide #5182 (#10384) --- doc/packaging.rst | 68 ++++++++++++++++++++++++++++++++++++++++++++++ readme.md | 6 ++++ tools/kochdocs.nim | 1 + 3 files changed, 75 insertions(+) create mode 100644 doc/packaging.rst diff --git a/doc/packaging.rst b/doc/packaging.rst new file mode 100644 index 0000000000..f67cb1ccee --- /dev/null +++ b/doc/packaging.rst @@ -0,0 +1,68 @@ +Packaging Nim +============= + + +Supported architectures +----------------------- + +Nim runs on a wide variety of platforms. Support on amd64 and i386 is tested regularly, while less popular platforms are tested by the community. + +- amd64 +- arm64 (aka aarch64) +- armel +- armhf +- i386 +- m68k +- mips64el +- mipsel +- powerpc +- ppc64 +- ppc64el (aka ppc64le) +- riscv64 + +The following platforms are seldomly tested: + +- alpha +- hppa +- ia64 +- mips +- s390x +- sparc64 + +Packaging for Linux +------------------- + +See https://github.com/nim-lang/Nim/labels/Installation for installation-related bugs. + +Build Nim from the released tarball at https://nim-lang.org/install_unix.html +It is different from the GitHub sources as it contains Nimble, C sources & other tools. + +The Debian package ships bash and ksh completion and manpages that can be reused. + +Hints on the build process: + +:: + + # build from C sources and then using koch + ./build.sh --os $os_type --cpu $cpu_arch + ./bin/nim c koch + ./koch boot -d:release + + # optionally generate docs into doc/html + ./koch docs + + ./koch tools -d:release + + # extract files to be really installed + ./install.sh + + # also include the tools + for fn in nimble nimsuggest nimgrep; do cp ./bin/$fn /nim/bin/; done + +What to install: + +- The expected stdlib location is /usr/lib/nim +- Global configuration files under /etc/nim +- Optionally: manpages, documentation, shell completion +- When installing documentation, .idx files are not required +- The "compiler" directory contains compiler sources and should not be part of the compiler binary package diff --git a/readme.md b/readme.md index 114086eb69..5480047a48 100644 --- a/readme.md +++ b/readme.md @@ -5,6 +5,7 @@ For more information about Nim, including downloads and documentation for the latest release, check out [Nim's website][nim-site] or [bleeding edge docs](https://nim-lang.github.io/Nim/). ## Community + [![Join the IRC chat][badge-nim-irc]][nim-irc] [![Join the Gitter chat][badge-nim-gitter]][nim-gitter] [![Get help][badge-nim-forum-gethelp]][nim-forum] @@ -23,6 +24,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( * [Github Wiki][nim-wiki] - Misc user-contributed content. ## Compiling + The compiler currently officially supports the following platform and architecture combinations: @@ -56,6 +58,8 @@ Nim from source using ``gcc``, ``git`` and the ``koch`` build tool. For most users, installing the latest stable version is enough. Check out the installation instructions on the website to do so: https://nim-lang.org/install.html. +For package mantainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). + ``` # step 1: git clone https://github.com/nim-lang/Nim.git @@ -83,6 +87,7 @@ Finally, once you have finished the build steps (on Windows, Mac or Linux) you should add the ``bin`` directory to your PATH. ## Koch + ``koch`` is the build tool used to build various parts of Nim and to generate documentation and the website, among other things. The ``koch`` tool can also be used to run the Nim test suite. @@ -106,6 +111,7 @@ This project exists thanks to all the people who contribute. ## Contributing + [![Backers on Open Collective](https://opencollective.com/nim/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/nim/sponsors/badge.svg)](#sponsors) [![Setup a bounty via Bountysource][badge-nim-bountysource]][nim-bountysource] [![Donate Bitcoins][badge-nim-bitcoin]][nim-bitcoin] diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index b85998cf8a..7919deec68 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -106,6 +106,7 @@ doc/nep1.rst doc/nims.rst doc/contributing.rst doc/codeowners.rst +doc/packaging.rst doc/manual/var_t_return.rst """.splitWhitespace() From efa4b9adaa6babb46b463e785ccc06e3494bd342 Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Wed, 23 Jan 2019 11:13:44 +0100 Subject: [PATCH 207/350] Fixes #10263; [backport] --- compiler/msgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 0dd5820b4a..698b810615 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -432,7 +432,7 @@ proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) = proc sourceLine*(conf: ConfigRef; i: TLineInfo): string = if i.fileIndex.int32 < 0: return "" - if not optPreserveOrigSource(conf) and conf.m.fileInfos[i.fileIndex.int32].lines.len == 0: + if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0: try: for line in lines(toFullPath(conf, i)): addSourceLine conf, i.fileIndex, line.string From d9ee377517fe22e307592f4d6c019388600b5e57 Mon Sep 17 00:00:00 2001 From: Vindaar Date: Wed, 23 Jan 2019 11:17:32 +0100 Subject: [PATCH 208/350] fix #10339 by returning type attached to nkEmpty (#10370) * fix #10339 by checking for nkObjConstr * revert check for nkObjConstr, return type from nkEmpty node The correct type needed in `semObjConstr` to fix #10339 is indeed available, but attached to an `nkEmpty` node. These were previously discarded in `semTypeNode`, which is used to extract the type for the object. * simplify return of PType from `nkEmpty` * also fixes #9866, add test case --- compiler/semtypes.nim | 2 +- tests/macros/tquotedo.nim | 15 +++++++++++++++ tests/statictypes/tstatictypes.nim | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index c28902b1f2..e4f7dc1479 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1474,7 +1474,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = if c.config.cmd == cmdIdeTools: suggestExpr(c, n) case n.kind - of nkEmpty: discard + of nkEmpty: result = n.typ of nkTypeOfExpr: # for ``type(countup(1,3))``, see ``tests/ttoseq``. checkSonsLen(n, 1, c.config) diff --git a/tests/macros/tquotedo.nim b/tests/macros/tquotedo.nim index 0aae87bf04..6acb8ef4eb 100644 --- a/tests/macros/tquotedo.nim +++ b/tests/macros/tquotedo.nim @@ -4,6 +4,7 @@ output: ''' Hallo Welt Hallo Welt 1 +() ''' """ @@ -34,3 +35,17 @@ macro t(): untyped = t() echo tp() + + +# https://github.com/nim-lang/Nim/issues/9866 +type + # Foo = int # works + Foo = object # fails + +macro dispatchGen(): untyped = + var shOpt: Foo + result = quote do: + let baz = `shOpt` + echo `shOpt` + +dispatchGen() diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index b7cde61247..2a4ab0c637 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -137,3 +137,20 @@ block: type Coord[N: static[int]] = tuple[col, row: range[0'i8 .. (N.int8-1)]] Point[N: static[int]] = range[0'i16 .. N.int16 * N.int16 - 1] + +# https://github.com/nim-lang/Nim/issues/10339 +block: + type + MicroKernel = object + a: float + b: int + + macro extractA(ukernel: static MicroKernel): untyped = + result = newLit ukernel.a + + proc tFunc[ukernel: static MicroKernel]() = + const x = ukernel.extractA + doAssert x == 5.5 + + const uk = MicroKernel(a: 5.5, b: 1) + tFunc[uk]() From c60bb94647871702c2c853fb40d4fb1363dc8247 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 23 Jan 2019 13:30:32 +0100 Subject: [PATCH 209/350] update gitignore [ci skip] --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2bf0bf30c3..38b7976120 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ xcuserdata/ /run.json # for `nim doc foo.nim` /*.html +lib/**/*.html #/testresults.html #covered by /*.html /testresults.json From bad5ad6dc764e2b1505d27a61b949079040a5f3c Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Wed, 23 Jan 2019 23:02:26 +0900 Subject: [PATCH 210/350] Fix issue #10358 (#10383) * Fix bug: vccexe finds oldest VCC * echo path to vcvarsall.bat as hint when failed to execute cl.exe --- tools/vccexe/vccenv.nim | 2 +- tools/vccexe/vccexe.nim | 25 +++++++++++++++++++------ tools/vccexe/vcvarsall.nim | 6 ++---- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/tools/vccexe/vccenv.nim b/tools/vccexe/vccenv.nim index bd3b0b30a0..12a6e6b3d8 100644 --- a/tools/vccexe/vccenv.nim +++ b/tools/vccexe/vccenv.nim @@ -39,7 +39,7 @@ proc vccEnvVcVarsAllPath*(version: VccEnvVersion = vsUndefined): string = for tryVersion in [vs140, vs120, vs110, vs100, vs90]: let tryPath = vccEnvVcVarsAllPath(tryVersion) if tryPath.len > 0: - result = tryPath + return tryPath else: # Specific version requested let key = $version let val = getEnv key diff --git a/tools/vccexe/vccexe.nim b/tools/vccexe/vccexe.nim index f794885f24..28d9e4173c 100644 --- a/tools/vccexe/vccexe.nim +++ b/tools/vccexe/vccexe.nim @@ -46,6 +46,8 @@ const sdktypeSepIdx = sdktypePrefix.len sdkversionSepIdx = sdkversionPrefix.len + vcvarsallDefaultPath = "vcvarsall.bat" + helpText = """ +-----------------------------------------------------------------+ | Microsoft C/C++ compiler wrapper for Nim | @@ -158,6 +160,10 @@ when isMainModule: vccversionValue = vccUndefined vcvarsallArg = discoverVccVcVarsAllPath() + if vcvarsallArg == "": + # Assume that default executable is in current directory or in PATH + vcvarsallArg = findExe(vcvarsallDefaultPath) + if printPathArg: var head = $vccversionValue if head.len < 1: @@ -180,9 +186,16 @@ when isMainModule: if not noCommandArg: # Run VCC command with the VCC process environment - let vccProcess = startProcess( - commandArg, - args = clArgs, - options = vccOptions - ) - quit vccProcess.waitForExit() + try: + let vccProcess = startProcess( + commandArg, + args = clArgs, + options = vccOptions + ) + quit vccProcess.waitForExit() + except: + if vcvarsallArg.len != 0: + echo "Hint: using " & vcvarsallArg + else: + echo "Hint: vcvarsall.bat was not found" + raise diff --git a/tools/vccexe/vcvarsall.nim b/tools/vccexe/vcvarsall.nim index defcf687f8..81b0fb42b1 100644 --- a/tools/vccexe/vcvarsall.nim +++ b/tools/vccexe/vcvarsall.nim @@ -10,7 +10,6 @@ import strtabs, strutils, os, osproc const comSpecEnvKey = "ComSpec" ## Environment Variable that specifies the command-line application path in Windows ## Usually set to cmd.exe - vcvarsallDefaultPath = "vcvarsall.bat" type VccArch* = enum ## The VCC compile target architectures @@ -48,11 +47,10 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type ## verbose ## Echo the command-line passed on to the system to load the VCC environment. Defaults to `false`. - var vccvarsallpath = path - # Assume that default executable is in current directory or in PATH if path == "": - vccvarsallpath = vcvarsallDefaultPath + return nil + let vccvarsallpath = path var args: seq[string] = @[] let archStr: string = $arch From 35d96d8749809863bbbb6ead1e8cf887a95a94e5 Mon Sep 17 00:00:00 2001 From: alaviss Date: Wed, 23 Jan 2019 22:35:34 +0700 Subject: [PATCH 211/350] compiler/sem*: better lineinfo for templates (#10428) * compiler/sem*: better lineinfo for templates Lineinfo for templates is inconsistant across the compiler, for example: doAssert true ^ ^ a[int](10) ^^ ^ The `^` marks where the compiler thinks the template starts. For qualified call, we got the same situation with `proc`s before #10427: system.once ^ Generics lineinfo within template declaration is also incorrect, for example, this is where the compiler believes the `T` in `[T]` is: template a[T](b: T) ^ This PR addresses all of these problems. * nimsuggest: add tests for template highlighting --- compiler/semexprs.nim | 14 +++++++++----- compiler/semtypes.nim | 4 ++-- nimsuggest/tests/tgeneric_highlight.nim | 4 +++- nimsuggest/tests/tqualified_highlight.nim | 5 ++++- nimsuggest/tests/ttemplate_highlight.nim | 9 +++++++++ 5 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 nimsuggest/tests/ttemplate_highlight.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 96fefa4b89..37e2ca33f8 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -24,15 +24,18 @@ const proc semTemplateExpr(c: PContext, n: PNode, s: PSym, flags: TExprFlags = {}): PNode = - markUsed(c.config, n.info, s, c.graph.usageSym) - onUse(n.info, s) + let info = getCallLineInfo(n) + markUsed(c.config, info, s, c.graph.usageSym) + onUse(info, s) + # Note: This is n.info on purpose. It prevents template from creating an info + # context when called from an another template pushInfoContext(c.config, n.info, s.detailedInfo) result = evalTemplate(n, s, getCurrOwner(c), c.config, efFromHlo in flags) if efNoSemCheck notin flags: result = semAfterMacroCall(c, n, result, s, flags) popInfoContext(c.config) # XXX: A more elaborate line info rewrite might be needed - result.info = n.info + result.info = info proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags = {}): PNode @@ -1084,8 +1087,9 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or (n.kind notin nkCallKinds and s.requiredParams > 0) or sfCustomPragma in sym.flags: - markUsed(c.config, n.info, s, c.graph.usageSym) - onUse(n.info, s) + let info = getCallLineInfo(n) + markUsed(c.config, info, s, c.graph.usageSym) + onUse(info, s) result = symChoice(c, n, s, scClosed) else: result = semTemplateExpr(c, n, s, flags) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index e4f7dc1479..fbf3638347 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1015,8 +1015,8 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, result = addImplicitGeneric(copyType(paramType, getCurrOwner(c), false)) of tyGenericParam: - markUsed(c.config, info, paramType.sym, c.graph.usageSym) - onUse(info, paramType.sym) + markUsed(c.config, paramType.sym.info, paramType.sym, c.graph.usageSym) + onUse(paramType.sym.info, paramType.sym) if tfWildcard in paramType.flags: paramType.flags.excl tfWildcard paramType.sym.kind = skType diff --git a/nimsuggest/tests/tgeneric_highlight.nim b/nimsuggest/tests/tgeneric_highlight.nim index 85bef7c43f..3343236132 100644 --- a/nimsuggest/tests/tgeneric_highlight.nim +++ b/nimsuggest/tests/tgeneric_highlight.nim @@ -1,8 +1,8 @@ newSeq[int]() system.newSeq[int]()#[!]# +offsetOf[int]() discard """ -disabled:true $nimsuggest --tester $file >highlight $1 highlight;;skType;;1;;7;;3 @@ -15,4 +15,6 @@ highlight;;skProc;;2;;7;;6 highlight;;skProc;;2;;7;;6 highlight;;skType;;2;;14;;3 highlight;;skProc;;2;;7;;6 +highlight;;skTemplate;;3;;0;;8 +highlight;;skType;;3;;9;;3 """ diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim index fe7c889d64..1b34dbade5 100644 --- a/nimsuggest/tests/tqualified_highlight.nim +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -1,9 +1,12 @@ system.echo#[!]# +system.once discard """ -disabled:true $nimsuggest --tester $file >highlight $1 highlight;;skProc;;1;;7;;4 highlight;;skProc;;1;;7;;4 +highlight;;skTemplate;;2;;7;;4 +highlight;;skTemplate;;2;;7;;4 +highlight;;skTemplate;;2;;7;;4 """ diff --git a/nimsuggest/tests/ttemplate_highlight.nim b/nimsuggest/tests/ttemplate_highlight.nim new file mode 100644 index 0000000000..2cbac3be5e --- /dev/null +++ b/nimsuggest/tests/ttemplate_highlight.nim @@ -0,0 +1,9 @@ +doAssert true#[!]# + +discard """ +$nimsuggest --tester $1 +>highlight $1 +highlight;;skTemplate;;1;;0;;8 +highlight;;skTemplate;;1;;0;;8 +highlight;;skEnumField;;1;;9;;4 +""" From 400b4e4e0ea33ef3d7ad94c503c5e00001dec536 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2019 12:27:44 +0100 Subject: [PATCH 212/350] improve the generated code size for --gc:regions --- compiler/cgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 190814b79e..78f9e72da1 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -321,7 +321,7 @@ proc resetLoc(p: BProc, loc: var TLoc) = else: if optNilCheck in p.options: linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", addrLoc(p.config, loc)) - if loc.storage != OnStack: + if loc.storage != OnStack and containsGcRef: linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n", addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)) # XXX: generated reset procs should not touch the m_type From 40d1c7d43a6459aedf1c64a7f5e145ebbe4badcb Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2019 14:32:00 +0100 Subject: [PATCH 213/350] OutOfMemError is never raised so there is no need to check for it (produce less code) --- lib/system/excpt.nim | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index e9d251bb4c..2f41ad1e3c 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -347,24 +347,17 @@ proc raiseExceptionAux(e: ref Exception) = if globalRaiseHook != nil: if not globalRaiseHook(e): return when defined(cpp) and not defined(noCppExceptions): - if e[] of OutOfMemError: - showErrorMessage(e.name) - quitOrDebug() - else: - pushCurrentException(e) - raiseCounter.inc - if raiseCounter == 0: - raiseCounter.inc # skip zero at overflow - e.raiseId = raiseCounter - {.emit: "`e`->raise();".} + pushCurrentException(e) + raiseCounter.inc + if raiseCounter == 0: + raiseCounter.inc # skip zero at overflow + e.raiseId = raiseCounter + {.emit: "`e`->raise();".} else: if excHandler != nil: if not excHandler.hasRaiseAction or excHandler.raiseAction(e): pushCurrentException(e) c_longjmp(excHandler.context, 1) - elif e[] of OutOfMemError: - showErrorMessage(e.name) - quitOrDebug() else: when hasSomeStackTrace: var buf = newStringOfCap(2000) From c4d2cb9ea62108f08b7c5e4fee082c2802489642 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2019 15:28:41 +0100 Subject: [PATCH 214/350] excpt.nim: code cleanup, don't produce nimFrame in -d:release mode --- lib/system/excpt.nim | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 2f41ad1e3c..cc0c1f54bd 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -446,27 +446,21 @@ when not defined(gcDestructors): ## a ``seq``. This is not yet available for the JS backend. shallowCopy(result, e.trace) -when defined(nimRequiresNimFrame): - const nimCallDepthLimit {.intdefine.} = 2000 +const nimCallDepthLimit {.intdefine.} = 2000 - proc callDepthLimitReached() {.noinline.} = - writeStackTrace() - showErrorMessage("Error: call depth limit reached in a debug build (" & - $nimCallDepthLimit & " function calls). You can change it with " & - "-d:nimCallDepthLimit= but really try to avoid deep " & - "recursions instead.\n") - quitOrDebug() +proc callDepthLimitReached() {.noinline.} = + writeStackTrace() + showErrorMessage("Error: call depth limit reached in a debug build (" & + $nimCallDepthLimit & " function calls). You can change it with " & + "-d:nimCallDepthLimit= but really try to avoid deep " & + "recursions instead.\n") + quitOrDebug() - proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = - s.calldepth = if framePtr == nil: 0 else: framePtr.calldepth+1 - s.prev = framePtr - framePtr = s - if s.calldepth == nimCallDepthLimit: callDepthLimitReached() -else: - proc pushFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = - # XXX only for backwards compatibility - s.prev = framePtr - framePtr = s +proc nimFrame(s: PFrame) {.compilerRtl, inl.} = + s.calldepth = if framePtr == nil: 0 else: framePtr.calldepth+1 + s.prev = framePtr + framePtr = s + if s.calldepth == nimCallDepthLimit: callDepthLimitReached() when defined(endb): var From 28cb81ea87aee0a6a2a9d8445abba2a580bb158e Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 23 Jan 2019 17:10:00 +0100 Subject: [PATCH 215/350] make DLLs tests compile again; remove dependency on nimbase.h slightly --- compiler/cgen.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 78f9e72da1..464ef39751 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1056,9 +1056,22 @@ proc genVarPrototype(m: BModule, n: PNode) = if sfVolatile in sym.flags: add(m.s[cfsVars], " volatile") addf(m.s[cfsVars], " $1;$n", [sym.loc.r]) +const + frameDefines = """ +$1 define nimfr_(proc, file) \ + TFrame FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; #nimFrame(&FR_); + +$1 define nimfrs_(proc, file, slots, length) \ + struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; #nimFrame((TFrame*)&FR_); + +$1 define nimln_(n, file) \ + FR_.line = n; FR_.filename = file; +""" + proc addIntTypes(result: var Rope; conf: ConfigRef) {.inline.} = - addf(result, "#define NIM_NEW_MANGLING_RULES\L" & - "#define NIM_INTBITS $1\L", [ + addf(result, "#define NIM_INTBITS $1\L", [ platform.CPU[conf.target.targetCPU].intSize.rope]) if conf.cppCustomNamespace.len > 0: result.add("#define USE_NIM_NAMESPACE ") @@ -1302,6 +1315,7 @@ proc genInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed + appcg(m, m.s[cfsForwardTypes], frameDefines, [rope("#")]) var moduleInitRequired = false let initname = getInitName(m.module) From f0be575ed189ac5ac69a700e8387e60a0e2ebca7 Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 23 Jan 2019 15:23:39 +0100 Subject: [PATCH 216/350] move tests from `tospaths` to `tos`, fixes #9671 Also, change some of `echo`s to `doAssert`. --- tests/stdlib/tos.nim | 154 +++++++++++++++++++++++++++----------- tests/stdlib/tospaths.nim | 99 ------------------------ 2 files changed, 111 insertions(+), 142 deletions(-) delete mode 100644 tests/stdlib/tospaths.nim diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index e4e14d5a14..23fa4d098a 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -1,13 +1,5 @@ discard """ - output: '''true -true -true -true -true -true -true -true -true + output: ''' All: __really_obscure_dir_name/are.x __really_obscure_dir_name/created @@ -27,31 +19,13 @@ __really_obscure_dir_name/created __really_obscure_dir_name/dirs __really_obscure_dir_name/some __really_obscure_dir_name/test -false -false -false -false -false -false -false -false -false -true -true Raises Raises -true -true -true -true -true -true - ''' """ # test os path creation, iteration, and deletion -import os, strutils +import os, strutils, pathnorm block fileOperations: let files = @["these.txt", "are.x", "testing.r", "files.q"] @@ -60,17 +34,17 @@ block fileOperations: let dname = "__really_obscure_dir_name" createDir(dname) - echo dirExists(dname) + doAssert dirExists(dname) # Test creating files and dirs for dir in dirs: createDir(dname/dir) - echo dirExists(dname/dir) + doAssert dirExists(dname/dir) for file in files: let fh = open(dname/file, fmReadWrite) fh.close() - echo fileExists(dname/file) + doAssert fileExists(dname/file) echo "All:" @@ -93,23 +67,23 @@ block fileOperations: # Test removal of files dirs for dir in dirs: removeDir(dname/dir) - echo dirExists(dname/dir) + doAssert: not dirExists(dname/dir) for file in files: removeFile(dname/file) - echo fileExists(dname/file) + doAssert: not fileExists(dname/file) removeDir(dname) - echo dirExists(dname) + doAssert: not dirExists(dname) # createDir should create recursive directories createDir(dirs[0] / dirs[1]) - echo dirExists(dirs[0] / dirs[1]) # true + doAssert dirExists(dirs[0] / dirs[1]) # true removeDir(dirs[0]) # createDir should properly handle trailing separator createDir(dname / "") - echo dirExists(dname) # true + doAssert dirExists(dname) # true removeDir(dname) # createDir should raise IOError if the path exists @@ -138,10 +112,10 @@ block fileOperations: copyDir("a", "../dest/a") removeDir("a") - echo dirExists("../dest/a/b") - echo fileExists("../dest/a/b/file.txt") + doAssert dirExists("../dest/a/b") + doAssert fileExists("../dest/a/b/file.txt") - echo fileExists("../dest/a/b/c/fileC.txt") + doAssert fileExists("../dest/a/b/c/fileC.txt") removeDir("../dest") # test copyDir: @@ -152,8 +126,8 @@ block fileOperations: copyDir("a/", "../dest/a/") removeDir("a") - echo dirExists("../dest/a/b") - echo fileExists("../dest/a/file.txt") + doAssert dirExists("../dest/a/b") + doAssert fileExists("../dest/a/file.txt") removeDir("../dest") import times @@ -165,9 +139,9 @@ block modificationTime: setLastModificationTime("a", tm) when defined(macosx): - echo "true" + doAssert true else: - echo getLastModificationTime("a") == tm + doAssert getLastModificationTime("a") == tm removeFile("a") block walkDirRec: @@ -265,3 +239,97 @@ block splitFile: doAssert splitFile("a/..") == ("a", "..", "") # execShellCmd is tested in tosproc + +block ospaths: + doAssert unixToNativePath("") == "" + doAssert unixToNativePath(".") == $CurDir + doAssert unixToNativePath("..") == $ParDir + doAssert isAbsolute(unixToNativePath("/")) + doAssert isAbsolute(unixToNativePath("/", "a")) + doAssert isAbsolute(unixToNativePath("/a")) + doAssert isAbsolute(unixToNativePath("/a", "a")) + doAssert isAbsolute(unixToNativePath("/a/b")) + doAssert isAbsolute(unixToNativePath("/a/b", "a")) + doAssert unixToNativePath("a/b") == joinPath("a", "b") + + when defined(macos): + doAssert unixToNativePath("./") == ":" + doAssert unixToNativePath("./abc") == ":abc" + doAssert unixToNativePath("../abc") == "::abc" + doAssert unixToNativePath("../../abc") == ":::abc" + doAssert unixToNativePath("/abc", "a") == "abc" + doAssert unixToNativePath("/abc/def", "a") == "abc:def" + elif doslikeFileSystem: + doAssert unixToNativePath("./") == ".\\" + doAssert unixToNativePath("./abc") == ".\\abc" + doAssert unixToNativePath("../abc") == "..\\abc" + doAssert unixToNativePath("../../abc") == "..\\..\\abc" + doAssert unixToNativePath("/abc", "a") == "a:\\abc" + doAssert unixToNativePath("/abc/def", "a") == "a:\\abc\\def" + else: + #Tests for unix + doAssert unixToNativePath("./") == "./" + doAssert unixToNativePath("./abc") == "./abc" + doAssert unixToNativePath("../abc") == "../abc" + doAssert unixToNativePath("../../abc") == "../../abc" + doAssert unixToNativePath("/abc", "a") == "/abc" + doAssert unixToNativePath("/abc/def", "a") == "/abc/def" + + block extractFilenameTest: + doAssert extractFilename("") == "" + when defined(posix): + doAssert extractFilename("foo/bar") == "bar" + doAssert extractFilename("foo/bar.txt") == "bar.txt" + doAssert extractFilename("foo/") == "" + doAssert extractFilename("/") == "" + when doslikeFileSystem: + doAssert extractFilename(r"foo\bar") == "bar" + doAssert extractFilename(r"foo\bar.txt") == "bar.txt" + doAssert extractFilename(r"foo\") == "" + doAssert extractFilename(r"C:\") == "" + + block lastPathPartTest: + doAssert lastPathPart("") == "" + when defined(posix): + doAssert lastPathPart("foo/bar.txt") == "bar.txt" + doAssert lastPathPart("foo/") == "foo" + doAssert lastPathPart("/") == "" + when doslikeFileSystem: + doAssert lastPathPart(r"foo\bar.txt") == "bar.txt" + doAssert lastPathPart(r"foo\") == "foo" + + template canon(x): untyped = normalizePath(x, '/') + doAssert canon"/foo/../bar" == "/bar" + doAssert canon"foo/../bar" == "bar" + + doAssert canon"/f/../bar///" == "/bar" + doAssert canon"f/..////bar" == "bar" + + doAssert canon"../bar" == "../bar" + doAssert canon"/../bar" == "/../bar" + + doAssert canon("foo/../../bar/") == "../bar" + doAssert canon("./bla/blob/") == "bla/blob" + doAssert canon(".hiddenFile") == ".hiddenFile" + doAssert canon("./bla/../../blob/./zoo.nim") == "../blob/zoo.nim" + + doAssert canon("C:/file/to/this/long") == "C:/file/to/this/long" + doAssert canon("") == "" + doAssert canon("foobar") == "foobar" + doAssert canon("f/////////") == "f" + + doAssert relativePath("/foo/bar//baz.nim", "/foo", '/') == "bar/baz.nim" + doAssert normalizePath("./foo//bar/../baz", '/') == "foo/baz" + + doAssert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + + doAssert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + doAssert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + doAssert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + doAssert relativePath("", "/users/moo", '/') == "" + doAssert relativePath("foo", "", '/') == "foo" + + doAssert joinPath("usr", "") == unixToNativePath"usr/" + doAssert joinPath("", "lib") == "lib" + doAssert joinPath("", "/lib") == unixToNativePath"/lib" + doAssert joinPath("usr/", "/lib") == unixToNativePath"usr/lib" diff --git a/tests/stdlib/tospaths.nim b/tests/stdlib/tospaths.nim deleted file mode 100644 index ce00b5a958..0000000000 --- a/tests/stdlib/tospaths.nim +++ /dev/null @@ -1,99 +0,0 @@ -discard """ - output: "" -""" -# test the ospaths module - -import os, pathnorm - -doAssert unixToNativePath("") == "" -doAssert unixToNativePath(".") == $CurDir -doAssert unixToNativePath("..") == $ParDir -doAssert isAbsolute(unixToNativePath("/")) -doAssert isAbsolute(unixToNativePath("/", "a")) -doAssert isAbsolute(unixToNativePath("/a")) -doAssert isAbsolute(unixToNativePath("/a", "a")) -doAssert isAbsolute(unixToNativePath("/a/b")) -doAssert isAbsolute(unixToNativePath("/a/b", "a")) -doAssert unixToNativePath("a/b") == joinPath("a", "b") - -when defined(macos): - doAssert unixToNativePath("./") == ":" - doAssert unixToNativePath("./abc") == ":abc" - doAssert unixToNativePath("../abc") == "::abc" - doAssert unixToNativePath("../../abc") == ":::abc" - doAssert unixToNativePath("/abc", "a") == "abc" - doAssert unixToNativePath("/abc/def", "a") == "abc:def" -elif doslikeFileSystem: - doAssert unixToNativePath("./") == ".\\" - doAssert unixToNativePath("./abc") == ".\\abc" - doAssert unixToNativePath("../abc") == "..\\abc" - doAssert unixToNativePath("../../abc") == "..\\..\\abc" - doAssert unixToNativePath("/abc", "a") == "a:\\abc" - doAssert unixToNativePath("/abc/def", "a") == "a:\\abc\\def" -else: - #Tests for unix - doAssert unixToNativePath("./") == "./" - doAssert unixToNativePath("./abc") == "./abc" - doAssert unixToNativePath("../abc") == "../abc" - doAssert unixToNativePath("../../abc") == "../../abc" - doAssert unixToNativePath("/abc", "a") == "/abc" - doAssert unixToNativePath("/abc/def", "a") == "/abc/def" - -block extractFilenameTest: - doAssert extractFilename("") == "" - when defined(posix): - doAssert extractFilename("foo/bar") == "bar" - doAssert extractFilename("foo/bar.txt") == "bar.txt" - doAssert extractFilename("foo/") == "" - doAssert extractFilename("/") == "" - when doslikeFileSystem: - doAssert extractFilename(r"foo\bar") == "bar" - doAssert extractFilename(r"foo\bar.txt") == "bar.txt" - doAssert extractFilename(r"foo\") == "" - doAssert extractFilename(r"C:\") == "" - -block lastPathPartTest: - doAssert lastPathPart("") == "" - when defined(posix): - doAssert lastPathPart("foo/bar.txt") == "bar.txt" - doAssert lastPathPart("foo/") == "foo" - doAssert lastPathPart("/") == "" - when doslikeFileSystem: - doAssert lastPathPart(r"foo\bar.txt") == "bar.txt" - doAssert lastPathPart(r"foo\") == "foo" - -template canon(x): untyped = normalizePath(x, '/') -doAssert canon"/foo/../bar" == "/bar" -doAssert canon"foo/../bar" == "bar" - -doAssert canon"/f/../bar///" == "/bar" -doAssert canon"f/..////bar" == "bar" - -doAssert canon"../bar" == "../bar" -doAssert canon"/../bar" == "/../bar" - -doAssert canon("foo/../../bar/") == "../bar" -doAssert canon("./bla/blob/") == "bla/blob" -doAssert canon(".hiddenFile") == ".hiddenFile" -doAssert canon("./bla/../../blob/./zoo.nim") == "../blob/zoo.nim" - -doAssert canon("C:/file/to/this/long") == "C:/file/to/this/long" -doAssert canon("") == "" -doAssert canon("foobar") == "foobar" -doAssert canon("f/////////") == "f" - -doAssert relativePath("/foo/bar//baz.nim", "/foo", '/') == "bar/baz.nim" -doAssert normalizePath("./foo//bar/../baz", '/') == "foo/baz" - -doAssert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - -doAssert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" -doAssert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" -doAssert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" -doAssert relativePath("", "/users/moo", '/') == "" -doAssert relativePath("foo", "", '/') == "foo" - -doAssert joinPath("usr", "") == unixToNativePath"usr/" -doAssert joinPath("", "lib") == "lib" -doAssert joinPath("", "/lib") == unixToNativePath"/lib" -doAssert joinPath("usr/", "/lib") == unixToNativePath"usr/lib" From ce5e8b93e2da4f5a5381e46a89fd89a1bfc4caa2 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 21 Jan 2019 14:19:15 +0100 Subject: [PATCH 217/350] better docs: xmltree --- changelog.md | 2 + lib/pure/xmltree.nim | 555 +++++++++++++++++++++++++++++++++---------- 2 files changed, 434 insertions(+), 123 deletions(-) diff --git a/changelog.md b/changelog.md index d56d225060..eb97ac9a6f 100644 --- a/changelog.md +++ b/changelog.md @@ -120,6 +120,8 @@ proc enumToString*(enums: openArray[enum]): string = - Added `os.getCurrentCompilerExe` (implmented as `getAppFilename` at CT), can be used to retrieve the currently executing compiler. +- Added `xmltree.toXmlAttributes`. + ### Library changes diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index b99d46bc89..e1d24ea420 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -7,21 +7,52 @@ # distribution, for details about the copyright. # -## A simple XML tree. +## A simple XML tree generator. +## +## .. code-block:: +## import xmltree +## +## var g = newElement("myTag") +## g.add newText("some text") +## g.add newComment("this is comment") +## +## var h = newElement("secondTag") +## h.add newEntity("some entity") +## +## let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes +## let k = newXmlTree("treeTag", [g, h], att) +## +## echo k +## # +## # some text +## # &some entity; +## # +## +## +## **See also:** +## * `xmlparser module `_ for high-level XML parsing +## * `parsexml module `_ for low-level XML parsing +## * `htmlgen module `_ for html code generator import macros, strtabs, strutils type - XmlNode* = ref XmlNodeObj ## an XML tree consists of ``XmlNode``'s. + XmlNode* = ref XmlNodeObj ## An XML tree consisting of XML nodes. + ## + ## Use `newXmlTree proc <#newXmlTree,string,openArray[XmlNode],XmlAttributes>`_ + ## for creating a new tree. - XmlNodeKind* = enum ## different kinds of ``XmlNode``'s + XmlNodeKind* = enum ## Different kinds of XML nodes. xnText, ## a text element xnElement, ## an element with 0 or more children xnCData, ## a CDATA node xnEntity, ## an entity (like ``&thing;``) xnComment ## an XML comment - XmlAttributes* = StringTableRef ## an alias for a string to string mapping + XmlAttributes* = StringTableRef ## An alias for a string to string mapping. + ## + ## Use `toXmlAttributes proc <#toXmlAttributes,varargs[tuple[string,string]]>`_ + ## to create `XmlAttributes`. XmlNodeObj {.acyclic.} = object case k: XmlNodeKind # private, use the kind() proc to read this field. @@ -33,67 +64,203 @@ type fAttr: XmlAttributes fClientData: int ## for other clients +const + xmlHeader* = "\n" + ## Header to use for complete XML output. + proc newXmlNode(kind: XmlNodeKind): XmlNode = - ## creates a new ``XmlNode``. + ## Creates a new ``XmlNode``. new(result) result.k = kind proc newElement*(tag: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnText`` with the given `tag`. + ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. + ## + ## See also: + ## * `newXmlTree proc <#newXmlTree,string,openArray[XmlNode],XmlAttributes>`_ + ## * [<> macro](#<>.m,untyped) + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert a.kind == xnElement + assert $a == "" + result = newXmlNode(xnElement) result.fTag = tag result.s = @[] - # init attributes lazily to safe memory + # init attributes lazily to save memory proc newText*(text: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnText`` with the text `text`. + ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. + runnableExamples: + var b = newText("my text") + assert b.kind == xnText + assert $b == "my text" + result = newXmlNode(xnText) result.fText = text proc newComment*(comment: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `comment`. + ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. + runnableExamples: + var c = newComment("my comment") + assert c.kind == xnComment + assert $c == "" + result = newXmlNode(xnComment) result.fText = comment proc newCData*(cdata: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `cdata`. + ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. + runnableExamples: + var d = newCData("my cdata") + assert d.kind == xnCData + assert $d == "" + result = newXmlNode(xnCData) result.fText = cdata proc newEntity*(entity: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnEntity`` with the text `entity`. + ## Creates a new ``XmlNode`` of kind ``xnEntity`` with the text `entity`. + runnableExamples: + var e = newEntity("my entity") + assert e.kind == xnEntity + assert $e == "&my entity;" + result = newXmlNode(xnEntity) result.fText = entity +proc newXmlTree*(tag: string, children: openArray[XmlNode], + attributes: XmlAttributes = nil): XmlNode = + ## Creates a new XML tree with `tag`, `children` and `attributes`. + ## + ## See also: + ## * `newElement proc <#newElement,string>`_ + ## * [<> macro](#<>.m,untyped) + runnableExamples: + from strutils import unindent + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + var h = newElement("secondTag") + h.add newEntity("some entity") + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + let k = newXmlTree("treeTag", [g, h], att) + assert ($k).unindent == """ + some text + &some entity; + """.unindent + + result = newXmlNode(xnElement) + result.fTag = tag + newSeq(result.s, children.len) + for i in 0..children.len-1: result.s[i] = children[i] + result.fAttr = attributes + proc text*(n: XmlNode): string {.inline.} = - ## gets the associated text with the node `n`. `n` can be a CDATA, Text, - ## comment, or entity node. + ## Gets the associated text with the node `n`. + ## + ## `n` can be a CDATA, Text, comment, or entity node. + ## + ## See also: + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + ## * `innerText proc <#innerText,XmlNode>`_ + runnableExamples: + var c = newComment("my comment") + assert $c == "" + assert c.text == "my comment" + assert n.k in {xnText, xnComment, xnCData, xnEntity} result = n.fText proc `text=`*(n: XmlNode, text: string){.inline.} = - ## sets the associated text with the node `n`. `n` can be a CDATA, Text, - ## comment, or entity node. + ## Sets the associated text with the node `n`. + ## + ## `n` can be a CDATA, Text, comment, or entity node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + runnableExamples: + var e = newEntity("my entity") + assert $e == "&my entity;" + e.text = "a new entity text" + assert $e == "&a new entity text;" + assert n.k in {xnText, xnComment, xnCData, xnEntity} n.fText = text +proc tag*(n: XmlNode): string {.inline.} = + ## Gets the tag name of `n`. + ## + ## `n` has to be an ``xnElement`` node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + ## * `innerText proc <#innerText,XmlNode>`_ + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert $a == "" + assert a.tag == "firstTag" + + assert n.k == xnElement + result = n.fTag + +proc `tag=`*(n: XmlNode, tag: string) {.inline.} = + ## Sets the tag name of `n`. + ## + ## `n` has to be an ``xnElement`` node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert $a == "" + a.tag = "newTag" + assert $a == "" + + assert n.k == xnElement + n.fTag = tag + proc rawText*(n: XmlNode): string {.inline.} = - ## returns the underlying 'text' string by reference. + ## Returns the underlying 'text' string by reference. + ## ## This is only used for speed hacks. shallowCopy(result, n.fText) proc rawTag*(n: XmlNode): string {.inline.} = - ## returns the underlying 'tag' string by reference. + ## Returns the underlying 'tag' string by reference. + ## ## This is only used for speed hacks. shallowCopy(result, n.fTag) proc innerText*(n: XmlNode): string = - ## gets the inner text of `n`: + ## Gets the inner text of `n`: ## ## - If `n` is `xnText` or `xnEntity`, returns its content. ## - If `n` is `xnElement`, runs recursively on each child node and ## concatenates the results. ## - Otherwise returns an empty string. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ + runnableExamples: + var f = newElement("myTag") + f.add newText("my text") + f.add newComment("my comment") + f.add newEntity("my entity") + assert $f == "my text&my entity;" + assert innerText(f) == "my textmy entity" + proc worker(res: var string, n: XmlNode) = case n.k of xnText, xnEntity: @@ -107,89 +274,218 @@ proc innerText*(n: XmlNode): string = result = "" worker(result, n) -proc tag*(n: XmlNode): string {.inline.} = - ## gets the tag name of `n`. `n` has to be an ``xnElement`` node. - assert n.k == xnElement - result = n.fTag - -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = - ## sets the tag name of `n`. `n` has to be an ``xnElement`` node. - assert n.k == xnElement - n.fTag = tag - proc add*(father, son: XmlNode) {.inline.} = - ## adds the child `son` to `father`. + ## Adds the child `son` to `father`. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + runnableExamples: + var f = newElement("myTag") + f.add newText("my text") + f.add newElement("sonTag") + f.add newEntity("my entity") + assert $f == "my text&my entity;" add(father.s, son) proc insert*(father, son: XmlNode, index: int) {.inline.} = - ## insert the child `son` to a given position in `father`. + ## Insert the child `son` to a given position in `father`. + ## + ## `father` and `son` must be of `xnElement` kind. + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + runnableExamples: + from strutils import unindent + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert ($f).unindent == "\n\n\n" + assert father.k == xnElement and son.k == xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) -proc len*(n: XmlNode): int {.inline.} = - ## returns the number `n`'s children. - if n.k == xnElement: result = len(n.s) - -proc kind*(n: XmlNode): XmlNodeKind {.inline.} = - ## returns `n`'s kind. - result = n.k - -proc `[]`* (n: XmlNode, i: int): XmlNode {.inline.} = - ## returns the `i`'th child of `n`. - assert n.k == xnElement - result = n.s[i] - proc delete*(n: XmlNode, i: Natural) {.noSideEffect.} = - ## delete the `i`'th child of `n`. + ## Delete the `i`'th child of `n`. + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.delete(0) + assert $f == "" + assert n.k == xnElement n.s.delete(i) +proc len*(n: XmlNode): int {.inline.} = + ## Returns the number of `n`'s children. + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert len(f) == 2 + if n.k == xnElement: result = len(n.s) + +proc kind*(n: XmlNode): XmlNodeKind {.inline.} = + ## Returns `n`'s kind. + runnableExamples: + var a = newElement("firstTag") + assert a.kind == xnElement + var b = newText("my text") + assert b.kind == xnText + result = n.k + +proc `[]`* (n: XmlNode, i: int): XmlNode {.inline.} = + ## Returns the `i`'th child of `n`. + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert $f[1] == "" + assert $f[0] == "" + + assert n.k == xnElement + result = n.s[i] + proc `[]`* (n: var XmlNode, i: int): var XmlNode {.inline.} = - ## returns the `i`'th child of `n` so that it can be modified + ## Returns the `i`'th child of `n` so that it can be modified. assert n.k == xnElement result = n.s[i] iterator items*(n: XmlNode): XmlNode {.inline.} = - ## iterates over any child of `n`. + ## Iterates over any child of `n`. + ## + ## **Examples:** + ## + ## .. code-block:: + ## var g = newElement("myTag") + ## g.add newText("some text") + ## g.add newComment("this is comment") + ## + ## var h = newElement("secondTag") + ## h.add newEntity("some entity") + ## g.add h + ## + ## assert $g == "some text&some entity;" + ## for x in g: # the same as `for x in items(g):` + ## echo x + ## + ## # some text + ## # + ## # &some entity; + assert n.k == xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = - ## iterates over any child of `n`. + ## Iterates over any child of `n` so that it can be modified. assert n.k == xnElement for i in 0 .. n.len-1: yield n[i] +proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, val: string]]): XmlAttributes = + ## Converts `{key: value}` pairs into `XmlAttributes`. + runnableExamples: + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + var j = newElement("myTag") + j.attrs = att + assert $j == """""" + + newStringTable(keyValuePairs) + proc attrs*(n: XmlNode): XmlAttributes {.inline.} = - ## gets the attributes belonging to `n`. + ## Gets the attributes belonging to `n`. + ## ## Returns `nil` if attributes have not been initialised for this node. + ## + ## See also: + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for numbef of attributes + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrs == nil + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrs == att + assert n.k == xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = - ## sets the attributes belonging to `n`. + ## Sets the attributes belonging to `n`. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for numbef of attributes + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrs == nil + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrs == att + assert n.k == xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = - ## returns the number of `n`'s attributes. + ## Returns the number of `n`'s attributes. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrsLen == 0 + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrsLen == 2 + assert n.k == xnElement if not isNil(n.fAttr): result = len(n.fAttr) +proc attr*(n: XmlNode, name: string): string = + ## Finds the first attribute of `n` with a name of `name`. + ## Returns "" on failure. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for numbef of attributes + runnableExamples: + var j = newElement("myTag") + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attr("key1") == "first value" + assert j.attr("key2") == "second value" + + assert n.kind == xnElement + if n.attrs == nil: return "" + return n.attrs.getOrDefault(name) + proc clientData*(n: XmlNode): int {.inline.} = - ## gets the client data of `n`. The client data field is used by the HTML - ## parser and generator. + ## Gets the client data of `n`. + ## + ## The client data field is used by the HTML parser and generator. result = n.fClientData proc `clientData=`*(n: XmlNode, data: int) {.inline.} = - ## sets the client data of `n`. The client data field is used by the HTML - ## parser and generator. + ## Sets the client data of `n`. + ## + ## The client data field is used by the HTML parser and generator. n.fClientData = data proc addEscaped*(result: var string, s: string) = - ## same as ``result.add(escape(s))``, but more efficient. + ## The same as `result.add(escape(s)) <#escape,string>`_, but more efficient. for c in items(s): case c of '<': result.add("<") @@ -201,7 +497,8 @@ proc addEscaped*(result: var string, s: string) = else: result.add(c) proc escape*(s: string): string = - ## escapes `s` for inclusion into an XML document. + ## Escapes `s` for inclusion into an XML document. + ## ## Escapes these characters: ## ## ------------ ------------------- @@ -214,6 +511,8 @@ proc escape*(s: string): string = ## ``'`` ``'`` ## ``/`` ``/`` ## ------------ ------------------- + ## + ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) addEscaped(result, s) @@ -223,14 +522,22 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1..indent: result.add(' ') proc noWhitespace(n: XmlNode): bool = - #for i in 1..n.len-1: - # if n[i].kind != n[0].kind: return true for i in 0..n.len-1: if n[i].kind in {xnText, xnEntity}: return true proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, addNewLines=true) = - ## adds the textual representation of `n` to `result`. + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "my text" proc addEscapedAttr(result: var string, s: string) = # `addEscaped` alternative with less escaped characters. @@ -291,24 +598,76 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') -const - xmlHeader* = "\n" - ## header to use for complete XML output - proc `$`*(n: XmlNode): string = - ## converts `n` into its string representation. No ``<$xml ...$>`` declaration - ## is produced, so that the produced XML fragments are composable. + ## Converts `n` into its string representation. + ## + ## No ``<$xml ...$>`` declaration is produced, so that the produced + ## XML fragments are composable. result = "" result.add(n) -proc newXmlTree*(tag: string, children: openArray[XmlNode], - attributes: XmlAttributes = nil): XmlNode = - ## creates a new XML tree with `tag`, `children` and `attributes` - result = newXmlNode(xnElement) - result.fTag = tag - newSeq(result.s, children.len) - for i in 0..children.len-1: result.s[i] = children[i] - result.fAttr = attributes +proc child*(n: XmlNode, name: string): XmlNode = + ## Finds the first child element of `n` with a name of `name`. + ## Returns `nil` on failure. + runnableExamples: + var f = newElement("myTag") + f.add newElement("firstSon") + f.add newElement("secondSon") + f.add newElement("thirdSon") + assert $(f.child("secondSon")) == "" + + assert n.kind == xnElement + for i in items(n): + if i.kind == xnElement: + if i.tag == name: + return i + +proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode]) = + ## Iterates over all the children of `n` returning those matching `tag`. + ## + ## Found nodes satisfying the condition will be appended to the `result` + ## sequence. + runnableExamples: + var + b = newElement("good") + c = newElement("bad") + d = newElement("bad") + e = newElement("good") + b.add newText("b text") + c.add newText("c text") + d.add newText("d text") + e.add newText("e text") + let a = newXmlTree("father", [b, c, d, e]) + var s = newSeq[XmlNode]() + a.findAll("good", s) + assert $s == "@[b text, e text]" + + assert n.k == xnElement + for child in n.items(): + if child.k != xnElement: + continue + if child.tag == tag: + result.add(child) + child.findAll(tag, result) + +proc findAll*(n: XmlNode, tag: string): seq[XmlNode] = + ## A shortcut version to assign in let blocks. + runnableExamples: + var + b = newElement("good") + c = newElement("bad") + d = newElement("bad") + e = newElement("good") + b.add newText("b text") + c.add newText("c text") + d.add newText("d text") + e.add newText("e text") + let a = newXmlTree("father", [b, c, d, e]) + assert $(a.findAll("good")) == "@[b text, e text]" + assert $(a.findAll("bad")) == "@[c text, d text]" + + newSeq(result, 0) + findAll(n, tag, result) proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = if a.kind == nnkCall: @@ -346,56 +705,6 @@ macro `<>`*(x: untyped): untyped = ## result = xmlConstructor(x) -proc child*(n: XmlNode, name: string): XmlNode = - ## Finds the first child element of `n` with a name of `name`. - ## Returns `nil` on failure. - assert n.kind == xnElement - for i in items(n): - if i.kind == xnElement: - if i.tag == name: - return i - -proc attr*(n: XmlNode, name: string): string = - ## Finds the first attribute of `n` with a name of `name`. - ## Returns "" on failure. - assert n.kind == xnElement - if n.attrs == nil: return "" - return n.attrs.getOrDefault(name) - -proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode]) = - ## Iterates over all the children of `n` returning those matching `tag`. - ## - ## Found nodes satisfying the condition will be appended to the `result` - ## sequence, which can't be nil or the proc will crash. Usage example: - ## - ## .. code-block:: - ## var - ## html: XmlNode - ## tags: seq[XmlNode] = @[] - ## - ## html = buildHtml() - ## findAll(html, "img", tags) - ## for imgTag in tags: - ## process(imgTag) - assert n.k == xnElement - for child in n.items(): - if child.k != xnElement: - continue - if child.tag == tag: - result.add(child) - child.findAll(tag, result) - -proc findAll*(n: XmlNode, tag: string): seq[XmlNode] = - ## Shortcut version to assign in let blocks. Example: - ## - ## .. code-block:: - ## var html: XmlNode - ## - ## html = buildHtml(html) - ## for imgTag in html.findAll("img"): - ## process(imgTag) - newSeq(result, 0) - findAll(n, tag, result) when isMainModule: assert """Nim rules.""" == From f9a0eaf89d6e14fa7b7288962628cafdd5b1256d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?= Date: Thu, 24 Jan 2019 10:28:07 +0100 Subject: [PATCH 218/350] better docs: algorithm (#10372) --- lib/pure/algorithm.nim | 439 ++++++++++++++++++++++++++++++++--------- 1 file changed, 347 insertions(+), 92 deletions(-) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index a5b18ae584..8c8838ed2e 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -8,19 +8,59 @@ # ## This module implements some common generic algorithms. +## +## Basic usage +## =========== +## +## .. code-block:: +## import algorithm +## +## type People = tuple +## year: int +## name: string +## +## var a: seq[People] +## +## a.add((2000, "John")) +## a.add((2005, "Marie")) +## a.add((2010, "Jane")) +## +## # Sorting with default system.cmp +## a.sort() +## assert a == @[(year: 2000, name: "John"), (year: 2005, name: "Marie"), +## (year: 2010, name: "Jane")] +## +## proc myCmp(x, y: People): int = +## if x.name < y.name: -1 else: 1 +## +## # Sorting with custom proc +## a.sort(myCmp) +## assert a == @[(year: 2010, name: "Jane"), (year: 2000, name: "John"), +## (year: 2005, name: "Marie")] +## +## +## See also +## ======== +## * `sequtils module`_ for working with the built-in seq type +## * `tables module`_ for sorting tables type SortOrder* = enum Descending, Ascending proc `*`*(x: int, order: SortOrder): int {.inline.} = - ## flips ``x`` if ``order == Descending``. + ## Flips ``x`` if ``order == Descending``. ## If ``order == Ascending`` then ``x`` is returned. ## ## ``x`` is supposed to be the result of a comparator, i.e. ## | ``< 0`` for *less than*, ## | ``== 0`` for *equal*, ## | ``> 0`` for *greater than*. + runnableExamples: + assert `*`(-123, Descending) == 123 + assert `*`(123, Descending) == -123 + assert `*`(-123, Ascending) == -123 + assert `*`(123, Ascending) == 123 var y = order.ord - 1 result = (x xor y) - y @@ -31,28 +71,44 @@ template fillImpl[T](a: var openArray[T], first, last: int, value: T) = inc(x) proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = - ## fills the slice ``a[first..last]`` with ``value``. + ## Fills the slice ``a[first..last]`` with ``value``. + ## + ## If an invalid range is passed, it raises IndexError. runnableExamples: - var a: array[6, int] - a.fill(1, 3, 9) - doAssert a == [0, 9, 9, 9, 0, 0] + var a: array[6, int] + a.fill(1, 3, 9) + assert a == [0, 9, 9, 9, 0, 0] + a.fill(3, 5, 7) + assert a == [0, 9, 9, 7, 7, 7] + doAssertRaises(IndexError, a.fill(1, 7, 9)) fillImpl(a, first, last, value) proc fill*[T](a: var openArray[T], value: T) = - ## fills the container ``a`` with ``value``. + ## Fills the container ``a`` with ``value``. runnableExamples: - var a: array[6, int] - a.fill(9) - doAssert a == [9, 9, 9, 9, 9, 9] + var a: array[6, int] + a.fill(9) + assert a == [9, 9, 9, 9, 9, 9] + a.fill(4) + assert a == [4, 4, 4, 4, 4, 4] fillImpl(a, 0, a.high, value) proc reverse*[T](a: var openArray[T], first, last: Natural) = - ## reverses the slice ``a[first..last]``. + ## Reverses the slice ``a[first..last]``. + ## + ## If an invalid range is passed, it raises IndexError. + ## + ## **See also:** + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a ``seq[T]`` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a ``seq[T]`` runnableExamples: - var a = [1, 2, 3, 4, 5, 6] - a.reverse(1, 3) - doAssert a == [1, 4, 3, 2, 5, 6] + var a = [1, 2, 3, 4, 5, 6] + a.reverse(1, 3) + assert a == [1, 4, 3, 2, 5, 6] + a.reverse(1, 3) + assert a == [1, 2, 3, 4, 5, 6] + doAssertRaises(IndexError, a.reverse(1, 7)) var x = first var y = last while x < y: @@ -61,20 +117,32 @@ proc reverse*[T](a: var openArray[T], first, last: Natural) = inc(x) proc reverse*[T](a: var openArray[T]) = - ## reverses the contents of the container ``a``. + ## Reverses the contents of the container ``a``. + ## + ## **See also:** + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a ``seq[T]`` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a ``seq[T]`` runnableExamples: - var a = [1, 2, 3, 4, 5, 6] - a.reverse() - doAssert a == [6, 5, 4, 3, 2, 1] + var a = [1, 2, 3, 4, 5, 6] + a.reverse() + assert a == [6, 5, 4, 3, 2, 1] + a.reverse() + assert a == [1, 2, 3, 4, 5, 6] reverse(a, 0, max(0, a.high)) proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## returns the reverse of the slice ``a[first..last]``. + ## Returns the reverse of the slice ``a[first..last]``. + ## + ## If an invalid range is passed, it raises IndexError. + ## + ## **See also:** + ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice + ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a, 1, 3) - doAssert b == @[4, 3, 2] + let + a = [1, 2, 3, 4, 5, 6] + b = a.reversed(1, 3) + assert b == @[4, 3, 2] assert last >= first-1 var i = last - first var x = first.int @@ -85,12 +153,16 @@ proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = inc(x) proc reversed*[T](a: openArray[T]): seq[T] = - ## returns the reverse of the container ``a``. + ## Returns the reverse of the container ``a``. + ## + ## **See also:** + ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice + ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a) - doAssert b == @[6, 5, 4, 3, 2, 1] + let + a = [1, 2, 3, 4, 5, 6] + b = reversed(a) + assert b == @[6, 5, 4, 3, 2, 1] reversed(a, 0, a.high) proc binarySearch*[T, K](a: openArray[T], key: K, @@ -99,6 +171,9 @@ proc binarySearch*[T, K](a: openArray[T], key: K, ## ## ``cmp`` is the comparator function to use, the expected return values are ## the same as that of system.cmp. + runnableExamples: + assert binarySearch(["a","b","c","d"], "d", system.cmp[string]) == 3 + assert binarySearch(["a","b","d","c"], "d", system.cmp[string]) == 2 if a.len == 0: return -1 @@ -141,31 +216,41 @@ proc binarySearch*[T, K](a: openArray[T], key: K, proc binarySearch*[T](a: openArray[T], key: T): int = ## Binary search for ``key`` in ``a``. Returns -1 if not found. + runnableExamples: + assert binarySearch([0, 1, 2, 3, 4], 4) == 4 + assert binarySearch([0, 1, 4, 2, 3], 4) == 2 binarySearch(a, key, cmp[T]) proc smartBinarySearch*[T](a: openArray[T], key: T): int {.deprecated.} = - ## **Deprecated since version 0.18.1**; Use ``binarySearch`` instead. + ## **Deprecated since version 0.18.1**; Use `binarySearch proc + ## <#binarySearch,openArray[T],T>`_ instead. binarySearch(a, key, cmp[T]) const onlySafeCode = true proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = - ## returns a position to the first element in the ``a`` that is greater than + ## Returns a position to the first element in the ``a`` that is greater than ## ``key``, or last if no such element is found. ## In other words if you have a sorted sequence and you call ## ``insert(thing, elm, lowerBound(thing, elm))`` ## the sequence will still be sorted. ## - ## The first version uses ``cmp`` to compare the elements. + ## If an invalid range is passed, it raises IndexError. + ## + ## The version uses ``cmp`` to compare the elements. ## The expected return values are the same as that of ``system.cmp``. - ## The second version uses the default comparison function ``cmp``. ## - ## .. code-block:: nim - ## - ## var arr = @[1,2,3,5,6,7,8,9] - ## arr.insert(4, arr.lowerBound(4)) - ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + ## **See also:** + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],T>`_ + runnableExamples: + var arr = @[1,2,3,5,6,7,8,9] + assert arr.lowerBound(3, system.cmp[int]) == 2 + assert arr.lowerBound(4, system.cmp[int]) == 3 + assert arr.lowerBound(5, system.cmp[int]) == 3 + arr.insert(4, arr.lowerBound(4, system.cmp[int])) + assert arr == [1,2,3,4,5,6,7,8,9] result = a.low var count = a.high - a.low + 1 var step, pos: int @@ -179,23 +264,40 @@ proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.clo count = step proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) + ## Returns a position to the first element in the ``a`` that is greater than + ## ``key``, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## ``insert(thing, elm, lowerBound(thing, elm))`` + ## the sequence will still be sorted. + ## + ## The version uses the default comparison function ``cmp``. + ## + ## **See also:** + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],T>`_ proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = - ## returns a position to the first element in the ``a`` that is not less + ## Returns a position to the first element in the ``a`` that is not less ## (i.e. greater or equal to) than ``key``, or last if no such element is found. ## In other words if you have a sorted sequence and you call ## ``insert(thing, elm, upperBound(thing, elm))`` ## the sequence will still be sorted. ## - ## The first version uses ``cmp`` to compare the elements. The expected + ## If an invalid range is passed, it raises IndexError. + ## + ## The version uses ``cmp`` to compare the elements. The expected ## return values are the same as that of ``system.cmp``. - ## The second version uses the default comparison function ``cmp``. ## - ## .. code-block:: nim - ## - ## var arr = @[1,2,3,4,6,7,8,9] - ## arr.insert(5, arr.upperBound(4)) - ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + ## **See also:** + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ + runnableExamples: + var arr = @[1,2,3,5,6,7,8,9] + assert arr.upperBound(2, system.cmp[int]) == 2 + assert arr.upperBound(3, system.cmp[int]) == 3 + assert arr.upperBound(4, system.cmp[int]) == 3 + arr.insert(4, arr.upperBound(3, system.cmp[int])) + assert arr == [1,2,3,4,5,6,7,8,9] result = a.low var count = a.high - a.low + 1 var step, pos: int @@ -209,6 +311,17 @@ proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.clo count = step proc upperBound*[T](a: openArray[T], key: T): int = upperBound(a, key, cmp[T]) + ## Returns a position to the first element in the ``a`` that is not less + ## (i.e. greater or equal to) than ``key``, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## ``insert(thing, elm, upperBound(thing, elm))`` + ## the sequence will still be sorted. + ## + ## The version uses the default comparison function ``cmp``. + ## + ## **See also:** + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ template `<-` (a, b) = when false: @@ -263,6 +376,7 @@ func sort*[T](a: var openArray[T], ## Default Nim sort (an implementation of merge sort). The sorting ## is guaranteed to be stable and the worst case is guaranteed to ## be O(n log n). + ## ## The current implementation uses an iterative ## mergesort to achieve this. It uses a temporary sequence of ## length ``a.len div 2``. If you do not wish to provide your own @@ -272,7 +386,6 @@ func sort*[T](a: var openArray[T], ## .. code-block:: nim ## ## sort(myIntArray, system.cmp[int]) - ## ## # do not use cmp[string] here as we want to use the specialized ## # overload: ## sort(myStrArray, system.cmp) @@ -286,6 +399,19 @@ func sort*[T](a: var openArray[T], ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## + ## **See also:** + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + runnableExamples: + var d = ["boo", "fo", "barr", "qux"] + proc myCmp(x, y: string): int = + if x.len() > y.len() or x.len() == y.len(): 1 + else: -1 + sort(d, myCmp) + assert d == ["fo", "qux", "boo", "barr"] var n = a.len var b: seq[T] newSeq(b, n div 2) @@ -299,17 +425,30 @@ func sort*[T](a: var openArray[T], proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, system.cmp[T], order) ## Shortcut version of ``sort`` that uses ``system.cmp[T]`` as the comparison function. + ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, order = SortOrder.Ascending): seq[T] = - ## returns ``a`` sorted by ``cmp`` in the specified ``order``. + ## Returns ``a`` sorted by ``cmp`` in the specified ``order``. + ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ runnableExamples: - let - a = [2, 3, 1, 5, 4] - b = sorted(a, system.cmp) - c = sorted(a, system.cmp, Descending) - doAssert b == @[1, 2, 3, 4, 5] - doAssert c == @[5, 4, 3, 2, 1] + let + a = [2, 3, 1, 5, 4] + b = sorted(a, system.cmp[int]) + c = sorted(a, system.cmp[int], Descending) + d = sorted(["adam", "dande", "brian", "cat"], system.cmp[string]) + assert b == @[1, 2, 3, 4, 5] + assert c == @[5, 4, 3, 2, 1] + assert d == @["adam", "brian", "cat", "dande"] result = newSeq[T](a.len) for i in 0 .. a.high: result[i] = a[i] @@ -317,33 +456,48 @@ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, proc sorted*[T](a: openArray[T], order = SortOrder.Ascending): seq[T] = ## Shortcut version of ``sorted`` that uses ``system.cmp[T]`` as the comparison function. + ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = sorted(a) + c = sorted(a, Descending) + d = sorted(["adam", "dande", "brian", "cat"]) + assert b == @[1, 2, 3, 4, 5] + assert c == @[5, 4, 3, 2, 1] + assert d == @["adam", "brian", "cat", "dande"] sorted[T](a, system.cmp[T], order) template sortedByIt*(seq1, op: untyped): untyped = ## Convenience template around the ``sorted`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an - ## expression. Example: - ## - ## .. code-block:: nim - ## - ## type Person = tuple[name: string, age: int] - ## var - ## p1: Person = (name: "p1", age: 60) - ## p2: Person = (name: "p2", age: 20) - ## p3: Person = (name: "p3", age: 30) - ## p4: Person = (name: "p4", age: 30) - ## people = @[p1,p2,p4,p3] - ## - ## echo people.sortedByIt(it.name) + ## expression. ## ## Because the underlying ``cmp()`` is defined for tuples you can do - ## a nested sort like in the following example: - ## - ## .. code-block:: nim - ## - ## echo people.sortedByIt((it.age, it.name)) + ## a nested sort. ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + runnableExamples: + type Person = tuple[name: string, age: int] + var + p1: Person = (name: "p1", age: 60) + p2: Person = (name: "p2", age: 20) + p3: Person = (name: "p3", age: 30) + p4: Person = (name: "p4", age: 30) + people = @[p1,p2,p4,p3] + + assert people.sortedByIt(it.name) == @[(name: "p1", age: 60), (name: "p2", age: 20), (name: "p3", age: 30), (name: "p4", age: 30)] + # Nested sort + assert people.sortedByIt((it.age, it.name)) == @[(name: "p2", age: 20), (name: "p3", age: 30), (name: "p4", age: 30), (name: "p1", age: 60)] var result = sorted(seq1, proc(x, y: type(seq1[0])): int = var it {.inject.} = x let a = op @@ -355,9 +509,25 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, order = SortOrder.Ascending): bool = - ## checks to see whether ``a`` is already sorted in ``order`` + ## Checks to see whether ``a`` is already sorted in ``order`` ## using ``cmp`` for the comparison. Parameters identical ## to ``sort``. + ## + ## **See also:** + ## * `isSorted proc<#isSorted,openArray[T]>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = [1, 2, 3, 4, 5] + c = [5, 4, 3, 2, 1] + d = ["adam", "brian", "cat", "dande"] + e = ["adam", "dande", "brian", "cat"] + assert isSorted(a) == false + assert isSorted(b) == true + assert isSorted(c) == false + assert isSorted(c, Descending) == true + assert isSorted(d) == true + assert isSorted(e) == false result = true for i in 0.. 0: @@ -365,11 +535,30 @@ func isSorted*[T](a: openArray[T], proc isSorted*[T](a: openarray[T], order = SortOrder.Ascending): bool = ## Shortcut version of ``isSorted`` that uses ``system.cmp[T]`` as the comparison function. + ## + ## **See also:** + ## * `isSorted func<#isSorted,openArray[T],proc(T,T)>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = [1, 2, 3, 4, 5] + c = [5, 4, 3, 2, 1] + d = ["adam", "brian", "cat", "dande"] + e = ["adam", "dande", "brian", "cat"] + assert isSorted(a) == false + assert isSorted(b) == true + assert isSorted(c) == false + assert isSorted(c, Descending) == true + assert isSorted(d) == true + assert isSorted(e) == false isSorted(a, system.cmp[T], order) proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = - ## produces the Cartesian product of the array. Warning: complexity + ## Produces the Cartesian product of the array. Warning: complexity ## may explode. + runnableExamples: + assert product(@[@[1], @[2]]) == @[@[1, 2]] + assert product(@[@["A", "K"], @["Q"]]) == @[@["K", "Q"], @["A", "Q"]] result = newSeq[seq[T]]() if x.len == 0: return @@ -401,15 +590,26 @@ proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = indexes[index] -= 1 proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} = - ## calculates the next lexicographic permutation, directly modifying ``x``. + ## Calculates the next lexicographic permutation, directly modifying ``x``. ## The result is whether a permutation happened, otherwise we have reached ## the last-ordered permutation. ## - ## .. code-block:: nim + ## If you start with an unsorted array/seq, the repeated permutations + ## will **not** give you all permutations but stop with last. ## - ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - ## v.nextPermutation() - ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] + ## **See also:** + ## * `prevPermutation proc<#prevPermutation,openArray[T]>`_ + runnableExamples: + var v = @[0, 1, 2, 3] + assert v.nextPermutation() == true + assert v == @[0, 1, 3, 2] + assert v.nextPermutation() == true + assert v == @[0, 2, 1, 3] + assert v.prevPermutation() == true + assert v == @[0, 1, 3, 2] + v = @[3, 2, 1, 0] + assert v.nextPermutation() == false + assert v == @[3, 2, 1, 0] if x.len < 2: return false @@ -430,15 +630,20 @@ proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} = result = true proc prevPermutation*[T](x: var openarray[T]): bool {.discardable.} = - ## calculates the previous lexicographic permutation, directly modifying + ## Calculates the previous lexicographic permutation, directly modifying ## ``x``. The result is whether a permutation happened, otherwise we have ## reached the first-ordered permutation. ## - ## .. code-block:: nim - ## - ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] - ## v.prevPermutation() - ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ## **See also:** + ## * `nextPermutation proc<#nextPermutation,openArray[T]>`_ + runnableExamples: + var v = @[0, 1, 2, 3] + assert v.prevPermutation() == false + assert v == @[0, 1, 2, 3] + assert v.nextPermutation() == true + assert v == @[0, 1, 3, 2] + assert v.prevPermutation() == true + assert v == @[0, 1, 2, 3] if x.len < 2: return false @@ -542,7 +747,7 @@ proc rotatedInternal[T](arg: openarray[T]; first, middle, last: int): seq[T] = result[i] = arg[i] proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): int {.discardable.} = - ## performs a left rotation on a range of elements. If you want to rotate + ## Performs a left rotation on a range of elements. If you want to rotate ## right, use a negative ``dist``. Specifically, ``rotateLeft`` rotates ## the elements at ``slice`` by ``dist`` positions. ## @@ -553,6 +758,7 @@ proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): ## ## Elements outside of ``slice`` will be left unchanged. ## The time complexity is linear to ``slice.b - slice.a + 1``. + ## If an invalid range (``HSlice``) is passed, it raises IndexError. ## ## ``slice`` ## The indices of the element range that should be rotated. @@ -561,11 +767,18 @@ proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): ## The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## - ## .. code-block:: nim - ## - ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - ## list.rotateLeft(1 .. 8, 3) - ## doAssert list == [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for a version which rotates the whole container + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which returns a ``seq[T]`` + runnableExamples: + var a = [0, 1, 2, 3, 4, 5] + a.rotateLeft(1 .. 4, 3) + assert a == [0, 4, 1, 2, 3, 5] + a.rotateLeft(1 .. 4, 3) + assert a == [0, 3, 4, 1, 2, 5] + a.rotateLeft(1 .. 4, -3) + assert a == [0, 4, 1, 2, 3, 5] + doAssertRaises(IndexError, a.rotateLeft(1 .. 7, 2)) let sliceLen = slice.b + 1 - slice.a let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen arg.rotateInternal(slice.a, slice.a+distLeft, slice.b + 1) @@ -573,10 +786,18 @@ proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): proc rotateLeft*[T](arg: var openarray[T]; dist: int): int {.discardable.} = ## Default arguments for slice, so that this procedure operates on the entire ## ``arg``, and not just on a part of it. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],HSlice[int,int],int>`_ for a version which rotates a range + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which returns a ``seq[T]`` runnableExamples: - var a = [1, 2, 3, 4, 5] - a.rotateLeft(2) - doAssert a == [3, 4, 5, 1, 2] + var a = [1, 2, 3, 4, 5] + a.rotateLeft(2) + assert a == [3, 4, 5, 1, 2] + a.rotateLeft(4) + assert a == [2, 3, 4, 5, 1] + a.rotateLeft(-6) + assert a == [1, 2, 3, 4, 5] let arglen = arg.len let distLeft = ((dist mod arglen) + arglen) mod arglen arg.rotateInternal(0, distLeft, arglen) @@ -584,6 +805,28 @@ proc rotateLeft*[T](arg: var openarray[T]; dist: int): int {.discardable.} = proc rotatedLeft*[T](arg: openarray[T]; slice: HSlice[int, int], dist: int): seq[T] = ## Same as ``rotateLeft``, just with the difference that it does ## not modify the argument. It creates a new ``seq`` instead. + ## + ## Elements outside of ``slice`` will be left unchanged. + ## If an invalid range (``HSlice``) is passed, it raises IndexError. + ## + ## ``slice`` + ## The indices of the element range that should be rotated. + ## + ## ``dist`` + ## The distance in amount of elements that the data should be rotated. + ## Can be negative, can be any number. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],HSlice[int,int],int>`_ for the in-place version of this proc + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which rotates the whole container + runnableExamples: + var a = @[1, 2, 3, 4, 5] + a = rotatedLeft(a, 1 .. 4, 3) + assert a == @[1, 5, 2, 3, 4] + a = rotatedLeft(a, 1 .. 3, 2) + assert a == @[1, 3, 5, 2, 4] + a = rotatedLeft(a, 1 .. 3, -2) + assert a == @[1, 5, 2, 3, 4] let sliceLen = slice.b + 1 - slice.a let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen arg.rotatedInternal(slice.a, slice.a+distLeft, slice.b+1) @@ -591,6 +834,18 @@ proc rotatedLeft*[T](arg: openarray[T]; slice: HSlice[int, int], dist: int): seq proc rotatedLeft*[T](arg: openarray[T]; dist: int): seq[T] = ## Same as ``rotateLeft``, just with the difference that it does ## not modify the argument. It creates a new ``seq`` instead. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for the in-place version of this proc + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which rotates a range + runnableExamples: + var a = @[1, 2, 3, 4, 5] + a = rotatedLeft(a, 2) + assert a == @[3, 4, 5, 1, 2] + a = rotatedLeft(a, 4) + assert a == @[2, 3, 4, 5, 1] + a = rotatedLeft(a, -6) + assert a == @[1, 2, 3, 4, 5] let arglen = arg.len let distLeft = ((dist mod arglen) + arglen) mod arglen arg.rotatedInternal(0, distLeft, arg.len) From 15ee5d65d16be620cf777d462105d0092a26dda4 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 24 Jan 2019 10:24:58 +0000 Subject: [PATCH 219/350] Add koch run example (#10434) --- doc/contributing.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index e3ab697d39..b90d61270e 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -60,7 +60,7 @@ Compiler -------- The tests for the compiler use a testing tool called ``testament``. They are all -located in ``tests/`` (eg: ``tests/destructor/tdestructor3.nim``). +located in ``tests/`` (e.g.: ``tests/destructor/tdestructor3.nim``). Each test has its own file. All test files are prefixed with ``t``. If you want to create a file for import into another test only, use the prefix ``m``. @@ -115,6 +115,13 @@ list of these, see ``testament/categories.nim``, at the bottom. ./koch tests c lib +To run a single test: + +:: + + ./koch tests c / + +E.g. ``./koch test run stdlib/thttpclient_ssl`` For reproducible tests (to reproduce an environment more similar to the one run by Continuous Integration on travis/appveyor), you may want to disable your From 6555d89c2c7f403a45800903457ee62a62ff327a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Thu, 24 Jan 2019 11:33:44 +0100 Subject: [PATCH 220/350] free some registers after usage, allows more complex macros (#10443) --- compiler/vmgen.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index e7993dfb21..f87821da4f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -326,8 +326,14 @@ proc genWhile(c: PCtx; n: PNode) = c.patch(L2) proc genBlock(c: PCtx; n: PNode; dest: var TDest) = + let oldRegisterCount = c.prc.maxSlots withBlock(n.sons[0].sym): c.gen(n.sons[1], dest) + + for i in oldRegisterCount ..< c.prc.maxSlots: + if c.prc.slots[i].kind in {slotFixedVar, slotFixedLet}: + c.prc.slots[i] = (inUse: false, kind: slotEmpty) + c.clearDest(n, dest) proc genBreak(c: PCtx; n: PNode) = From d72921ecf6b3c62fdc0be2c179752ee08b4169b8 Mon Sep 17 00:00:00 2001 From: alaviss Date: Thu, 24 Jan 2019 18:22:19 +0700 Subject: [PATCH 221/350] compiler/sem: correct macros lineinfo (#10438) The compiler believes these are where a macro call starts (marked with `^`): m "string" ^ ^ This commit addresses that. --- compiler/sem.nim | 9 +++++---- nimsuggest/tests/tmacro_highlight.nim | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 nimsuggest/tests/tmacro_highlight.nim diff --git a/compiler/sem.nim b/compiler/sem.nim index 3763c9b84c..0a3b60ab35 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -450,17 +450,18 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, flags: TExprFlags = {}): PNode = pushInfoContext(c.config, nOrig.info, sym.detailedInfo) - markUsed(c.config, n.info, sym, c.graph.usageSym) - onUse(n.info, sym) + let info = getCallLineInfo(n) + markUsed(c.config, info, sym, c.graph.usageSym) + onUse(info, sym) if sym == c.p.owner: - globalError(c.config, n.info, "recursive dependency: '$1'" % sym.name.s) + globalError(c.config, info, "recursive dependency: '$1'" % sym.name.s) let genericParams = if sfImmediate in sym.flags: 0 else: sym.ast[genericParamsPos].len let suppliedParams = max(n.safeLen - 1, 0) if suppliedParams < genericParams: - globalError(c.config, n.info, errMissingGenericParamsForTemplate % n.renderTree) + globalError(c.config, info, errMissingGenericParamsForTemplate % n.renderTree) #if c.evalContext == nil: # c.evalContext = c.createEvalContext(emStatic) diff --git a/nimsuggest/tests/tmacro_highlight.nim b/nimsuggest/tests/tmacro_highlight.nim new file mode 100644 index 0000000000..6f5b5e8a70 --- /dev/null +++ b/nimsuggest/tests/tmacro_highlight.nim @@ -0,0 +1,13 @@ +macro a(b: string): untyped = discard + +a "string"#[!]# + +discard """ +$nimsuggest --tester $file +>highlight $1 +highlight;;skMacro;;1;;6;;1 +highlight;;skType;;1;;11;;6 +highlight;;skType;;1;;20;;7 +highlight;;skMacro;;3;;0;;1 +highlight;;skMacro;;3;;0;;1 +""" From e75049d7b63d0527b4d66dbf50380a3736571c52 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 24 Jan 2019 12:22:59 +0100 Subject: [PATCH 222/350] fixes #10433 (#10444) --- lib/pure/collections/sequtils.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index b0d50bce26..2533403794 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -846,6 +846,15 @@ template mapIt*(s: typed, op: untyped): untyped = result.add(op) result +template mapIt*(s, typ, op: untyped): untyped {.error: + "Use 'mapIt(seq1, op)' - without specifying the type of the returned seqence".} = + ## **Deprecated since version 0.12.0:** Use the `mapIt(seq1, op) template + ## <#mapIt.t,typed,untyped>`_ instead. + var result: seq[typ] = @[] + for it {.inject.} in items(s): + result.add(op) + result + template applyIt*(varSeq, op: untyped) = ## Convenience template around the mutable ``apply`` proc to reduce typing. ## From 251b014bf4b1cdd0d79511ab4a140db301925d0b Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 24 Jan 2019 14:40:16 +0100 Subject: [PATCH 223/350] fix #9556 (#10445) The old logic wasn't very useful because `relPath` is almost always shorter than `absPath`, e.g. `../../../../../` is shorter than `C:\Program Files`. This way allows the usage of a relative path for at most two levels deep, e.g. `../../relPath`, otherwise the absolute path is used. --- compiler/msgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 698b810615..6bb2c3fa3c 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -203,7 +203,7 @@ proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string = result = absPath else: let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string - result = if absPath.len < relPath.len: absPath else: relPath + result = if relPath.count("..") > 2: absPath else: relPath proc toLinenumber*(info: TLineInfo): int {.inline.} = result = int info.line From bc31eb301a1c06146011af7147750cde83260d8c Mon Sep 17 00:00:00 2001 From: jiro Date: Fri, 25 Jan 2019 01:14:23 +0900 Subject: [PATCH 224/350] better docs: parsecsv (#10391) --- lib/pure/parsecsv.nim | 162 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 22 deletions(-) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 796114d37d..35ed0b3773 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -10,13 +10,18 @@ ## This module implements a simple high performance `CSV`:idx: ## (`comma separated value`:idx:) parser. ## -## Example: How to use the parser -## ============================== +## Basic usage +## =========== ## ## .. code-block:: nim -## import os, parsecsv, streams +## import parsecsv +## from os import paramStr +## from streams import newFileStream +## ## var s = newFileStream(paramStr(1), fmRead) -## if s == nil: quit("cannot open the file" & paramStr(1)) +## if s == nil: +## quit("cannot open the file" & paramStr(1)) +## ## var x: CsvParser ## open(x, s, paramStr(1)) ## while readRow(x): @@ -26,11 +31,11 @@ ## close(x) ## ## For CSV files with a header row, the header can be read and then used as a -## reference for item access with `rowEntry <#rowEntry.CsvParser.string>`_: +## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: ## ## .. code-block:: nim ## import parsecsv -## import os +## ## # Prepare a file ## let content = """One,Two,Three,Four ## 1,2,3,4 @@ -47,24 +52,40 @@ ## for col in items(p.headers): ## echo "##", col, ":", p.rowEntry(col), "##" ## p.close() +## +## See also +## ======== +## +## * `streams module `_ for using +## `open proc <#open,CsvParser,Stream,string,Char,Char,Char>`_ +## and other stream processing (like `close proc `_) +## * `parseopt module `_ for a command line parser +## * `parsecfg module `_ for a configuration file parser +## * `parsexml module `_ for a XML / HTML parser +## * `parsesql module `_ for a SQL parser +## * `other parsers `_ for other parsers import lexbase, streams type - CsvRow* = seq[string] ## a row in a CSV file - CsvParser* = object of BaseLexer ## the parser object. - row*: CsvRow ## the current row + CsvRow* = seq[string] ## A row in a CSV file. + CsvParser* = object of BaseLexer ## The parser object. + ## + ## It consists of two public fields: + ## * `row` is the current row + ## * `headers` are the columns that are defined in the csv file + ## (read using `readHeaderRow <#readHeaderRow,CsvParser>`_). + ## Used with `rowEntry <#rowEntry,CsvParser,string>`_). + row*: CsvRow filename: string sep, quote, esc: char skipWhite: bool currRow: int - headers*: seq[string] ## The columns that are defined in the csv file - ## (read using `readHeaderRow <#readHeaderRow.CsvParser>`_). - ## Used with `rowEntry <#rowEntry.CsvParser.string>`_). + headers*: seq[string] - CsvError* = object of IOError ## exception that is raised if - ## a parsing error occurs + CsvError* = object of IOError ## An exception that is raised if + ## a parsing error occurs. proc raiseEInvalidCsv(filename: string, line, col: int, msg: string) {.noreturn.} = @@ -82,7 +103,7 @@ proc error(my: CsvParser, pos: int, msg: string) = proc open*(my: var CsvParser, input: Stream, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = - ## initializes the parser with an input stream. `Filename` is only used + ## Initializes the parser with an input stream. `Filename` is only used ## for nice error messages. The parser's behaviour can be controlled by ## the diverse optional parameters: ## - `separator`: character used to separate fields @@ -94,6 +115,18 @@ proc open*(my: var CsvParser, input: Stream, filename: string, ## two `quote` characters are parsed one literal `quote` character. ## - `skipInitialSpace`: If true, whitespace immediately following the ## `separator` is ignored. + ## + ## See also: + ## * `open proc <#open,CsvParser,string,Char,Char,Char>`_ which creates the + ## file stream for you + runnableExamples: + import streams + var strm = newStringStream("One,Two,Three\n1,2,3\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + parser.close() + strm.close() + lexbase.open(my, input) my.filename = filename my.sep = separator @@ -106,7 +139,16 @@ proc open*(my: var CsvParser, input: Stream, filename: string, proc open*(my: var CsvParser, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = - ## same as the other `open` but creates the file stream for you. + ## Similar to the `other open proc<#open,CsvParser,Stream,string,Char,Char,Char>`_, + ## but creates the file stream for you. + runnableExamples: + from os import removeFile + writeFile("tmp.csv", "One,Two,Three\n1,2,3\n10,20,300") + var parser: CsvParser + parser.open("tmp.csv") + parser.close() + removeFile("tmp.csv") + var s = newFileStream(filename, fmRead) if s == nil: my.error(0, "cannot open: " & filename) open(my, s, filename, separator, @@ -159,15 +201,58 @@ proc parseField(my: var CsvParser, a: var string) = my.bufpos = pos proc processedRows*(my: var CsvParser): int = - ## returns number of the processed rows + ## Returns number of the processed rows. + ## + ## But even if `readRow <#readRow,CsvParser,int>`_ arrived at EOF then + ## processed rows counter is incremented. + runnableExamples: + import streams + + var strm = newStringStream("One,Two,Three\n1,2,3") + var parser: CsvParser + parser.open(strm, "tmp.csv") + doAssert parser.readRow() + doAssert parser.processedRows() == 1 + doAssert parser.readRow() + doAssert parser.processedRows() == 2 + ## Even if `readRow` arrived at EOF then `processedRows` is incremented. + doAssert parser.readRow() == false + doAssert parser.processedRows() == 3 + doAssert parser.readRow() == false + doAssert parser.processedRows() == 4 + parser.close() + strm.close() + return my.currRow proc readRow*(my: var CsvParser, columns = 0): bool = - ## reads the next row; if `columns` > 0, it expects the row to have + ## Reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. ## ## Blank lines are skipped. + runnableExamples: + import streams + var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + doAssert parser.readRow() + doAssert parser.row == @["One", "Two", "Three"] + doAssert parser.readRow() + doAssert parser.row == @["1", "2", "3"] + ## Blank lines are skipped. + doAssert parser.readRow() + doAssert parser.row == @["10", "20", "30"] + + var emptySeq: seq[string] + doAssert parser.readRow() == false + doAssert parser.row == emptySeq + doAssert parser.readRow() == false + doAssert parser.row == emptySeq + + parser.close() + strm.close() + var col = 0 # current column let oldpos = my.bufpos while my.buf[my.bufpos] != '\0': @@ -200,12 +285,31 @@ proc readRow*(my: var CsvParser, columns = 0): bool = inc(my.currRow) proc close*(my: var CsvParser) {.inline.} = - ## closes the parser `my` and its associated input stream. + ## Closes the parser `my` and its associated input stream. lexbase.close(my) proc readHeaderRow*(my: var CsvParser) = ## Reads the first row and creates a look-up table for column numbers - ## See also `rowEntry <#rowEntry.CsvParser.string>`_. + ## See also: + ## * `rowEntry proc <#rowEntry,CsvParser,string>`_ + runnableExamples: + import streams + + var strm = newStringStream("One,Two,Three\n1,2,3") + var parser: CsvParser + parser.open(strm, "tmp.csv") + + parser.readHeaderRow() + doAssert parser.headers == @["One", "Two", "Three"] + doAssert parser.row == @["One", "Two", "Three"] + + doAssert parser.readRow() + doAssert parser.headers == @["One", "Two", "Three"] + doAssert parser.row == @["1", "2", "3"] + + parser.close() + strm.close() + let present = my.readRow() if present: my.headers = my.row @@ -213,8 +317,23 @@ proc readHeaderRow*(my: var CsvParser) = proc rowEntry*(my: var CsvParser, entry: string): var string = ## Acceses a specified `entry` from the current row. ## - ## Assumes that `readHeaderRow <#readHeaderRow.CsvParser>`_ has already been + ## Assumes that `readHeaderRow <#readHeaderRow,CsvParser>`_ has already been ## called. + runnableExamples: + import streams + var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + ## Need calling `readHeaderRow`. + parser.readHeaderRow() + doAssert parser.readRow() + doAssert parser.rowEntry("One") == "1" + doAssert parser.rowEntry("Two") == "2" + doAssert parser.rowEntry("Three") == "3" + ## `parser.rowEntry("NotExistEntry")` causes SIGSEGV fault. + parser.close() + strm.close() + let index = my.headers.find(entry) if index >= 0: result = my.row[index] @@ -262,4 +381,3 @@ when isMainModule: # Tidy up removeFile("temp.csv") - From baf72691cb9fa63d22692003b9a59d5581a1dd31 Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 24 Jan 2019 16:47:33 +0100 Subject: [PATCH 225/350] include missing modules in `lib.rst` --- doc/lib.rst | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 1f19f9bf42..082c3c0bd1 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -67,6 +67,8 @@ Core * `lenientops `_ Provides binary operators for mixed integer/float expressions for convenience. +* `bitops `_ + Provides a series of low level methods for bit manipulation. Collections and algorithms @@ -74,26 +76,38 @@ Collections and algorithms * `algorithm `_ Implements some common generic algorithms like sort or binary search. + * `tables `_ Nim hash table support. Contains tables, ordered tables and count tables. + * `sets `_ Nim hash and bit set support. + * `lists `_ Nim linked list support. Contains singly and doubly linked lists and circular lists ("rings"). + * `deques `_ Implementation of a double-ended queue. The underlying implementation uses a ``seq``. + +* `heapqueue `_ + This module implements Heap queue (a.k.a. priority queue) algorithm. + * `intsets `_ Efficient implementation of a set of ints as a sparse bit set. + * `critbits `_ This module implements a *crit bit tree* which is an efficient container for a sorted set of strings, or for a sorted mapping of strings. + * `sequtils `_ This module implements operations for the built-in seq type which were inspired by functional programming languages. + * `sharedtables `_ Nim shared hash table support. Contains shared tables. + * `sharedlist `_ Nim shared linked list support. Contains shared singly linked list. @@ -129,6 +143,13 @@ String handling * `unicode `_ This module provides support to handle the Unicode UTF-8 encoding. +* `unidecode `_ + It provides a single proc that does Unicode to ASCII transliterations. + Based on Python's Unidecode module. + +* `punycode `_ + Implements a representation of Unicode with the limited ASCII character subset. + * `encodings `_ Converts between different character encodings. On UNIX, this uses the ``iconv`` library, on Windows the Windows API. @@ -194,10 +215,16 @@ Generic Operating System Services ``asyncdispatch``. * `distros `_ - This module implements the basics for OS distribution ("distro") detection and the OS's native package manager. - Its primary purpose is to produce output for Nimble packages, but it also contains the widely used **Distribution** enum + This module implements the basics for OS distribution ("distro") detection + and the OS's native package manager. + Its primary purpose is to produce output for Nimble packages, + but it also contains the widely used **Distribution** enum that is useful for writing platform specific code. +* `volatile `_ + This module contains code for generating volatile loads and stores, + which are useful in embedded and systems programming. + Math libraries -------------- @@ -276,6 +303,7 @@ Internet Protocols and Support This module implements a selector API with backends specific to each OS. Currently epoll on Linux and select on other operating systems. + Parsers ------- @@ -402,6 +430,7 @@ Miscellaneous * `segfaults `_ Turns access violations or segfaults into a ``NilAccessError`` exception. + Modules for JS backend ---------------------- @@ -430,7 +459,6 @@ Regular expressions expressions. The current implementation uses PCRE. - Database support ---------------- @@ -447,13 +475,13 @@ Database support for other databases too. - Wrappers ======== The generated HTML for some of these wrappers is so huge that it is not contained in the distribution. You can then find them on the website. + Windows specific ---------------- @@ -509,5 +537,5 @@ Nimble is a package manager for the Nim programming language. For instructions on how to install Nimble packages see `its README `_. -To see a list of Nimble's packages, check out `https://nimble.directory/ `_ +To see a list of Nimble's packages, check out ``_ or the `packages repo `_ on GitHub. From 44923a1b1f158732b13fb938c94c0b02a2463e6d Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 25 Jan 2019 08:34:02 +0100 Subject: [PATCH 226/350] fix #10287 (#10448) The only thing separating these two files (which wrongly appear on index.html as modules) and the rest of the (correctly placed) documents was the style of the title :D --- doc/contributing.rst | 6 +++++- doc/packaging.rst | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index b90d61270e..c67db081d4 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -1,6 +1,10 @@ +============ Contributing ============ +.. contents:: + + Contributing happens via "Pull requests" (PR) on github. Every PR needs to be reviewed before it can be merged and the Continuous Integration should be green. @@ -120,7 +124,7 @@ To run a single test: :: ./koch tests c / - + E.g. ``./koch test run stdlib/thttpclient_ssl`` For reproducible tests (to reproduce an environment more similar to the one diff --git a/doc/packaging.rst b/doc/packaging.rst index f67cb1ccee..efbffeba56 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -1,3 +1,4 @@ +============= Packaging Nim ============= From 46d5fb9ed72f7d3a6dd504898a95c7fb286f7856 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 24 Jan 2019 08:23:24 +0100 Subject: [PATCH 227/350] docgen: nicer indentation --- compiler/docgen.nim | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 4271a116ef..a2166784b7 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -144,28 +144,28 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, initStrTable result.types result.onTestSnippet = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = - var d = TDocumentor(gen) - var outp: AbsoluteFile - if filename.len == 0: - inc(d.id) - let nameOnly = splitFile(d.filename).name - let subdir = getNimcacheDir(conf) / RelativeDir(nameOnly) - createDir(subdir) - outp = subdir / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim") - elif isAbsolute(filename): - outp = AbsoluteFile filename - else: - # Nim's convention: every path is relative to the file it was written in: - outp = splitFile(d.filename).dir.AbsoluteDir / RelativeFile(filename) - # Include the current file if we're parsing a nim file - let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename] - writeFile(outp, importStmt & content) - let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3) - else: cmd - let c2 = c % quoteShell(outp) - rawMessage(conf, hintExecuting, c2) - if execShellCmd(c2) != status: - rawMessage(conf, errGenerated, "executing of external program failed: " & c2) + var d = TDocumentor(gen) + var outp: AbsoluteFile + if filename.len == 0: + inc(d.id) + let nameOnly = splitFile(d.filename).name + let subdir = getNimcacheDir(conf) / RelativeDir(nameOnly) + createDir(subdir) + outp = subdir / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim") + elif isAbsolute(filename): + outp = AbsoluteFile filename + else: + # Nim's convention: every path is relative to the file it was written in: + outp = splitFile(d.filename).dir.AbsoluteDir / RelativeFile(filename) + # Include the current file if we're parsing a nim file + let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename] + writeFile(outp, importStmt & content) + let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3) + else: cmd + let c2 = c % quoteShell(outp) + rawMessage(conf, hintExecuting, c2) + if execShellCmd(c2) != status: + rawMessage(conf, errGenerated, "executing of external program failed: " & c2) result.emitted = initIntSet() result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath), outExt, RelativeDir"htmldocs", false) From 13d667ee5312a4705a7ca51ead933739514c70dd Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 25 Jan 2019 09:11:06 +0100 Subject: [PATCH 228/350] docgen: produce links for proc names --- compiler/docgen.nim | 22 ++++++++++++++----- nimdoc/tester.nim | 3 ++- .../expected/subdir/subdir_b/utils.html | 6 ++--- nimdoc/testproject/expected/testproject.html | 10 ++++----- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index a2166784b7..f34a831454 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -300,13 +300,17 @@ proc externalDep(d: PDoc; module: PSym): string = else: result = extractFilename toFullPath(d.conf, FileIndex module.position) -proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) = +proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}; + procLink: Rope) = var r: TSrcGen var literal = "" initTokRender(r, n, renderFlags) var kind = tkEof + var tokenPos = 0 + var procTokenPos = 0 while true: getNextTok(r, kind, literal) + inc tokenPos case kind of tkEof: break @@ -314,6 +318,8 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe dispA(d.conf, result, "$1", "\\spanComment{$1}", [rope(esc(d.target, literal))]) of tokKeywordLow..tokKeywordHigh: + if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}: + procTokenPos = tokenPos dispA(d.conf, result, "$1", "\\spanKeyword{$1}", [rope(literal)]) of tkOpr: @@ -333,7 +339,11 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))]) of tkSymbol: let s = getTokSym(r) - if s != nil and s.kind == skType and sfExported in s.flags and + # -2 because of the whitespace in between: + if procTokenPos == tokenPos-2 and procLink != nil: + dispA(d.conf, result, "$1", + "\\spanIdentifier{$1}", [rope(esc(d.target, literal)), procLink]) + elif s != nil and s.kind == skType and sfExported in s.flags and s.owner != nil and belongsToPackage(d.conf, s.owner) and d.target == outHtml: let external = externalDep(d, s.owner) @@ -445,7 +455,7 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) = for b in body: if i > 0: dest.add "\n" inc i - nodeToHighlightedHtml(d, b, dest, {}) + nodeToHighlightedHtml(d, b, dest, {}, nil) dest.add(d.config.getOrDefault"doc.listing_end" % id) else: discard for i in 0 ..< n.safeLen: @@ -615,9 +625,6 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = break plainName.add(literal) - nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, - renderDocComments, renderSyms}) - inc(d.id) let plainNameRope = rope(xmltree.escape(plainName.strip)) @@ -630,6 +637,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = symbolOrIdRope = symbolOrId.rope symbolOrIdEncRope = encodeUrl(symbolOrId).rope + nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, + renderDocComments, renderSyms}, symbolOrIdEncRope) + var seeSrcRope: Rope = nil let docItemSeeSrc = getConfigVar(d.conf, "doc.item.seesrc") if docItemSeeSrc.len > 0: diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index db20951283..997ed85962 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -26,7 +26,8 @@ proc test(dir: string; fixup = false) = copyFile(produced, expected) else: echo "SUCCESS: files identical: ", produced - removeDir(dir / "htmldocs") + if failures == 0: + removeDir(dir / "htmldocs") test("nimdoc/testproject", defined(fixup)) if failures > 0: quit($failures & " failures occurred.") diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 9ba3c24e15..5d6dbbc5b4 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1305,7 +1305,7 @@ function main() {

        Procs

        -
        proc someType(): SomeType {...}{.raises: [], tags: [].}
        +
        proc someType(): SomeType {...}{.raises: [], tags: [].}
        constructor. @@ -1316,13 +1316,13 @@ constructor.

        Templates

        -
        template aEnum(): untyped
        +
        template aEnum(): untyped
        -
        template bEnum(): untyped
        +
        template bEnum(): untyped
        diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index dd1541a56f..9c13eb6e69 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1350,13 +1350,13 @@ The enum B.

        Procs

        -
        proc bar[T](a, b: T): T
        +
        proc bar[T](a, b: T): T
        -
        proc isValid[T](x: T): bool
        +
        proc isValid[T](x: T): bool
        @@ -1367,7 +1367,7 @@ The enum B.

        Funcs

        -
        func someFunc() {...}{.raises: [], tags: [].}
        +
        func someFunc() {...}{.raises: [], tags: [].}
        My someFunc. Stuff in quotes here. Some link @@ -1378,7 +1378,7 @@ My someFunc. Stuff in quotesMacros
        -
        macro bar(): untyped
        +
        macro bar(): untyped
        @@ -1389,7 +1389,7 @@ My someFunc. Stuff in quotesTemplates
        -
        template foo(a, b: SomeType)
        +
        template foo(a, b: SomeType)
        This does nothing From 3de255541a978f2f51ffdc4e49e0876d61cec128 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 25 Jan 2019 09:21:35 +0100 Subject: [PATCH 229/350] docgen: do not produce trailing commas for links --- compiler/docgen.nim | 17 ++++++++++------- .../expected/subdir/subdir_b/utils.html | 18 +++++++++--------- nimdoc/testproject/expected/testproject.html | 12 ++++++------ nimdoc/testproject/expected/theindex.html | 10 +++++----- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f34a831454..207d7e30b9 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -559,15 +559,18 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string = ## section of ``doc/docgen.txt``. result = baseName case k: - of skProc, skFunc: result.add(defaultParamSeparator) - of skMacro: result.add(".m" & defaultParamSeparator) - of skMethod: result.add(".e" & defaultParamSeparator) - of skIterator: result.add(".i" & defaultParamSeparator) - of skTemplate: result.add(".t" & defaultParamSeparator) - of skConverter: result.add(".c" & defaultParamSeparator) + of skProc, skFunc: discard + of skMacro: result.add(".m") + of skMethod: result.add(".e") + of skIterator: result.add(".i") + of skTemplate: result.add(".t") + of skConverter: result.add(".c") else: discard if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams: - result.add(renderParamTypes(n[paramsPos])) + let params = renderParamTypes(n[paramsPos]) + if params.len > 0: + result.add(defaultParamSeparator) + result.add(params) proc isCallable(n: PNode): bool = ## Returns true if `n` contains a callable node. diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 5d6dbbc5b4..f057df7e36 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1255,7 +1255,7 @@ function main() {
      • Procs @@ -1263,9 +1263,9 @@ function main() {
      • Templates @@ -1304,8 +1304,8 @@ function main() {

        Procs

        - -
        proc someType(): SomeType {...}{.raises: [], tags: [].}
        + +
        proc someType(): SomeType {...}{.raises: [], tags: [].}
        constructor. @@ -1315,14 +1315,14 @@ constructor.

        Templates

        - -
        template aEnum(): untyped
        + +
        template aEnum(): untyped
        - -
        template bEnum(): untyped
        + +
        template bEnum(): untyped
        diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 9c13eb6e69..21a7a0cf83 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1277,7 +1277,7 @@ function main() {
      • Funcs @@ -1285,7 +1285,7 @@ function main() {
      • Macros @@ -1366,8 +1366,8 @@ The enum B.

        Funcs

        - -
        func someFunc() {...}{.raises: [], tags: [].}
        + +
        func someFunc() {...}{.raises: [], tags: [].}
        My someFunc. Stuff in quotes here. Some link @@ -1377,8 +1377,8 @@ My someFunc. Stuff in quotes

        Macros

        - -
        macro bar(): untyped
        + +
        macro bar(): untyped
        diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 95c667acca..5a6e575bf5 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1227,7 +1227,7 @@ function main() {
      aEnum:
      • utils: aEnum(): untyped
        • + data-doc-search-tag="utils: aEnum(): untyped" href="subdir/subdir_b/utils.html#aEnum.t">utils: aEnum(): untyped
      aVariable:
      bEnum:
      • utils: bEnum(): untyped
        • + data-doc-search-tag="utils: bEnum(): untyped" href="subdir/subdir_b/utils.html#bEnum.t">utils: bEnum(): untyped
      enumValueA:
      someFunc:
      • testproject: someFunc()
        • + data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()
      SomeType:
      someType:
      • utils: someType(): SomeType
        • + data-doc-search-tag="utils: someType(): SomeType" href="subdir/subdir_b/utils.html#someType_2">utils: someType(): SomeType
      From 43613ff49dcbfb171fd14786dfb0fb3845b8692f Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 25 Jan 2019 09:22:31 +0100 Subject: [PATCH 230/350] docgen: style tweak --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 207d7e30b9..a7f7d77b5c 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -558,7 +558,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string = ## If you modify the output of this proc, please update the anchor generation ## section of ``doc/docgen.txt``. result = baseName - case k: + case k of skProc, skFunc: discard of skMacro: result.add(".m") of skMethod: result.add(".e") From 1c2abb8d880b8a7553251891e816617f060b6b69 Mon Sep 17 00:00:00 2001 From: cooldome Date: Fri, 25 Jan 2019 08:25:11 +0000 Subject: [PATCH 231/350] fixes double object field symbol lookups (no test case available) (#10450) --- compiler/semexprs.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 37e2ca33f8..68f1c6c3a7 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1291,7 +1291,11 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.sons[0] == nil: break ty = skipTypes(ty.sons[0], skipPtrs) if f != nil: - if fieldVisible(c, f): + let visibilityCheckNeeded = + if n[1].kind == nkSym and n[1].sym == f: + false # field lookup was done already, likely by hygienic template or bindSym + else: true + if not visibilityCheckNeeded or fieldVisible(c, f): # is the access to a public field or in the same module or in a friend? markUsed(c.config, n.sons[1].info, f, c.graph.usageSym) onUse(n.sons[1].info, f) From 8313e5f24e46d65701decff65e9470c8b0521d82 Mon Sep 17 00:00:00 2001 From: Christian Lyder Jacobsen Date: Fri, 25 Jan 2019 09:29:32 +0100 Subject: [PATCH 232/350] Use hexadecimal in json escaping (#10437) [backport] --- lib/pure/json.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index cf979e2d0c..ab7cabed69 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -625,7 +625,7 @@ proc escapeJsonUnquoted*(s: string; result: var string) = of '\r': result.add("\\r") of '"': result.add("\\\"") of '\0'..'\7': result.add("\\u000" & $ord(c)) - of '\14'..'\31': result.add("\\u00" & $ord(c)) + of '\14'..'\31': result.add("\\u00" & toHex(ord(c), 2)) of '\\': result.add("\\\\") else: result.add(c) @@ -1686,9 +1686,9 @@ when isMainModule: doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") doAssert escapeJsonUnquoted("\10Foo🎃barÄ") == "\\nFoo🎃barÄ" - doAssert escapeJsonUnquoted("\0\7\20") == "\\u0000\\u0007\\u0020" # for #7887 + doAssert escapeJsonUnquoted("\0\7\20") == "\\u0000\\u0007\\u0014" # for #7887 doAssert escapeJson("\10Foo🎃barÄ") == "\"\\nFoo🎃barÄ\"" - doAssert escapeJson("\0\7\20") == "\"\\u0000\\u0007\\u0020\"" # for #7887 + doAssert escapeJson("\0\7\20") == "\"\\u0000\\u0007\\u0014\"" # for #7887 # Test with extra data when not defined(js): From 83caa58c7aab3e32e4c6a30559515b15683b8b80 Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 25 Jan 2019 09:43:47 +0100 Subject: [PATCH 233/350] docs: fix trailing comma in links --- doc/docgen.rst | 2 +- lib/pure/collections/intsets.nim | 2 +- lib/pure/collections/lists.nim | 8 ++++---- lib/pure/uri.nim | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/docgen.rst b/doc/docgen.rst index e6604d3bd1..84969d8ee4 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -283,7 +283,7 @@ symbols in the `system module `_. * ``const NimVersion = "0.0.0"`` **=>** `#NimVersion `_ * ``proc getTotalMem(): int {.rtl, raises: [], tags: [].}`` **=>** - `#getTotalMem, `_ + `#getTotalMem, `_ * ``proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}`` **=>** `#len,seq[T] `_ * ``iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}`` **=>** diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 654b297f62..226401b926 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -363,7 +363,7 @@ proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil and x.elems == 0 proc assign*(dest: var IntSet, src: IntSet) = ## Copies `src` to `dest`. - ## `dest` does not need to be initialized by `initIntSet proc <#initIntSet,>`_. + ## `dest` does not need to be initialized by `initIntSet proc <#initIntSet>`_. runnableExamples: var a = initIntSet() diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 182eb8442d..1fd32c9fad 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -93,25 +93,25 @@ type SinglyLinkedList*[T] = object ## A singly linked list. ## - ## Use `initSinglyLinkedList proc <#initSinglyLinkedList,>`_ to create + ## Use `initSinglyLinkedList proc <#initSinglyLinkedList>`_ to create ## a new empty list. head*, tail*: SinglyLinkedNode[T] DoublyLinkedList*[T] = object ## A doubly linked list. ## - ## Use `initDoublyLinkedList proc <#initDoublyLinkedList,>`_ to create + ## Use `initDoublyLinkedList proc <#initDoublyLinkedList>`_ to create ## a new empty list. head*, tail*: DoublyLinkedNode[T] SinglyLinkedRing*[T] = object ## A singly linked ring. ## - ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing,>`_ to create + ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing>`_ to create ## a new empty ring. head*, tail*: SinglyLinkedNode[T] DoublyLinkedRing*[T] = object ## A doubly linked ring. ## - ## Use `initDoublyLinkedRing proc <#initDoublyLinkedRing,>`_ to create + ## Use `initDoublyLinkedRing proc <#initDoublyLinkedRing>`_ to create ## a new empty ring. head*: DoublyLinkedNode[T] diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index bfb411fc84..c8350dfad4 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -189,7 +189,7 @@ proc parseUri*(uri: string, result: var Uri) = ## ## **See also:** ## * `Uri type <#Uri>`_ for available fields in the URI type - ## * `initUri proc <#initUri,>`_ for initializing a URI + ## * `initUri proc <#initUri>`_ for initializing a URI runnableExamples: var res = initUri() parseUri("https://nim-lang.org/docs/manual.html", res) From 70c7348d3e502fa718fb56848fca8df608a085f7 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 25 Jan 2019 09:49:01 +0100 Subject: [PATCH 234/350] gc:regions: undo the regression introducing changes --- lib/system/gc_regions.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 3b908fb08c..59f68918fd 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -257,14 +257,14 @@ proc deallocAll*() = tlRegion.deallocAll() proc deallocOsPages(r: var MemRegion) = r.deallocAll() -template withScratchRegion*(body: untyped) = +when false: let obs = obstackPtr() try: body finally: setObstackPtr(obs) -when false: +template withScratchRegion*(body: untyped) = var scratch: MemRegion let oldRegion = tlRegion tlRegion = scratch From 87fce94d65296c8aab3b8103c3b7cf48c097e1d2 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Fri, 25 Jan 2019 10:29:46 +0000 Subject: [PATCH 235/350] Add "cmd" description, minor improvements (#10436) --- doc/contributing.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index c67db081d4..b89102a3b5 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -71,9 +71,10 @@ to create a file for import into another test only, use the prefix ``m``. At the beginning of every test is the expected behavior of the test. Possible keys are: -- output: The expected output, most likely via ``echo`` +- cmd: A compilation command template e.g. "nim $target --threads:on $options $file" +- output: The expected output (stdout + stderr), most likely via ``echo`` - exitcode: Exit code of the test (via ``exit(number)``) -- errormsg: The expected error message +- errormsg: The expected compiler error message - file: The file the errormsg was produced at - line: The line the errormsg was produced at From 268197add8e9fbcbf15d6a042c8095828c29869f Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 25 Jan 2019 13:13:55 +0100 Subject: [PATCH 236/350] properly deprecate parseopt2 (#10452) --- lib/{ => deprecated}/pure/parseopt2.nim | 0 tests/ccgbugs/tmangle_field.nim | 4 ++-- tests/misc/tparseopt.nim | 30 ++----------------------- tests/system/tparams.nim | 4 ++-- 4 files changed, 6 insertions(+), 32 deletions(-) rename lib/{ => deprecated}/pure/parseopt2.nim (100%) diff --git a/lib/pure/parseopt2.nim b/lib/deprecated/pure/parseopt2.nim similarity index 100% rename from lib/pure/parseopt2.nim rename to lib/deprecated/pure/parseopt2.nim diff --git a/tests/ccgbugs/tmangle_field.nim b/tests/ccgbugs/tmangle_field.nim index 9e4012b8b1..da2720aaa8 100644 --- a/tests/ccgbugs/tmangle_field.nim +++ b/tests/ccgbugs/tmangle_field.nim @@ -3,7 +3,7 @@ discard """ # bug #5404 -import parseopt2 +import parseopt {.emit: """typedef struct { int key; @@ -12,5 +12,5 @@ import parseopt2 type foo* {.importc: "foo", nodecl.} = object key* {.importc: "key".}: cint -for kind, key, value in parseopt2.getopt(): +for kind, key, value in parseopt.getopt(): discard diff --git a/tests/misc/tparseopt.nim b/tests/misc/tparseopt.nim index 7d5071c3ec..b5da6b572a 100644 --- a/tests/misc/tparseopt.nim +++ b/tests/misc/tparseopt.nim @@ -21,14 +21,7 @@ kind: cmdShortOption key:val -- r:1 kind: cmdShortOption key:val -- r:0 kind: cmdShortOption key:val -- l: kind: cmdShortOption key:val -- r:4 -parseopt2 -first round -kind: cmdLongOption key:val -- left: -second round -kind: cmdLongOption key:val -- left: -kind: cmdLongOption key:val -- debug:3 -kind: cmdShortOption key:val -- l:4 -kind: cmdShortOption key:val -- r:2''' +''' joinable: false """ @@ -42,7 +35,6 @@ when defined(testament_tparseopt): main() else: from parseopt import nil - from parseopt2 import nil block: echo "parseopt" @@ -76,28 +68,11 @@ else: for kind, key, val in parseopt.getopt(p): echo "kind: ", kind, "\tkey:val -- ", key, ":", val - block: - echo "parseopt2" - for kind, key, val in parseopt2.getopt(): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val - - # pass custom cmdline arguments - echo "first round" - var argv: seq[string] = @["--left", "--debug:3", "-l=4", "-r:2"] - var p = parseopt2.initOptParser(argv) - for kind, key, val in parseopt2.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val - break - # reset getopt iterator and check arguments are returned correctly. - echo "second round" - for kind, key, val in parseopt2.getopt(p): - echo "kind: ", kind, "\tkey:val -- ", key, ":", val - import osproc, os, strutils from stdtest/specialpaths import buildDir import "../.." / compiler/unittest_light - block: # fix #9951 (and make it work for parseopt and parseopt2) + block: # fix #9951 template runTest(parseoptCustom) = var p = parseoptCustom.initOptParser(@["echo \"quoted\""]) let expected = when defined(windows): @@ -117,7 +92,6 @@ else: doAssert "a5'b" == "a5\'b" assertEquals parseoptCustom.cmdLineRest(p2), expected2 runTest(parseopt) - runTest(parseopt2) block: # fix #9842 let exe = buildDir / "D20190112T145450".addFileExt(ExeExt) diff --git a/tests/system/tparams.nim b/tests/system/tparams.nim index dcd620b20e..b20cfce1e1 100644 --- a/tests/system/tparams.nim +++ b/tests/system/tparams.nim @@ -5,7 +5,7 @@ joinable: false # not joinable because it executes itself with parameters import os import osproc -import parseopt2 +import parseopt import sequtils let argv = commandLineParams() @@ -17,6 +17,6 @@ else: let f = toSeq(getopt()) doAssert f[0].kind == cmdArgument and f[0].key == "foo bar" and f[0].val == "" doAssert f[1].kind == cmdLongOption and f[1].key == "aa" and f[1].val == "bar=a" - doAssert f[2].kind == cmdLongOption and f[2].key == "a=c" and f[2].val == "d" + doAssert f[2].kind == cmdLongOption and f[2].key == "a" and f[2].val == "c:d" doAssert f[3].kind == cmdLongOption and f[3].key == "ab" and f[3].val == "" doAssert f[4].kind == cmdShortOption and f[4].key == "c" and f[4].val == "" From 4d230db5abb630db71001e7f70b8a5bbdf6a3e7e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 25 Jan 2019 10:28:03 -0800 Subject: [PATCH 237/350] closes #8610 ; adds test case (#10454) --- tests/errmsgs/t8610.nim | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/errmsgs/t8610.nim diff --git a/tests/errmsgs/t8610.nim b/tests/errmsgs/t8610.nim new file mode 100644 index 0000000000..dd1a3ed292 --- /dev/null +++ b/tests/errmsgs/t8610.nim @@ -0,0 +1,5 @@ +discard """ + errmsg: "'typedesc' metatype is not valid here; typed '=' instead of ':'?" +""" +## issue #8610 +const Foo = int From 68dd9be0f527470a9166d22fff01ae7a4152ca70 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 26 Jan 2019 09:27:08 +0000 Subject: [PATCH 238/350] Handle empty lines in parsecsv; fixes #8365 (#9343) --- lib/pure/parsecsv.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 35ed0b3773..e0c4f38a4f 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -255,6 +255,12 @@ proc readRow*(my: var CsvParser, columns = 0): bool = var col = 0 # current column let oldpos = my.bufpos + # skip initial empty lines #8365 + while true: + case my.buf[my.bufpos] + of '\c': my.bufpos = handleCR(my, my.bufpos) + of '\l': my.bufpos = handleLF(my, my.bufpos) + else: break while my.buf[my.bufpos] != '\0': let oldlen = my.row.len if oldlen < col+1: @@ -354,7 +360,7 @@ when isMainModule: import os import strutils block: # Tests for reading the header row - let content = "One,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" + let content = "\nOne,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" writeFile("temp.csv", content) var p: CsvParser From 3c106541faf61b6ab370d2d6b176bb19642f17d3 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Sat, 26 Jan 2019 04:26:36 -0700 Subject: [PATCH 239/350] Increase verbosity of logGC (#10449) * Increase verbosity of logGC * Remove & c.repr to avoid re-entering the GC --- doc/nimc.rst | 1 + lib/system/gc.nim | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index 4ffb595c0e..7c07e09d8e 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -362,6 +362,7 @@ Define Effect ``useRealtimeGC`` Enables support of Nim's GC for *soft* realtime systems. See the documentation of the `gc `_ for further information. +``logGC`` Enable GC logging to stdout. ``nodejs`` The JS target is actually ``node.js``. ``ssl`` Enables OpenSSL support for the sockets module. ``memProfiler`` Enables memory profiling for the native GC. diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 9ae388aa90..416827f21f 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -105,6 +105,8 @@ template gcAssert(cond: bool, msg: string) = when defined(useGcAssert): if not cond: echo "[GCASSERT] ", msg + when defined(logGC): + echo "[GCASSERT] statistics:\L", GC_getStatistics() GC_disable() writeStackTrace() #var x: ptr int @@ -159,6 +161,10 @@ when defined(logGC): c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; thread=%ld\n", msg, c, kind, typName, c.refcount shr rcShift, gch.gcThreadId) +template logCell(msg: cstring, c: PCell) = + when defined(logGC): + writeCell(msg, c) + template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) @@ -174,7 +180,7 @@ proc incRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") c.refcount = c.refcount +% rcIncrement # and not colorMask - #writeCell("incRef", c) + logCell("incRef", c) proc nimGCref(p: pointer) {.compilerProc.} = # we keep it from being collected by pretending it's not even allocated: @@ -192,6 +198,7 @@ proc decRef(c: PCell) {.inline.} = c.refcount = c.refcount -% rcIncrement if c.refcount <% rcIncrement: rtlAddZCT(c) + logCell("decRef", c) proc nimGCunref(p: pointer) {.compilerProc.} = let cell = usrToCell(p) @@ -410,7 +417,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") # its refcount is zero, so add it to the ZCT: addNewObjToZCT(res, gch) - when logGC: writeCell("new cell", res) + logCell("new cell", res) track("rawNewObj", res, size) gcTrace(res, csAllocated) when useCellIds: @@ -455,7 +462,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = setFrameInfo(res) res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - when logGC: writeCell("new cell", res) + logCell("new cell", res) track("newObjRC1", res, size) gcTrace(res, csAllocated) when useCellIds: @@ -493,9 +500,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") - when logGC: - writeCell("growObj old cell", ol) - writeCell("growObj new cell", res) + logCell("growObj old cell", ol) + logCell("growObj new cell", res) gcTrace(ol, csZctFreed) gcTrace(res, csAllocated) track("growObj old", ol, 0) @@ -547,7 +553,7 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = prepareDealloc(c) gcTrace(c, csCycFreed) track("cycle collector dealloc cell", c, 0) - when logGC: writeCell("cycle collector dealloc cell", c) + logCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") beforeDealloc(gch, c, "freeCyclicCell: stack trash") @@ -616,7 +622,7 @@ proc doOperation(p: pointer, op: WalkOp) = # c_fprintf(stdout, "[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") - when logGC: writeCell("decref (from doOperation)", c) + logCell("decref (from doOperation)", c) track("waZctDecref", p, 0) decRef(c) of waPush: @@ -704,7 +710,7 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when logGC: writeCell("zct dealloc cell", c) + logCell("zct dealloc cell", c) track("zct dealloc cell", c, 0) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its @@ -858,6 +864,7 @@ when not defined(useNimRtl): for stack in items(gch.stack): result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & cast[pointer](stack.maxStackSize).repr & "\n" else: + result.add "[GC] stack bottom: " & gch.stack.bottom.repr result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" {.pop.} # profiler: off, stackTrace: off From a0623a235664cb4d575f9c4575a70c7db41eac4e Mon Sep 17 00:00:00 2001 From: Mandeep Singh Date: Sat, 26 Jan 2019 09:29:25 -0500 Subject: [PATCH 240/350] Updated httpClient.nim: added import httpClient statements in examples Added import httpClient statements in examples, as it was not obvious for a new started from python regarding what to import --- lib/pure/httpclient.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 269ae476fe..e5c0c6ac46 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -17,6 +17,7 @@ ## ``http://google.com``: ## ## .. code-block:: Nim +## import httpClient ## var client = newHttpClient() ## echo client.getContent("http://google.com") ## @@ -24,6 +25,7 @@ ## ``AsyncHttpClient``: ## ## .. code-block:: Nim +## import httpClient ## var client = newAsyncHttpClient() ## echo await client.getContent("http://google.com") ## From 3ce6b2acb9ce21d33f0f14161b0abf89a9ff7af9 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 27 Jan 2019 10:32:44 +0100 Subject: [PATCH 241/350] Fix exception tracking in try blocks (#10455) Exceptions raised inside a nkFinally/nkExcept block are not caught by the block itself. Fixes #3886 --- compiler/sempass2.nim | 10 +++++++++- tests/effects/teffects7.nim | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/effects/teffects7.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 81d637fee3..622e72074b 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -353,6 +353,8 @@ proc trackTryStmt(tracked: PEffects, n: PNode) = var branches = 1 var hasFinally = false + + # Collect the exceptions caught by the except branches for i in 1 ..< n.len: let b = n.sons[i] let blen = sonsLen(b) @@ -368,12 +370,18 @@ proc trackTryStmt(tracked: PEffects, n: PNode) = else: assert(b.sons[j].kind == nkType) catches(tracked, b.sons[j].typ) + else: + assert b.kind == nkFinally + # Add any other exception raised in the except bodies + for i in 1 ..< n.len: + let b = n.sons[i] + let blen = sonsLen(b) + if b.kind == nkExceptBranch: setLen(tracked.init, oldState) track(tracked, b.sons[blen-1]) for i in oldState.. Date: Sun, 27 Jan 2019 07:52:46 -0500 Subject: [PATCH 242/350] Make MemSlice stringification both simpler and faster. (#10464) --- lib/pure/memfiles.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index d1a006eee1..277c4ddb62 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -372,9 +372,8 @@ proc `==`*(x, y: MemSlice): bool = proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. - var buf = newString(ms.size) - copyMem(addr(buf[0]), ms.data, ms.size) - result = buf + result.setLen(ms.size) + copyMem(addr(result[0]), ms.data, ms.size) iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`. From 0b02241fc53eff2687b4fad3997159b953888634 Mon Sep 17 00:00:00 2001 From: alaviss Date: Sun, 27 Jan 2019 20:58:10 +0700 Subject: [PATCH 243/350] semcall: correct lineinfo for accquoted symbols (#10461) --- compiler/semcall.nim | 2 +- nimsuggest/tests/tqualified_highlight.nim | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 0613a41452..e8723e3df2 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -464,7 +464,7 @@ proc updateDefaultParams(call: PNode) = proc getCallLineInfo(n: PNode): TLineInfo = case n.kind - of nkBracketExpr, nkCall, nkCommand: getCallLineInfo(n.sons[0]) + of nkAccQuoted, nkBracketExpr, nkCall, nkCommand: getCallLineInfo(n.sons[0]) of nkDotExpr: getCallLineInfo(n.sons[1]) else: n.info diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim index 1b34dbade5..3b521ecc18 100644 --- a/nimsuggest/tests/tqualified_highlight.nim +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -1,5 +1,6 @@ system.echo#[!]# system.once +system.`$` 1 discard """ $nimsuggest --tester $file @@ -9,4 +10,5 @@ highlight;;skProc;;1;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 +highlight;;skProc;;3;;8;;1 """ From 7533f3bcbd1990badce757ad3c318211a6866303 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 27 Jan 2019 16:27:54 +0000 Subject: [PATCH 244/350] Add link to nims.html (#10472) [ci skip] --- lib/system/nimscript.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index e879fda833..1c4986aa4c 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -7,6 +7,7 @@ # distribution, for details about the copyright. # +## To learn about scripting in Nim see `NimScript`_ # Nim's configuration system now uses Nim for scripting. This module provides # a few things that are required for this to work. From 4f0c3274ee96bbefe716f8725b0975799b7663b9 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Sun, 27 Jan 2019 21:59:33 +0530 Subject: [PATCH 245/350] Fix example in manual for line pragma (#10468) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 09b9b4d784..f8a5887916 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -6922,7 +6922,7 @@ statement as seen in stack backtraces: template myassert*(cond: untyped, msg = "") = if not cond: # change run-time line information of the 'raise' statement: - {.line: InstantiationInfo().}: + {.line: instantiationInfo().}: raise newException(EAssertionFailed, msg) If the ``line`` pragma is used with a parameter, the parameter needs be a From 91b39f2219773a5905524d33c64248fa64603087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?= Date: Sun, 27 Jan 2019 21:59:58 +0100 Subject: [PATCH 246/350] uri: Update documentation typo (#10471) --- lib/pure/uri.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index c8350dfad4..e49bfb3c6b 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -33,8 +33,7 @@ ## import uri ## let res = parseUri("sftp://127.0.0.1:4343") ## if isAbsolute(res): -## echo "Connect to port: " & res.port -## # --> Connect to port: 4343 +## assert res.port == "4343" ## else: ## echo "Wrong format" @@ -343,9 +342,9 @@ proc combine*(uris: varargs[Uri]): Uri = ## **See also:** ## * `/ proc <#/,Uri,string>`_ for building URIs runnableExamples: - let foo = combine(parseUri("https://nim-lang.org/blog.html"), parseUri("/install.html")) + let foo = combine(parseUri("https://nim-lang.org/"), parseUri("docs/"), parseUri("manual.html")) assert foo.hostname == "nim-lang.org" - assert foo.path == "/install.html" + assert foo.path == "/docs/manual.html" result = uris[0] for i in 1 ..< uris.len: result = combine(result, uris[i]) From 5fa8a61a238c08bca12da272f71692f2f103b6d8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 27 Jan 2019 22:10:20 -0800 Subject: [PATCH 247/350] add git diff hints to help with reviews (#10435) --- doc/contributing.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index b89102a3b5..3edf8352cb 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -410,6 +410,26 @@ Code reviews saves time explaining the change or applying it; see also https://forum.nim-lang.org/t/4317 +2. When reviewing large diffs that may involve code moving around, github's interface + doesn't help much as it doesn't highlight moves. Instead you can use something + like this, see visual results `here `_: + + .. code-block:: sh + + git fetch origin pull/10431/head && git checkout FETCH_HEAD + git show --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^ + +3. In addition, you can view github-like diffs locally to identify what was changed + within a code block using `diff-highlight` or `diff-so-fancy`, eg: + + .. code-block:: sh + + # put this in ~/.gitconfig: + [core] + pager = "diff-so-fancy | less -R" # or: use: `diff-highlight` + + + .. include:: docstyle.rst From 690f21043d7c1cad321e116fdc666b2951ba1da3 Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 28 Jan 2019 07:32:14 +0000 Subject: [PATCH 248/350] isLastRead regression fix (#10463) * fixes #10462 * add a test --- compiler/destroyer.nim | 10 ++++++---- compiler/dfa.nim | 15 +++++++++------ tests/destructor/tmove_objconstr.nim | 9 +++++++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 06d4dcbefe..e21d532ea9 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -139,7 +139,7 @@ proc isLastRead(s: PSym; c: var Con; pc, comesFrom: int): int = of def: if c.g[pc].sym == s: # the path lead to a redefinition of 's' --> abandon it. - return pc + return high(int) inc pc of use: if c.g[pc].sym == s: @@ -150,14 +150,16 @@ proc isLastRead(s: PSym; c: var Con; pc, comesFrom: int): int = pc = pc + c.g[pc].dest of fork: # every branch must lead to the last read of the location: - let variantA = isLastRead(s, c, pc+1, pc) + var variantA = isLastRead(s, c, pc+1, pc) if variantA < 0: return -1 let variantB = isLastRead(s, c, pc + c.g[pc].dest, pc) if variantB < 0: return -1 - pc = variantA+1 + elif variantA == high(int): + variantA = variantB + pc = variantA of InstrKind.join: let dest = pc + c.g[pc].dest - if dest == comesFrom: return pc + if dest == comesFrom: return pc + 1 inc pc return pc diff --git a/compiler/dfa.nim b/compiler/dfa.nim index f349810006..462cf0fb76 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -74,7 +74,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) = while i <= last: if i in jumpTargets: result.add("L" & $i & ":\n") result.add "\t" - result.add $c[i].kind + result.add ($i & " " & $c[i].kind) result.add "\t" case c[i].kind of def, use: @@ -540,13 +540,17 @@ proc genTry(c: var Con; n: PNode) = c.gen(fin.sons[0]) doAssert(c.forks.len == oldLen) +template genNoReturn(c: var Con; n: PNode) = + # leave the graph + c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len) + proc genRaise(c: var Con; n: PNode) = genJoins(c, n) gen(c, n.sons[0]) if c.inTryStmt > 0: c.tryStmtFixups.add c.gotoI(n) else: - c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len) + genNoReturn(c, n) proc genImplicitReturn(c: var Con) = if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len: @@ -558,7 +562,7 @@ proc genReturn(c: var Con; n: PNode) = gen(c, n.sons[0]) else: genImplicitReturn(c) - c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len) + genNoReturn(c, n) const InterestingSyms = {skVar, skResult, skLet, skParam} @@ -612,9 +616,6 @@ proc genMagic(c: var Con; n: PNode; m: TMagic) = of mNew, mNewFinalize: genDef(c, n[1]) for i in 2.. 1: newMySeq(3, 1.0) else: newMySeq(0, 0.0)] var seqOfSeq2 = @[newMySeq(2, 5.0), newMySeq(3, 1.0)] + + +## issue #10462 +proc myfuncLoop(x: int): MySeqNonCopyable = + for i in 0.. Date: Mon, 28 Jan 2019 15:29:49 +0700 Subject: [PATCH 249/350] suggest: quote operators and keywords on suggestion (#10460) --- compiler/suggest.nim | 8 ++++++-- nimsuggest/tests/tsug_accquote.nim | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 nimsuggest/tests/tsug_accquote.nim diff --git a/compiler/suggest.nim b/compiler/suggest.nim index f3f960136f..09eacbbede 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -33,7 +33,7 @@ # included from sigmatch.nim import algorithm, prefixmatches, lineinfos, pathutils -from wordrecg import wDeprecated, wError +from wordrecg import wDeprecated, wError, wAddr, wYield, specialWords when defined(nimsuggest): import passes, tables # importer @@ -109,7 +109,11 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info result.qualifiedPath.add(ow2.origModuleName) if ow != nil: result.qualifiedPath.add(ow.origModuleName) - result.qualifiedPath.add(s.name.s) + if s.name.s[0] in OpChars + {'[', '{', '('} or + s.name.id in ord(wAddr)..ord(wYield): + result.qualifiedPath.add('`' & s.name.s & '`') + else: + result.qualifiedPath.add(s.name.s) if s.typ != nil: result.forth = typeToString(s.typ) diff --git a/nimsuggest/tests/tsug_accquote.nim b/nimsuggest/tests/tsug_accquote.nim new file mode 100644 index 0000000000..5b98feac4d --- /dev/null +++ b/nimsuggest/tests/tsug_accquote.nim @@ -0,0 +1,10 @@ +proc `%%%`(a: int) = discard +proc `cast`() = discard +tsug_accquote.#[!]# + +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skProc;;tsug_accquote.`%%%`;;proc (a: int);;$file;;1;;5;;"";;100;;None +sug;;skProc;;tsug_accquote.`cast`;;proc ();;$file;;2;;5;;"";;100;;None +""" From dfa5faea0fa5ef8cd33b16233828ebaa03192f1e Mon Sep 17 00:00:00 2001 From: slangmgh <37659406+slangmgh@users.noreply.github.com> Date: Mon, 28 Jan 2019 16:34:03 +0800 Subject: [PATCH 250/350] Fixes #10352 (#10416) --- compiler/cgen.nim | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 464ef39751..2d9814621d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1327,16 +1327,6 @@ proc genInitCode(m: BModule) = appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", [m.nimTypesName, rope(m.nimTypes)]) - if m.initProc.gcFrameId > 0: - moduleInitRequired = true - add(prc, initGCFrame(m.initProc)) - - if m.initProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.initProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) - if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: # Give this small function its own scope addf(prc, "{$N", []) @@ -1362,6 +1352,19 @@ proc genInitCode(m: BModule) = add(prc, genSectionEnd(cpsStmts, m.config)) addf(prc, "}$N", []) + # add new scope for following code, because old vcc compiler need variable + # be defined at the top of the block + addf(prc, "{$N", []) + if m.initProc.gcFrameId > 0: + moduleInitRequired = true + add(prc, initGCFrame(m.initProc)) + + if m.initProc.s(cpsLocals).len > 0: + moduleInitRequired = true + add(prc, genSectionStart(cpsLocals, m.config)) + add(prc, m.initProc.s(cpsLocals)) + add(prc, genSectionEnd(cpsLocals, m.config)) + if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: moduleInitRequired = true if optStackTrace in m.initProc.options and frameDeclared notin m.flags: @@ -1388,6 +1391,7 @@ proc genInitCode(m: BModule) = if m.initProc.gcFrameId > 0: moduleInitRequired = true add(prc, deinitGCFrame(m.initProc)) + addf(prc, "}$N", []) addf(prc, "}$N$N", []) From db2a4d230dcc9fe31d4184ad65e369b846cb5dd2 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 28 Jan 2019 09:51:54 +0100 Subject: [PATCH 251/350] Add logic to deserialize tyRange fields from json (#10469) --- lib/pure/json.nim | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index ab7cabed69..ffb8f4f351 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1328,6 +1328,12 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = let obj = getType(typeSym[1]) result = processType(newIdentNode(typeName), obj, jsonNode, true) + of "range": + let typeNode = typeSym + # Deduce the base type from one of the endpoints + let baseType = getType(typeNode[1]) + + result = createConstructor(baseType, jsonNode) of "seq": let seqT = typeSym[1] let forLoopI = genSym(nskForVar, "i") @@ -1721,3 +1727,21 @@ when isMainModule: foo = js.to Foo doAssert(foo.b == "abc") + + # Generate constructors for range[T] types + block: + type + Q1 = range[0..10] + Q2 = range[0'i8..10'i8] + Q3 = range[0'u16..10'u16] + X = object + m1: Q1 + m2: Q2 + m3: Q3 + + let + obj = X(m1: 1, m2: 2'i8, m3: 3'u16) + jsonObj = %obj + desObj = to(jsonObj, type(obj)) + + doAssert(desObj == obj) From 2b0ed9995d6fd64873ad0b42c2953944b0384908 Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Mon, 28 Jan 2019 14:22:48 +0530 Subject: [PATCH 252/350] Fixes https://github.com/nim-lang/Nim/issues/8484 (#10470) [backport] --- compiler/vm.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 131501380f..e95a491fd1 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -934,7 +934,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = stackTrace(c, tos, pc, errNilAccess) of opcGetImpl: decodeB(rkNode) - let a = regs[rb].node + var a = regs[rb].node + if a.kind == nkVarTy: a = a[0] if a.kind == nkSym: regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit) else: copyTree(a.sym.ast) From 9402c82e803d133e0b845a7c5c79c261781e7d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Mon, 28 Jan 2019 09:58:37 +0100 Subject: [PATCH 253/350] Cleanup heapqueue module (#10476) --- doc/lib.rst | 2 +- lib/pure/collections/heapqueue.nim | 153 +++++++++++++++++++++-------- 2 files changed, 114 insertions(+), 41 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 082c3c0bd1..d00591e10b 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -92,7 +92,7 @@ Collections and algorithms The underlying implementation uses a ``seq``. * `heapqueue `_ - This module implements Heap queue (a.k.a. priority queue) algorithm. + Implementation of a heap data structure that can be used as a priority queue. * `intsets `_ Efficient implementation of a set of ints as a sparse bit set. diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index 60869142ec..cdb8db6e12 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -1,4 +1,3 @@ - # # # Nim's Runtime Library @@ -7,32 +6,74 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -##[ Heap queue algorithm (a.k.a. priority queue). Ported from Python heapq. +##[ + The `heapqueue` module implements a + `heap data structure`_ + that can be used as a + `priority queue`_. + Heaps are arrays for which `a[k] <= a[2*k+1]` and `a[k] <= a[2*k+2]` for + all `k`, counting elements from 0. The interesting property of a heap is that + `a[0]` is always its smallest element. -Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for -all k, counting elements from 0. For the sake of comparison, -non-existing elements are considered to be infinite. The interesting -property of a heap is that a[0] is always its smallest element. + Basic usage + ----------- + .. code-block:: Nim + import heapqueue + var heap = initHeapQueue[int]() + heap.push(8) + heap.push(2) + heap.push(5) + # The first element is the lowest element + assert heap[0] == 2 + # Remove and return the lowest element + assert heap.pop() == 2 + # The lowest element remaining is 5 + assert heap[0] == 5 + + Usage with custom object + ------------------------ + To use a `HeapQueue` with a custom object, the `<` operator must be + implemented. + + .. code-block:: Nim + import heapqueue + + type Job = object + priority: int + + proc `<`(a, b: Job): bool = a.priority < b.priority + + var jobs = initHeapQueue[Job]() + jobs.push(Job(priority: 1)) + jobs.push(Job(priority: 2)) + + assert jobs[0].priority == 1 ]## -type HeapQueue*[T] = distinct seq[T] +type HeapQueue*[T] = object + ## A heap queue, commonly known as a priority queue. + data: seq[T] -proc newHeapQueue*[T](): HeapQueue[T] {.inline.} = HeapQueue[T](newSeq[T]()) -proc newHeapQueue*[T](h: var HeapQueue[T]) {.inline.} = h = HeapQueue[T](newSeq[T]()) +proc initHeapQueue*[T](): HeapQueue[T] = + ## Create a new empty heap. + discard -proc len*[T](h: HeapQueue[T]): int {.inline.} = seq[T](h).len -proc `[]`*[T](h: HeapQueue[T], i: int): T {.inline.} = seq[T](h)[i] -proc `[]=`[T](h: var HeapQueue[T], i: int, v: T) {.inline.} = seq[T](h)[i] = v -proc add[T](h: var HeapQueue[T], v: T) {.inline.} = seq[T](h).add(v) +proc len*[T](heap: HeapQueue[T]): int {.inline.} = + ## Return the number of elements of `heap`. + heap.data.len + +proc `[]`*[T](heap: HeapQueue[T], i: Natural): T {.inline.} = + ## Access the i-th element of `heap`. + heap.data[i] proc heapCmp[T](x, y: T): bool {.inline.} = return (x < y) -# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos -# is the index of a leaf with a possibly out-of-order value. Restore the -# heap invariant. proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = + ## 'heap' is a heap at all indices >= startpos, except possibly for pos. pos + ## is the index of a leaf with a possibly out-of-order value. Restore the + ## heap invariant. var pos = p var newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place @@ -41,11 +82,11 @@ proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = let parentpos = (pos - 1) shr 1 let parent = heap[parentpos] if heapCmp(newitem, parent): - heap[pos] = parent + heap.data[pos] = parent pos = parentpos else: break - heap[pos] = newitem + heap.data[pos] = newitem proc siftup[T](heap: var HeapQueue[T], p: int) = let endpos = len(heap) @@ -60,48 +101,50 @@ proc siftup[T](heap: var HeapQueue[T], p: int) = if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): childpos = rightpos # Move the smaller child up. - heap[pos] = heap[childpos] + heap.data[pos] = heap[childpos] pos = childpos childpos = 2*pos + 1 # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). - heap[pos] = newitem + heap.data[pos] = newitem siftdown(heap, startpos, pos) proc push*[T](heap: var HeapQueue[T], item: T) = - ## Push item onto heap, maintaining the heap invariant. - (seq[T](heap)).add(item) + ## Push `item` onto heap, maintaining the heap invariant. + heap.data.add(item) siftdown(heap, 0, len(heap)-1) proc pop*[T](heap: var HeapQueue[T]): T = - ## Pop the smallest item off the heap, maintaining the heap invariant. - let lastelt = seq[T](heap).pop() + ## Pop and return the smallest item from `heap`, + ## maintaining the heap invariant. + let lastelt = heap.data.pop() if heap.len > 0: result = heap[0] - heap[0] = lastelt + heap.data[0] = lastelt siftup(heap, 0) else: result = lastelt -proc del*[T](heap: var HeapQueue[T], index: int) = - ## Removes element at `index`, maintaining the heap invariant. - swap(seq[T](heap)[^1], seq[T](heap)[index]) +proc del*[T](heap: var HeapQueue[T], index: Natural) = + ## Removes the element at `index` from `heap`, maintaining the heap invariant. + swap(heap.data[^1], heap.data[index]) let newLen = heap.len - 1 - seq[T](heap).setLen(newLen) + heap.data.setLen(newLen) if index < newLen: heap.siftup(index) proc replace*[T](heap: var HeapQueue[T], item: T): T = ## Pop and return the current smallest value, and add the new item. ## This is more efficient than pop() followed by push(), and can be - ## more appropriate when using a fixed-size heap. Note that the value - ## returned may be larger than item! That constrains reasonable uses of + ## more appropriate when using a fixed-size heap. Note that the value + ## returned may be larger than item! That constrains reasonable uses of ## this routine unless written as part of a conditional replacement: - + ## + ## .. code-block:: nim ## if item > heap[0]: ## item = replace(heap, item) result = heap[0] - heap[0] = item + heap.data[0] = item siftup(heap, 0) proc pushpop*[T](heap: var HeapQueue[T], item: T): T = @@ -111,6 +154,36 @@ proc pushpop*[T](heap: var HeapQueue[T], item: T): T = siftup(heap, 0) return item +proc clear*[T](heap: var HeapQueue[T]) = + ## Remove all elements from `heap`, making it empty. + runnableExamples: + var heap = initHeapQueue[int]() + heap.push(1) + heap.clear() + assert heap.len == 0 + heap.data.setLen(0) + +proc `$`*[T](heap: HeapQueue[T]): string = + ## Turn a heap into its string representation. + runnableExamples: + var heap = initHeapQueue[int]() + heap.push(1) + heap.push(2) + assert $heap == "[1, 2]" + result = "[" + for x in heap.data: + if result.len > 1: result.add(", ") + result.addQuoted(x) + result.add("]") + +proc newHeapQueue*[T](): HeapQueue[T] {.deprecated.} = + ## **Deprecated since v0.20.0:** use ``initHeapQueue`` instead. + initHeapQueue[T]() + +proc newHeapQueue*[T](heap: var HeapQueue[T]) {.deprecated.} = + ## **Deprecated since v0.20.0:** use ``clear`` instead. + heap.clear() + when isMainModule: proc toSortedSeq[T](h: HeapQueue[T]): seq[T] = var tmp = h @@ -119,7 +192,7 @@ when isMainModule: result.add(pop(tmp)) block: # Simple sanity test - var heap = newHeapQueue[int]() + var heap = initHeapQueue[int]() let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] for item in data: push(heap, item) @@ -127,27 +200,27 @@ when isMainModule: doAssert(heap.toSortedSeq == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) block: # Test del - var heap = newHeapQueue[int]() + var heap = initHeapQueue[int]() let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] for item in data: push(heap, item) heap.del(0) doAssert(heap[0] == 1) - heap.del(seq[int](heap).find(7)) + heap.del(heap.data.find(7)) doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 5, 6, 8, 9]) - heap.del(seq[int](heap).find(5)) + heap.del(heap.data.find(5)) doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 6, 8, 9]) - heap.del(seq[int](heap).find(6)) + heap.del(heap.data.find(6)) doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 8, 9]) - heap.del(seq[int](heap).find(2)) + heap.del(heap.data.find(2)) doAssert(heap.toSortedSeq == @[1, 3, 4, 8, 9]) block: # Test del last - var heap = newHeapQueue[int]() + var heap = initHeapQueue[int]() let data = [1, 2, 3] for item in data: push(heap, item) From 96a8caa9ef400be768d62597f62186c4ac8f64b7 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 28 Jan 2019 10:58:24 +0100 Subject: [PATCH 254/350] fix documentation css --- config/nimdoc.cfg | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index f89739df69..f534264a0a 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -1314,11 +1314,8 @@ dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; } -dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { - color: inherit; - font-weight: inherit; } - -dt pre > span.Operator ~ span.Identifier { +dt pre > span.Keyword ~ span.Identifier, dt pre > span.Identifier ~ span.Identifier, +dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { color: inherit; font-weight: inherit; } From ee8bc96a89e6139f10704c4c46aab402e2fe7966 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 28 Jan 2019 16:59:07 +0100 Subject: [PATCH 255/350] change the expected output of nim doc --- nimdoc/testproject/expected/subdir/subdir_b/utils.html | 7 ++----- nimdoc/testproject/expected/testproject.html | 7 ++----- nimdoc/testproject/expected/theindex.html | 7 ++----- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f057df7e36..58108b5f21 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1139,11 +1139,8 @@ dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; } -dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { - color: inherit; - font-weight: inherit; } - -dt pre > span.Operator ~ span.Identifier { +dt pre > span.Keyword ~ span.Identifier, dt pre > span.Identifier ~ span.Identifier, +dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { color: inherit; font-weight: inherit; } diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 21a7a0cf83..63e7bcd6a8 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1139,11 +1139,8 @@ dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; } -dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { - color: inherit; - font-weight: inherit; } - -dt pre > span.Operator ~ span.Identifier { +dt pre > span.Keyword ~ span.Identifier, dt pre > span.Identifier ~ span.Identifier, +dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { color: inherit; font-weight: inherit; } diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 5a6e575bf5..4a8744b96a 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1139,11 +1139,8 @@ dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; } -dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { - color: inherit; - font-weight: inherit; } - -dt pre > span.Operator ~ span.Identifier { +dt pre > span.Keyword ~ span.Identifier, dt pre > span.Identifier ~ span.Identifier, +dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Identifier { color: inherit; font-weight: inherit; } From ac8f6c57a1eb915e9b25ea52bf868c8b89b30c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Mon, 28 Jan 2019 17:18:07 +0100 Subject: [PATCH 256/350] Improve exception tracking in the streams module (#10453) --- changelog.md | 2 ++ lib/pure/streams.nim | 37 ++++++++++++++++--------------------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/changelog.md b/changelog.md index eb97ac9a6f..85e06112bb 100644 --- a/changelog.md +++ b/changelog.md @@ -38,6 +38,8 @@ - The procs `parseutils.parseBiggsetInt`, `parseutils.parseInt`, `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a `ValueError` when the parsed integer is outside of the valid range. Previously they sometimes raised a `OverflowError` and sometimes returned `0`. +- `streams.StreamObject` now restricts its fields to only raise `system.Defect`, `system.IOError` and `system.OSError`. This change only affects custom stream implementations. + - nre's `RegexMatch.{captureBounds,captures}[]` no longer return `Option` or `nil`/`""`, respectivly. Use the newly added `n in p.captures` method to check if a group is captured, otherwise you'll recieve an exception. diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 10de86e9f3..b6e77bbc02 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -45,17 +45,22 @@ type ## here shouldn't be used directly. They are ## accessible so that a stream implementation ## can override them. - closeImpl*: proc (s: Stream) {.nimcall, tags: [], gcsafe.} - atEndImpl*: proc (s: Stream): bool {.nimcall, tags: [], gcsafe.} - setPositionImpl*: proc (s: Stream, pos: int) {.nimcall, tags: [], gcsafe.} - getPositionImpl*: proc (s: Stream): int {.nimcall, tags: [], gcsafe.} - readDataImpl*: proc (s: Stream, buffer: pointer, - bufLen: int): int {.nimcall, tags: [ReadIOEffect], gcsafe.} - peekDataImpl*: proc (s: Stream, buffer: pointer, - bufLen: int): int {.nimcall, tags: [ReadIOEffect], gcsafe.} - writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int) {.nimcall, - tags: [WriteIOEffect], gcsafe.} - flushImpl*: proc (s: Stream) {.nimcall, tags: [WriteIOEffect], gcsafe.} + closeImpl*: proc (s: Stream) + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + atEndImpl*: proc (s: Stream): bool + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + setPositionImpl*: proc (s: Stream, pos: int) + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + getPositionImpl*: proc (s: Stream): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + readDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + peekDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int) + {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + flushImpl*: proc (s: Stream) + {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} proc flush*(s: Stream) = ## flushes the buffers that the stream `s` might use. @@ -65,10 +70,6 @@ proc close*(s: Stream) = ## closes the stream `s`. if not isNil(s.closeImpl): s.closeImpl(s) -proc close*(s, unused: Stream) {.deprecated.} = - ## closes the stream `s`. - s.closeImpl(s) - proc atEnd*(s: Stream): bool = ## checks if more data can be read from `f`. Returns true if all data has ## been read. @@ -111,12 +112,6 @@ proc writeData*(s: Stream, buffer: pointer, bufLen: int) = ## to the stream `s`. s.writeDataImpl(s, buffer, bufLen) -proc writeData*(s, unused: Stream, buffer: pointer, - bufLen: int) {.deprecated.} = - ## low level proc that writes an untyped `buffer` of `bufLen` size - ## to the stream `s`. - s.writeDataImpl(s, buffer, bufLen) - proc write*[T](s: Stream, x: T) = ## generic write procedure. Writes `x` to the stream `s`. Implementation: ## From cf323104fc8b4b40b2a4cb1f0b405a22dc134bde Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 28 Jan 2019 16:32:30 +0000 Subject: [PATCH 257/350] Print error summary in Travis CI (#10474) --- .travis.yml | 4 ++++ tools/ci_testresults.nim | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tools/ci_testresults.nim diff --git a/.travis.yml b/.travis.yml index ea5d54eadc..293ad6bb23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ before_script: - cd .. - export PATH=$(pwd)/bin${PATH:+:$PATH} - echo PATH:${PATH} + - set +e # prevents breaking after_failure script: - nim c koch @@ -59,3 +60,6 @@ deploy: # https://nim-lang.github.io/Nim keep-history: false on: branch: devel + +# Extract failed tests +after_failure: nim c -r tools/ci_testresults.nim diff --git a/tools/ci_testresults.nim b/tools/ci_testresults.nim new file mode 100644 index 0000000000..3201606d7f --- /dev/null +++ b/tools/ci_testresults.nim @@ -0,0 +1,24 @@ +## Print summary of failed tests for CI + +import os, json, sets, strformat + +const skip = toSet(["reDisabled", "reIgnored", "reSuccess", "reJoined"]) + +when isMainModule: + for fn in walkFiles("testresults/*.json"): + let entries = fn.readFile().parseJson() + for j in entries: + let res = j["result"].getStr() + if skip.contains(res): + continue + echo fmt """ +Category: {j["category"].getStr()} +Name: {j["name"].getStr()} +Action: {j["action"].getStr()} +Result: {res} +-------- Expected ------- +{j["expected"].getStr()} +--------- Given -------- +{j["given"].getStr()} +------------------------- +""" From aa0a9dd78c777437793f71872d48669d2d030f6e Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 28 Jan 2019 16:33:35 +0000 Subject: [PATCH 258/350] Improve Travis CI folding (#10473) --- .travis.yml | 2 ++ koch.nim | 8 ++++---- tools/kochdocs.nim | 9 +++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 293ad6bb23..0e55d3affc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,9 @@ before_script: - set +e # prevents breaking after_failure script: + - echo "travis_fold:start:nim_c_koch" - nim c koch + - echo "travis_fold:end:nim_c_koch" - ./koch runCI before_deploy: diff --git a/koch.nim b/koch.nim index d21c5240e5..89ee28d5a7 100644 --- a/koch.nim +++ b/koch.nim @@ -163,7 +163,7 @@ proc bundleNimsuggest() = nimCompile("nimsuggest/nimsuggest.nim", options = "-d:release") proc buildVccTool() = - nimCompile("tools/vccexe/vccexe.nim") + nimCompileFold("Compile Vcc", "tools/vccexe/vccexe.nim") proc bundleWinTools() = # TODO: consider building under `bin` instead of `.` @@ -208,10 +208,10 @@ proc buildTool(toolname, args: string) = proc buildTools() = bundleNimsuggest() - nimCompile("tools/nimgrep.nim", options = "-d:release") + nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", options = "-d:release") when defined(windows): buildVccTool() - nimCompile("nimpretty/nimpretty.nim", options = "-d:release") - nimCompile("tools/nimfind.nim", options = "-d:release") + nimCompileFold("Compile nimpretty", "nimpretty/nimpretty.nim", options = "-d:release") + nimCompileFold("Compile nimfind", "tools/nimfind.nim", options = "-d:release") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 7919deec68..c0e7ce66b7 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -42,10 +42,10 @@ proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = ## Execute shell command. Add log folding on Travis CI. # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "") + echo "travis_fold:start:" & desc.replace(" ", "_") exec(cmd, errorcode, additionalPath) if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "") + echo "travis_fold:end:" & desc.replace(" ", "_") proc execCleanPath*(cmd: string, additionalPath = ""; errorcode: int = QuitFailure) = @@ -69,6 +69,11 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = + let output = outputDir / input.splitFile.name.exe + let cmd = findNim() & " " & mode & " -o:" & output & " " & options & " " & input + execFold(desc, cmd) + const pdf = """ doc/manual.rst From 6bae78f365caa6d9afc2c83973ae97d415697079 Mon Sep 17 00:00:00 2001 From: deansher Date: Mon, 28 Jan 2019 08:11:59 -0500 Subject: [PATCH 259/350] First pass at compile-time/runtime terminology changes. --- doc/manual.rst | 326 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 216 insertions(+), 110 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index f8a5887916..20aa320a79 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -46,7 +46,7 @@ and ``a ^* b`` is short for ``(a (b a)*)?``. Example:: arrayConstructor = '[' expr ^* ',' ']' -Other parts of Nim - like scoping rules or runtime semantics are only +Other parts of Nim - like scoping rules or execution semantics are only described in the, more easily comprehensible, informal manner for now. @@ -55,7 +55,7 @@ described in the, more easily comprehensible, informal manner for now. Definitions =========== -A Nim program specifies a computation that acts on a memory consisting of +Nim code specifies a computation that acts on a memory consisting of components called `locations`:idx:. A variable is basically a name for a location. Each variable and location is of a certain `type`:idx:. The variable's type is called `static type`:idx:, the location's type is called @@ -71,21 +71,37 @@ identifier is declared unless overloading resolution rules suggest otherwise. An expression specifies a computation that produces a value or location. Expressions that produce locations are called `l-values`:idx:. An l-value can denote either a location or the value the location contains, depending on -the context. Expressions whose values can be determined statically are called -`constant expressions`:idx:; they are never l-values. +the context. -A `static error`:idx: is an error that the implementation detects before -program execution. Unless explicitly classified, an error is a static error. +A Nim `program`:idx: consists of one or more text `source files`:idx: containing +Nim code. It is processed by a Nim `compiler`:idx: into an `executable`:idx:. +The nature of this executable depends on the compiler implementation; it may, +for example, be a native binary or JavaScript source code. A Nim program can +contain Nim code that will be executed at `compile time`:idx:, such as constant +expressions, macro definitions, and Nim procedures used by macro definitions. +Typically, the bulk of a Nim program's code is compiled into the executable and +executed at `runtime`:idx:. Only a subset of the Nim language is supported at +compile time, but it is a substantial subset. See `Restrictions on +Compile-Time Execution `_ for details. -A `checked runtime error`:idx: is an error that the implementation detects -and reports at runtime. The method for reporting such errors is via *raising -exceptions* or *dying with a fatal error*. However, the implementation -provides a means to disable these runtime checks. See the section pragmas_ -for details. +The compiler parses Nim source code into an internal data structure called the +`abstract syntax tree`:idx: (`AST`:idx). Then, before executing the code or +processing it into the executable, it transforms the AST through `semantic +analysis`:idx:. This adds considerable semantic information to the AST, such as +expression types, identifier meanings, and in some cases expression values. An +error detected during semantic analysis is called a `static error`:idx:. Errors +mentioned in this manual are static errors when not otherwise specified. -Whether a checked runtime error results in an exception or in a fatal error at -runtime is implementation specific. Thus the following program is always -invalid: +An error detected during code execution (whether at compile time or at runtime) +is a `checked execution error`:idx:. The method for reporting such errors is via +*raising exceptions* or *dying with a fatal error*. However, the implementation +provides a means to disable these `execution-time checks`:idx:. See the section +pragmas_ for details. + +Whether a checked execution error results in an exception or in a fatal error +is implementation specific. Thus the following program is invalid; even +though it purports to catch the `IndexError` from an out-of-bounds array access, +the compiler may instead choose to allow execution to die with a fatal error. .. code-block:: nim var a: array[0..1, char] @@ -95,11 +111,28 @@ invalid: except IndexError: echo "invalid index" -An `unchecked runtime error`:idx: is an error that is not guaranteed to be +An `unchecked execution error`:idx: is an error that is not guaranteed to be detected, and can cause the subsequent behavior of the computation to -be arbitrary. Unchecked runtime errors cannot occur if only `safe`:idx: -language features are used. +be arbitrary. Unchecked execution errors cannot occur if only `safe`:idx: +language features are used and if no execution-time checks are disabled. +A `constant expression`:idx: is an expression whose value can be computed during +semantic analysis of the code in which it appears. It is never an l-value and +never has side effects. Constant expressions are not limited to the capabilities +of semantic analysis, such as constant folding; they can use the substantial +subset of the Nim language that is supported at compile time. Since constant +expressions can be used as an input to semantic analysis (such as for defining +array bounds), this flexibility requires the compiler to interleave semantic +analysis and compile-time code execution. + +It is mostly accurate to picture semantic analysis proceeding top to bottom and +left to right in the source code, with compile-time code execution interleaved +when necessary to compute values that are required for subsequent semantic +analysis. We saw just above that this interleaving is necessary for handling +constant expressions. We will see much later in this document that macro +invocation not only requires this interleaving, but also creates a situation +where semantic analyis does not entirely proceed top to bottom and left to +right. Lexical Analysis @@ -673,10 +706,90 @@ Rationale: Consistency with overloaded assignment or assignment-like operations, ``a = b`` can be read as ``performSomeCopy(a, b)``. +Constants and Constant Expressions +================================== + +A `constant`:idx: is a symbol that is bound to the value of a `constant +expression`. This is an expression whose value can be computed during +semantic analysis of the code in which it appears. However, constant +expressions are not limited to the capabilities of semantic analysis; they +can use the substantial subset of the Nim language that is supported for +compile-time execution. Compile-time execution is interleaved with semantic +analysis as necessary. A constant's value cannot change after it is first +computed. + +A constant expression must be composed of the following elements: + +* literals +* previously declared constants and compile-time variables +* previously declared macros and templates +* previously declared procedures that have no side effects beyond + possibly modifying compile-time variables +* operators representing such procedures +* code blocks that can internally use all Nim features supported at + compile time (the next section below), but that cannot + refer to any external values beyond those listed above + +Constant expressions must adhere to the restrictions on compile-time +execution described in `Restrictions on +Compile-Time Execution `_. + +For example, the following code echoes the beginning of the Fibonacci +series **at compile time**. This is a demonstration of flexibility in +defining constants, not a recommended style for solving this problem! + +.. code-block:: nim + :test: "nim c $1" + import strformat + + var fib_n {.compileTime.}: int + var fib_prev {.compileTime.}: int + var fib_prev_prev {.compileTime.}: int + + proc next_fib(): int = + result = if fib_n < 2: + fib_n + else: + fib_prev_prev + fib_prev + inc(fib_n) + fib_prev_prev = fib_prev + fib_prev = result + + const f0 = next_fib() + const f1 = next_fib() + + const display_fib = block: + const f2 = next_fib() + var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}, " + for i in 0..10: + if i > 0: + add(result, ", ") + add(result, $next_fib()) + result + + static: + echo display_fib + + +Restrictions on Compile-Time Execution +====================================== + +Nim code that will be executed at compile-time cannot use the following +language features: + +* methods +* closure iterators +* ``cast`` +* ``ptr``s or ``ref``s +* the FFI + +Some or all of these restrictions are likely to be lifted over time. + + Types ===== -All expressions have a type which is known at compile time. Nim +All expressions have a type which is known during semantic analysis. Nim is statically typed. One can declare new types, which is in essence defining an identifier that can be used to denote this custom type. @@ -700,9 +813,9 @@ Ordinal types have the following characteristics: the operation of functions as ``inc``, ``ord``, ``dec`` on ordinal types to be defined. - Ordinal values have a smallest possible value. Trying to count further - down than the smallest value gives a checked runtime or static error. + down than the smallest value gives a checked execution or static error. - Ordinal values have a largest possible value. Trying to count further - than the largest value gives a checked runtime or static error. + than the largest value gives a checked execution or static error. Integers, bool, characters and enumeration types (and subranges of these types) belong to ordinal types. For reasons of simplicity of implementation @@ -810,7 +923,7 @@ lowest and highest value of the type: to 5. ``PositiveFloat`` defines a subrange of all positive floating point values. NaN does not belong to any subrange of floating point types. Assigning any other value to a variable of type ``Subrange`` is a -checked runtime error (or static error if it can be statically +checked execution error (or static error if it can be statically determined). Assignments from the base type to one of its subrange types (and vice versa) are allowed. @@ -854,7 +967,7 @@ The IEEE standard defines five types of floating-point exceptions: * Inexact: operation produces a result that cannot be represented with infinite precision, for example, 2.0 / 3.0, log(1.1) and 0.1 in input. -The IEEE exceptions are either ignored at runtime or mapped to the +The IEEE exceptions are either ignored during execution or mapped to the Nim exceptions: `FloatInvalidOpError`:idx:, `FloatDivByZeroError`:idx:, `FloatOverflowError`:idx:, `FloatUnderflowError`:idx:, and `FloatInexactError`:idx:. @@ -881,8 +994,9 @@ The only operations that are affected by the ``floatChecks`` pragma are the ``+``, ``-``, ``*``, ``/`` operators for floating point types. An implementation should always use the maximum precision available to evaluate -floating pointer values at compile time; this means expressions like -``0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64`` are true. +floating pointer values during semantic analysis; this means expressions like +``0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64`` that are evaluating during +constant folding are true. Boolean type @@ -1110,8 +1224,8 @@ tuples, objects and sets belong to the structured types. Array and sequence types ------------------------ Arrays are a homogeneous type, meaning that each element in the array -has the same type. Arrays always have a fixed length which is specified at -compile time (except for open arrays). They can be indexed by any ordinal type. +has the same type. Arrays always have a fixed length which is known during +semantic analysis (except for open arrays). They can be indexed by any ordinal type. A parameter ``A`` may be an *open array*, in which case it is indexed by integers from 0 to ``len(A)-1``. An array expression may be constructed by the array constructor ``[]``. The element type of this array expression is @@ -1119,7 +1233,7 @@ inferred from the type of the first element. All other elements need to be implicitly convertable to this type. Sequences are similar to arrays but of dynamic length which may change -during runtime (like strings). Sequences are implemented as growable arrays, +during execution (like strings). Sequences are implemented as growable arrays, allocating pieces of memory as items are added. A sequence ``S`` is always indexed by integers from 0 to ``len(S)-1`` and its bounds are checked. Sequences can be constructed by the array constructor ``[]`` in conjunction @@ -1153,7 +1267,7 @@ operator, and remove (and get) the last element of a sequence with the The notation ``x[i]`` can be used to access the i-th element of ``x``. -Arrays are always bounds checked (at compile-time or at runtime). These +Arrays are always bounds checked (statically or during execution). These checks can be disabled via pragmas or invoking the compiler with the ``--boundChecks:off`` command line switch. @@ -1260,7 +1374,7 @@ is currently not checked. **Future directions**: GC'ed memory should be allowed in unchecked arrays and there should be an explicit annotation of how the GC is to determine the -runtime size of the array. +execution-time size of the array. @@ -1317,7 +1431,7 @@ can also be defined with indentation instead of ``[]``: age: natural # and an age Objects provide many features that tuples do not. Object provide inheritance -and information hiding. Objects have access to their type at runtime, so that +and information hiding. Objects have access to their type during execution, so that the ``of`` operator can be used to determine the object's type. The ``of`` operator is similar to the ``instanceof`` operator in Java. @@ -1415,7 +1529,7 @@ safety its address cannot be taken and assignments to it are restricted: The new value must not lead to a change of the active object branch. For an object branch switch ``system.reset`` has to be used. Also, when the fields of a particular branch are specified during object construction, the correct value -for the discriminator must be supplied at compile-time. +for the discriminator must be available for semantic analysis. Package level objects --------------------- @@ -1429,7 +1543,7 @@ contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does not yet know the size of the object. To complete an incomplete object the ``package`` pragma has to be used. ``package`` implies ``byref``. -As long as a type ``T`` is incomplete ``sizeof(T)`` or "runtime type +As long as a type ``T`` is incomplete ``sizeof(T)`` or "execution-time type information" for ``T`` is not available. @@ -1859,7 +1973,7 @@ that don't. Distinct types provide a means to introduce a new string type username: string db.query("SELECT FROM users WHERE name = '$1'" % username) - # Error at compile time: `query` expects an SQL string! + # Static error: `query` expects an SQL string! It is an essential property of abstract types that they **do not** imply a @@ -2246,11 +2360,8 @@ Overloading resolution ====================== In a call ``p(args)`` the routine ``p`` that matches best is selected. If -multiple routines match equally well, the ambiguity is reported at compiletime. - -Every arg in args needs to match. There are multiple different categories how an -argument can match. Let ``f`` be the formal parameter's type and ``a`` the type -of the argument. +multiple routines match equally well, the ambiguity is reported during +semantic analysis. 1. Exact match: ``a`` and ``f`` are of the same type. 2. Literal match: ``a`` is an integer literal of value ``v`` @@ -2627,53 +2738,36 @@ identifier ``_`` can be used to ignore some parts of the tuple: Const section ------------- -`Constants`:idx: are symbols which are bound to a value. The constant's value -cannot change. The compiler must be able to evaluate the expression in a -constant declaration at compile time. +A const section declares constants whose values are constant expressions: -Nim contains a sophisticated compile-time evaluator, so procedures which -have no side-effect can be used in constant expressions too: - -.. code-block:: nim +.. code-block:: import strutils const + roundPi = 3.1415 constEval = contains("abc", 'b') # computed at compile time! +Once declared, a constant's symbol can be used as a constant expression. -The rules for compile-time computability are: - -1. Literals are compile-time computable. -2. Type conversions are compile-time computable. -3. Procedure calls of the form ``p(X)`` are compile-time computable if - ``p`` is a proc without side-effects (see the `noSideEffect pragma - <#pragmas-nosideeffect-pragma>`_ for details) and if ``X`` is a - (possibly empty) list of compile-time computable arguments. - - -Constants cannot be of type ``ptr``, ``ref`` or ``var``, nor can -they contain such a type. - +See `Constants and Constant Expressions <#constants-and-constant-expressions>`_ +for details. Static statement/expression --------------------------- -A static statement/expression can be used to enforce compile -time evaluation explicitly. Enforced compile time evaluation can even evaluate -code that has side effects: +A static statement/expression explicitly requires compile-time execution. +Even some code that has side effects is permitted in a static block: .. code-block:: static: echo "echo at compile time" -It's a static error if the compiler cannot perform the evaluation at compile +There are limitations on what Nim code can be executed at compile time; +see `Restrictions on Compile-Time Execution +<#restrictions-on-compile-time-execution>`_ for details. +It's a static error if the compiler cannot execute the block at compile time. -The current implementation poses some restrictions for compile time -evaluation: Code which contains ``cast`` or makes use of the foreign function -interface cannot be evaluated at compile time. Later versions of Nim will -support the FFI at compile time. - If statement ------------ @@ -2747,9 +2841,9 @@ empty ``discard`` statement should be used. For non ordinal types it is not possible to list every possible value and so these always require an ``else`` part. -As case statements perform compile-time exhaustiveness checks, the value in -every ``of`` branch must be known at compile time. This fact is also exploited -to generate more performant code. +Because case statements are checked for exhaustiveness during semantic analysis, +the value in every ``of`` branch must be computable during analysis. +This restriction also allows the compiler to generate more performant code. As a special semantic extension, an expression in an ``of`` branch of a case statement may evaluate to a set or array constructor; the set or array is then @@ -2814,10 +2908,10 @@ Example: .. code-block:: nim proc someProcThatMayRunInCompileTime(): bool = when nimvm: - # This code runs in compile time + # This code executes at compile time result = true else: - # This code runs in runtime + # This code executes at runtime result = false const ctValue = someProcThatMayRunInCompileTime() let rtValue = someProcThatMayRunInCompileTime() @@ -3557,7 +3651,7 @@ returned value is an l-value and can be modified by the caller: writeAccessToG() = 6 assert g == 6 -It is a compile time error if the implicitly introduced pointer could be +It is a static error if the implicitly introduced pointer could be used to access a location beyond its lifetime: .. code-block:: nim @@ -3673,7 +3767,7 @@ Invocation of a multi-method cannot be ambiguous: collide 2 is preferred over collide 1 because the resolution works from left to right. In the example ``Unit, Thing`` is preferred over ``Thing, Unit``. -**Note**: Compile time evaluation is not (yet) supported for methods. +**Note**: Compile time execution is not (yet) supported for methods. Inhibit dynamic method resolution via procCall @@ -3794,7 +3888,7 @@ In contrast to that, a `closure iterator`:idx: can be passed around more freely: Closure iterators have other restrictions than inline iterators: 1. ``yield`` in a closure iterator can not occur in a ``try`` statement. -2. For now, a closure iterator cannot be evaluated at compile time. +2. For now, a closure iterator cannot be executed at compile time. 3. ``return`` is allowed in a closure iterator (but rarely useful) and ends iteration. 4. Neither inline nor closure iterators can be recursive. @@ -4113,7 +4207,7 @@ The exception tree is defined in the `system `_ module. Every exception inherits from ``system.Exception``. Exceptions that indicate programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``) and are stricly speaking not catchable as they can also be mapped to an operation -that terminates the whole process. Exceptions that indicate any other runtime error +that terminates the whole process. Exceptions that indicate any other execution error that can be caught inherit from ``system.CatchableError`` (which is a subtype of ``Exception``). @@ -4351,7 +4445,7 @@ a `type variable`:idx:. Is operator ----------- -The ``is`` operator checks for type equivalence at compile time. It is +The ``is`` operator checks for type equivalence during semantic analysis. It is therefore very useful for type specialization within generic code: .. code-block:: nim @@ -5357,6 +5451,14 @@ invocation would have been replaced by its result in the source code. This can be used to implement `domain specific languages`:idx:. +Macro invocation leads to a case where semantic analyis does **not** entirely +proceed top to bottom and left to right. The compiler must + +* perform semantic analysis through the end of the macro invocation, +* execute the macro body, +* replace the AST of the macro invocation with the AST returned by the macro, +* and finally repeat semantic analysis of that region of the code. + While macros enable advanced compile-time code transformations, they cannot change Nim's syntax. However, this is no real restriction because Nim's syntax is flexible enough anyway. @@ -5639,7 +5741,8 @@ static[T] **Note**: static[T] is still in development. -As their name suggests, static parameters must be known at compile-time: +As their name suggests, static parameters must be computable during +semantic analysis: .. code-block:: nim @@ -5651,7 +5754,8 @@ As their name suggests, static parameters must be known at compile-time: # regex, stored in a global variable precompiledRegex(paramStr(1)) # Error, command-line options - # are not known at compile-time + # are not computable during + # semantic analysis For the purposes of code generation, all static params are treated as @@ -5665,7 +5769,7 @@ Static params can also appear in the signatures of generic types: type Matrix[M,N: static int; T: Number] = array[0..(M*N - 1), T] # Note how `Number` is just a type constraint here, while - # `static int` requires us to supply a compile-time int value + # `static int` requires us to supply an int value AffineTransform2D[T] = Matrix[3, 3, T] AffineTransform3D[T] = Matrix[4, 4, T] @@ -5673,13 +5777,12 @@ Static params can also appear in the signatures of generic types: var m1: AffineTransform3D[float] # OK var m2: AffineTransform2D[string] # Error, `string` is not a `Number` -Please note that ``static T`` is just a syntactic convenience for the -underlying generic type ``static[T]``. The type param can be omitted -to obtain the type class of all values known at compile-time. A more -specific type class can be created by instantiating ``static`` with -another type class. +Please note that ``static T`` is just a syntactic convenience for the underlying +generic type ``static[T]``. The type param can be omitted to obtain the type +class of all values computable during semantic analysis. A more specific type +class can be created by instantiating ``static`` with another type class. -You can force the evaluation of a certain expression at compile-time by +You can force the evaluation of a certain expression during semantic analysis by coercing it to a corresponding ``static`` type: .. code-block:: nim @@ -6766,11 +6869,11 @@ pragma block can be used: compileTime pragma ------------------ -The ``compileTime`` pragma is used to mark a proc or variable to be used at -compile time only. No code will be generated for it. Compile time procs are -useful as helpers for macros. Since version 0.12.0 of the language, a proc -that uses ``system.NimNode`` within its parameter types is implicitly declared -``compileTime``: +The ``compileTime`` pragma is used to mark a proc or variable to be used only +during compile-time execution. No code will be generated for it. Compile time +procs are useful as helpers for macros. Since version 0.12.0 of the language, a +proc that uses ``system.NimNode`` within its parameter types is implicitly +declared ``compileTime``: .. code-block:: nim proc astHelper(n: NimNode): NimNode = @@ -6858,7 +6961,7 @@ structure: pure pragma ----------- An object type can be marked with the ``pure`` pragma so that its type -field which is used for runtime type identification is omitted. This used to be +field which is used for execution-time type identification is omitted. This used to be necessary for binary compatibility with other compiled languages. An enum type can be marked as ``pure``. Then access of its fields always @@ -6884,7 +6987,7 @@ though. The ``error`` pragma can also be used to annotate a symbol (like an iterator or proc). The *usage* of the symbol then -triggers a compile-time error. This is especially useful to rule out that some +triggers a static error. This is especially useful to rule out that some operation is valid due to overloading and type conversions: .. code-block:: nim @@ -7006,7 +7109,7 @@ extension the pragma is simply ignored. unroll pragma ------------- The ``unroll`` pragma can be used to tell the compiler that it should unroll -a `for`:idx: or `while`:idx: loop for runtime efficiency: +a `for`:idx: or `while`:idx: loop for execution efficiency: .. code-block:: nim proc searchChar(s: string, c: char): int = @@ -7039,7 +7142,7 @@ others may be added later). =============== =============== ============================================ pragma allowed values description =============== =============== ============================================ -checks on|off Turns the code generation for all runtime +checks on|off Turns the code generation for all execution checks on or off. boundChecks on|off Turns the code generation for array bound checks on or off. @@ -7066,7 +7169,7 @@ Example: .. code-block:: nim {.checks: off, optimization: speed.} - # compile without runtime checks and optimize for speed + # compile without execution-time checks and optimize for speed push and pop pragmas @@ -7076,7 +7179,7 @@ but are used to override the settings temporarily. Example: .. code-block:: nim {.push checks: off.} - # compile this section without runtime checks as it is + # compile this section without execution-time checks as it is # speed critical # ... some code ... {.pop.} # restore old settings @@ -7321,7 +7424,8 @@ compiler like you would using the commandline switch ``--passC``: {.passC: "-Wall -Werror".} Note that you can use ``gorge`` from the `system module `_ to -embed parameters from an external command at compile time: +embed parameters from an external command that will be executed +during semantic analysis: .. code-block:: Nim {.passC: gorge("pkg-config --cflags sdl").} @@ -7335,7 +7439,8 @@ like you would using the commandline switch ``--passL``: {.passL: "-lSDLmain -lSDL".} Note that you can use ``gorge`` from the `system module `_ to -embed parameters from an external command at compile time: +embed parameters from an external command that will be executed +during semantic analysis: .. code-block:: Nim {.passL: gorge("pkg-config --libs sdl").} @@ -7946,7 +8051,7 @@ interoperability with C. Combining packed pragma with inheritance is not defined, and it should not be used with GC'ed memory (ref's). **Future directions**: Using GC'ed memory in packed pragma will result in -compile-time error. Usage with inheritance should be defined and documented. +a static error. Usage with inheritance should be defined and documented. Dynlib pragma for import @@ -8229,8 +8334,8 @@ Example: The parallel statement is the preferred mechanism to introduce parallelism in a Nim program. A subset of the Nim language is valid within a -``parallel`` section. This subset is checked to be free of data races at -compile time. A sophisticated `disjoint checker`:idx: ensures that no data +``parallel`` section. This subset is checked to be free of data races during +semantic analysis. A sophisticated `disjoint checker`:idx: ensures that no data races are possible even though shared memory is extensively supported! The subset is in fact the full language with the following @@ -8263,8 +8368,8 @@ pragmas: 1) A `guard`:idx: annotation is introduced to prevent data races. 2) Every access of a guarded memory location needs to happen in an appropriate `locks`:idx: statement. -3) Locks and routines can be annotated with `lock levels`:idx: to prevent - deadlocks at compile time. +3) Locks and routines can be annotated with `lock levels`:idx: to allow + potential deadlocks to be detected during static analysis. Guards and the locks section @@ -8296,9 +8401,9 @@ Top level accesses to ``gdata`` are always allowed so that it can be initialized conveniently. It is *assumed* (but not enforced) that every top level statement is executed before any concurrent action happens. -The ``locks`` section deliberately looks ugly because it has no runtime +The ``locks`` section deliberately looks ugly because it has no execution-time semantics and should not be used directly! It should only be used in templates -that also implement some form of locking at runtime: +that also implement some form of locking during execution: .. code-block:: nim template lock(a: TLock; body: untyped) = @@ -8384,9 +8489,10 @@ This means the following compiles (for now) even though it really should not: Lock levels ----------- -Lock levels are used to enforce a global locking order in order to prevent -deadlocks at compile-time. A lock level is an constant integer in the range -0..1_000. Lock level 0 means that no lock is acquired at all. +Lock levels are used to enforce a global locking order in order to detect +potential deadlocks during semantic analysis. A lock level is an constant +integer in the range 0..1_000. Lock level 0 means that no lock is acquired at +all. If a section of code holds a lock of level ``M`` than it can also acquire any lock of level ``N < M``. Another lock of level ``M`` cannot be acquired. Locks @@ -8416,7 +8522,7 @@ single ``locks`` section: Here is how a typical multilock statement can be implemented in Nim. Note how -the runtime check is required to ensure a global ordering for two locks ``a`` +the execution check is required to ensure a global ordering for two locks ``a`` and ``b`` of the same lock level: .. code-block:: nim From 03a628dddcb5676492cc2a5a34523b6176c97bd7 Mon Sep 17 00:00:00 2001 From: deansher Date: Tue, 29 Jan 2019 05:33:54 -0500 Subject: [PATCH 260/350] Solid edit pass on changes and successful rst2html; still need to review HTML output. --- doc/manual.rst | 121 ++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 20aa320a79..953f89e7ff 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -76,21 +76,21 @@ the context. A Nim `program`:idx: consists of one or more text `source files`:idx: containing Nim code. It is processed by a Nim `compiler`:idx: into an `executable`:idx:. The nature of this executable depends on the compiler implementation; it may, -for example, be a native binary or JavaScript source code. A Nim program can -contain Nim code that will be executed at `compile time`:idx:, such as constant -expressions, macro definitions, and Nim procedures used by macro definitions. -Typically, the bulk of a Nim program's code is compiled into the executable and -executed at `runtime`:idx:. Only a subset of the Nim language is supported at -compile time, but it is a substantial subset. See `Restrictions on +for example, be a native binary or JavaScript source code. In addition to the +code that is compiled into the executable and executed at `runtime`:idx:, +a Nim program can contain Nim code that will be executed at `compile time`:idx:. +This can include constant expressions, macro definitions, and Nim procedures +used by macro definitions. Most of the Nim language is supported at +compile time, but there are some restrictions -- see `Restrictions on Compile-Time Execution `_ for details. The compiler parses Nim source code into an internal data structure called the `abstract syntax tree`:idx: (`AST`:idx). Then, before executing the code or processing it into the executable, it transforms the AST through `semantic -analysis`:idx:. This adds considerable semantic information to the AST, such as -expression types, identifier meanings, and in some cases expression values. An -error detected during semantic analysis is called a `static error`:idx:. Errors -mentioned in this manual are static errors when not otherwise specified. +analysis`:idx:. This adds semantic information such as expression types, +identifier meanings, and in some cases expression values. An error detected +during semantic analysis is called a `static error`:idx:. Errors described in +this manual are static errors when not otherwise specified. An error detected during code execution (whether at compile time or at runtime) is a `checked execution error`:idx:. The method for reporting such errors is via @@ -98,10 +98,10 @@ is a `checked execution error`:idx:. The method for reporting such errors is via provides a means to disable these `execution-time checks`:idx:. See the section pragmas_ for details. -Whether a checked execution error results in an exception or in a fatal error -is implementation specific. Thus the following program is invalid; even -though it purports to catch the `IndexError` from an out-of-bounds array access, -the compiler may instead choose to allow execution to die with a fatal error. +Whether a checked execution error results in an exception or in a fatal error is +implementation specific. Thus the following program is invalid; even though the +code purports to catch the `IndexError` from an out-of-bounds array access, the +compiler may instead choose to allow execution to die with a fatal error. .. code-block:: nim var a: array[0..1, char] @@ -119,8 +119,8 @@ language features are used and if no execution-time checks are disabled. A `constant expression`:idx: is an expression whose value can be computed during semantic analysis of the code in which it appears. It is never an l-value and never has side effects. Constant expressions are not limited to the capabilities -of semantic analysis, such as constant folding; they can use the substantial -subset of the Nim language that is supported at compile time. Since constant +of semantic analysis, such as constant folding; they can use all Nim language +features that are supported for compile-time execution. Since constant expressions can be used as an input to semantic analysis (such as for defining array bounds), this flexibility requires the compiler to interleave semantic analysis and compile-time code execution. @@ -713,30 +713,32 @@ A `constant`:idx: is a symbol that is bound to the value of a `constant expression`. This is an expression whose value can be computed during semantic analysis of the code in which it appears. However, constant expressions are not limited to the capabilities of semantic analysis; they -can use the substantial subset of the Nim language that is supported for +can use all Nim language features that are supported for compile-time execution. Compile-time execution is interleaved with semantic analysis as necessary. A constant's value cannot change after it is first computed. -A constant expression must be composed of the following elements: +Constant expressions can only depend on the following values and operations +that are either built into the language or available during compilation of +the constant expression: * literals +* built-in operators * previously declared constants and compile-time variables * previously declared macros and templates * previously declared procedures that have no side effects beyond possibly modifying compile-time variables -* operators representing such procedures -* code blocks that can internally use all Nim features supported at - compile time (the next section below), but that cannot - refer to any external values beyond those listed above -Constant expressions must adhere to the restrictions on compile-time -execution described in `Restrictions on -Compile-Time Execution `_. +A constant expression can contain code blocks that may internally use all Nim +features supported at compile time (as detailed in the next section below), +but that cannot refer to any external values beyond those listed above. -For example, the following code echoes the beginning of the Fibonacci -series **at compile time**. This is a demonstration of flexibility in -defining constants, not a recommended style for solving this problem! +The ability to access and modify compile-time variables adds flexibility to +constant expressions that may be surprising to those coming from other +statically typed languages. For example, the following code echoes the beginning +of the Fibonacci series **at compile time**. (This is a demonstration of +flexibility in defining constants, not a recommended style for solving this +problem!) .. code-block:: nim :test: "nim c $1" @@ -760,11 +762,9 @@ defining constants, not a recommended style for solving this problem! const display_fib = block: const f2 = next_fib() - var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}, " - for i in 0..10: - if i > 0: - add(result, ", ") - add(result, $next_fib()) + var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}" + for i in 3..12: + add(result, fmt", {next_fib()}") result static: @@ -779,8 +779,8 @@ language features: * methods * closure iterators -* ``cast`` -* ``ptr``s or ``ref``s +* the ``cast`` operator +* reference (pointer) types * the FFI Some or all of these restrictions are likely to be lifted over time. @@ -923,8 +923,8 @@ lowest and highest value of the type: to 5. ``PositiveFloat`` defines a subrange of all positive floating point values. NaN does not belong to any subrange of floating point types. Assigning any other value to a variable of type ``Subrange`` is a -checked execution error (or static error if it can be statically -determined). Assignments from the base type to one of its subrange types +checked execution error (or static error if it can be determined during +semantic analysis). Assignments from the base type to one of its subrange types (and vice versa) are allowed. A subrange type has the same size as its base type (``int`` in the @@ -1223,9 +1223,9 @@ tuples, objects and sets belong to the structured types. Array and sequence types ------------------------ -Arrays are a homogeneous type, meaning that each element in the array -has the same type. Arrays always have a fixed length which is known during -semantic analysis (except for open arrays). They can be indexed by any ordinal type. +Arrays are a homogeneous type, meaning that each element in the array has the +same type. Arrays always have a fixed length specified as a constant expression +(except for open arrays). They can be indexed by any ordinal type. A parameter ``A`` may be an *open array*, in which case it is indexed by integers from 0 to ``len(A)-1``. An array expression may be constructed by the array constructor ``[]``. The element type of this array expression is @@ -1430,10 +1430,10 @@ can also be defined with indentation instead of ``[]``: name: string # a person consists of a name age: natural # and an age -Objects provide many features that tuples do not. Object provide inheritance -and information hiding. Objects have access to their type during execution, so that -the ``of`` operator can be used to determine the object's type. The ``of`` operator -is similar to the ``instanceof`` operator in Java. +Objects provide many features that tuples do not. Object provide inheritance and +information hiding. Objects have access to their type during execution, so that +the ``of`` operator can be used to determine the object's type. The ``of`` +operator is similar to the ``instanceof`` operator in Java. .. code-block:: nim type @@ -1528,8 +1528,8 @@ In the example the ``kind`` field is called the `discriminator`:idx:\: For safety its address cannot be taken and assignments to it are restricted: The new value must not lead to a change of the active object branch. For an object branch switch ``system.reset`` has to be used. Also, when the fields of a -particular branch are specified during object construction, the correct value -for the discriminator must be available for semantic analysis. +particular branch are specified during object construction, the corresponding +discriminator value must be specified as a constant expression. Package level objects --------------------- @@ -2842,7 +2842,7 @@ For non ordinal types it is not possible to list every possible value and so these always require an ``else`` part. Because case statements are checked for exhaustiveness during semantic analysis, -the value in every ``of`` branch must be computable during analysis. +the value in every ``of`` branch must be a constant expression. This restriction also allows the compiler to generate more performant code. As a special semantic extension, an expression in an ``of`` branch of a case @@ -4207,8 +4207,8 @@ The exception tree is defined in the `system `_ module. Every exception inherits from ``system.Exception``. Exceptions that indicate programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``) and are stricly speaking not catchable as they can also be mapped to an operation -that terminates the whole process. Exceptions that indicate any other execution error -that can be caught inherit from ``system.CatchableError`` +that terminates the whole process. Exceptions that indicate any other execution +error that can be caught inherit from ``system.CatchableError`` (which is a subtype of ``Exception``). @@ -4445,8 +4445,9 @@ a `type variable`:idx:. Is operator ----------- -The ``is`` operator checks for type equivalence during semantic analysis. It is -therefore very useful for type specialization within generic code: +The ``is`` operator is evaluated during semantic analysis to check for type +equivalence. It is therefore very useful for type specialization within generic +code: .. code-block:: nim type @@ -5741,8 +5742,7 @@ static[T] **Note**: static[T] is still in development. -As their name suggests, static parameters must be computable during -semantic analysis: +As their name suggests, static parameters must be constant expressions: .. code-block:: nim @@ -5754,8 +5754,7 @@ semantic analysis: # regex, stored in a global variable precompiledRegex(paramStr(1)) # Error, command-line options - # are not computable during - # semantic analysis + # are not constant expressions For the purposes of code generation, all static params are treated as @@ -5779,11 +5778,11 @@ Static params can also appear in the signatures of generic types: Please note that ``static T`` is just a syntactic convenience for the underlying generic type ``static[T]``. The type param can be omitted to obtain the type -class of all values computable during semantic analysis. A more specific type -class can be created by instantiating ``static`` with another type class. +class of all constant expressions. A more specific type class can be created by +instantiating ``static`` with another type class. -You can force the evaluation of a certain expression during semantic analysis by -coercing it to a corresponding ``static`` type: +You can force an expression to be evaluated at compile time as a constant +expression by coercing it to a corresponding ``static`` type: .. code-block:: nim import math @@ -6960,8 +6959,8 @@ structure: pure pragma ----------- -An object type can be marked with the ``pure`` pragma so that its type -field which is used for execution-time type identification is omitted. This used to be +An object type can be marked with the ``pure`` pragma so that its type field +which is used for execution-time type identification is omitted. This used to be necessary for binary compatibility with other compiled languages. An enum type can be marked as ``pure``. Then access of its fields always From 07a0a61875f4f5c7ac74acfd0c7dd4e4eb333dce Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 29 Jan 2019 14:31:43 +0100 Subject: [PATCH 261/350] fixes #9149 [backport] --- lib/system.nim | 10 ++++++---- tests/overload/tconverter_to_string.nim | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 tests/overload/tconverter_to_string.nim diff --git a/lib/system.nim b/lib/system.nim index 0241b92e08..4951961cac 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -4307,15 +4307,17 @@ proc `==`*(x, y: cstring): bool {.magic: "EqCString", noSideEffect, when defined(nimNoNilSeqs2): when not compileOption("nilseqs"): when defined(nimHasUserErrors): - proc `==`*(x: string; y: type(nil)): bool {. + # bug #9149; ensure that 'type(nil)' does not match *too* well by using 'type(nil) | type(nil)'. + # Eventually (in 0.20?) we will be able to remove this hack completely. + proc `==`*(x: string; y: type(nil) | type(nil)): bool {. error: "'nil' is now invalid for 'string'; compile with --nilseqs:on for a migration period".} = discard - proc `==`*(x: type(nil); y: string): bool {. + proc `==`*(x: type(nil) | type(nil); y: string): bool {. error: "'nil' is now invalid for 'string'; compile with --nilseqs:on for a migration period".} = discard else: - proc `==`*(x: string; y: type(nil)): bool {.error.} = discard - proc `==`*(x: type(nil); y: string): bool {.error.} = discard + proc `==`*(x: string; y: type(nil) | type(nil)): bool {.error.} = discard + proc `==`*(x: type(nil) | type(nil); y: string): bool {.error.} = discard template closureScope*(body: untyped): untyped = ## Useful when creating a closure in a loop to capture local loop variables by diff --git a/tests/overload/tconverter_to_string.nim b/tests/overload/tconverter_to_string.nim new file mode 100644 index 0000000000..1960372d8a --- /dev/null +++ b/tests/overload/tconverter_to_string.nim @@ -0,0 +1,22 @@ +discard """ + output: '''123 +c is not nil''' +""" + +# bug #9149 + +type + Container = ref object + data: int + +converter containerToString*(x: Container): string = $x.data + +var c = Container(data: 123) +var str = string c +echo str + +if c == nil: # this line can compile on v0.18, but not on 0.19 + echo "c is nil" + +if not c.isNil: + echo "c is not nil" From 15422a3e5a24d6c10d1f713cff7e04289bf7a232 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 29 Jan 2019 15:08:43 +0100 Subject: [PATCH 262/350] Genode fixes (#10491) Readline pasthru, add linker to config, do not pass -lm to linker. --- config/nim.cfg | 5 ++++- lib/impure/rdstdin.nim | 9 +++++++++ lib/pure/fenv.nim | 2 +- lib/pure/math.nim | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 2a118c5cf3..c3e0823b65 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -290,13 +290,16 @@ tcc.options.always = "-w" # Configuration for the Genode toolchain @if genode: + noCppExceptions # avoid std C++ + tlsEmulation:on # no TLS segment register magic gcc.path = "/usr/local/genode-gcc/bin" - gcc.cpp.options.always = "-D__GENODE__ -fno-stack-protector" @if i386 or amd64: gcc.exe = "genode-x86-gcc" gcc.cpp.exe = "genode-x86-g++" + gcc.cpp.linkerexe = "genode-x86-ld" @elif arm: gcc.exe = "genode-arm-gcc" gcc.cpp.exe = "genode-arm-g++" + gcc.cpp.linkerexe = "genode-arm-ld" @end @end diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 54bab82f0b..ac38addbac 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -73,6 +73,15 @@ when defined(Windows): discard readConsoleInputW(hStdin, irInputRecord, 1, dwEventsRead) return result +elif defined(genode): + proc readLineFromStdin*(prompt: string): TaintedString {. + tags: [ReadIOEffect, WriteIOEffect].} = + stdin.readLine() + + proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + tags: [ReadIOEffect, WriteIOEffect].} = + stdin.readLine(line) + else: import linenoise, termios diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index 0725973cad..ab47da08ea 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -12,7 +12,7 @@ {.deadCodeElim: on.} # dce option deprecated -when defined(Posix): +when defined(Posix) and not defined(genode): {.passl: "-lm".} var diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 460be1cd0e..526ddbbb2e 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -95,7 +95,7 @@ proc fac*(n: int): int = {.push checks:off, line_dir:off, stack_trace:off.} -when defined(Posix): +when defined(Posix) and not defined(genode): {.passl: "-lm".} const From dee8e6e98ae868b8d933a718250c8e471bc125ea Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Jan 2019 15:12:16 +0100 Subject: [PATCH 263/350] gc: destructors is beginning to work (#10483) * kochdocs.nim: code cleanup * docgen: nicer indentation * parser.nim: code cleanup * fixes #10458 * make tests green again * make =destroy mixins * gc:destructors: produced C code is almost working * --gc:destructors simple program compiles (but leaks memory) * gc:destructors make examples compile in C++ mode * destructors: string implementation bugfixes * strs.nim: minor code cleanup * destructors: builtin seqs are beginning to work * remove debugging helpers --- compiler/ast.nim | 1 - compiler/ccgexprs.nim | 6 ++-- compiler/cgen.nim | 6 ++++ compiler/destroyer.nim | 14 +++++++-- compiler/parser.nim | 48 +++++++++++++---------------- compiler/semasgn.nim | 5 +++ compiler/semstmts.nim | 3 +- compiler/semtypes.nim | 17 +++++----- compiler/semtypinst.nim | 37 ++++++++++++++-------- compiler/sigmatch.nim | 5 +++ lib/core/seqs.nim | 10 ++++-- lib/core/strs.nim | 17 +++++----- lib/system.nim | 36 +++++++++++----------- lib/system/excpt.nim | 21 +++++++------ lib/system/gc_regions.nim | 20 +++++++++--- lib/system/helpers2.nim | 4 +-- tests/misc/tinvalidarrayaccess.nim | 2 +- tests/misc/tinvalidarrayaccess2.nim | 2 +- tests/parser/tprecedence.nim | 9 ++++++ 19 files changed, 162 insertions(+), 101 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 24891d6d3e..fc470b7a89 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1352,7 +1352,6 @@ proc copySym*(s: PSym): PSym = result = newSym(s.kind, s.name, s.owner, s.info, s.options) #result.ast = nil # BUGFIX; was: s.ast which made problems result.typ = s.typ - result.id = getID() when debugIds: registerId(result) result.flags = s.flags result.magic = s.magic diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ed62550040..5bcbcda1c2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -460,7 +460,7 @@ proc binaryStmtAddr(p: BProc, e: PNode, d: var TLoc, frmt: string) = if d.k != locNone: internalError(p.config, e.info, "binaryStmtAddr") initLocExpr(p, e.sons[1], a) initLocExpr(p, e.sons[2], b) - lineCg(p, cpsStmts, frmt, addrLoc(p.config, a), rdLoc(b)) + lineCg(p, cpsStmts, frmt, byRefLoc(p, a), rdLoc(b)) proc unaryStmt(p: BProc, e: PNode, d: var TLoc, frmt: string) = var a: TLoc @@ -1028,7 +1028,7 @@ proc gcUsage(conf: ConfigRef; n: PNode) = proc strLoc(p: BProc; d: TLoc): Rope = if p.config.selectedGc == gcDestructors: - result = addrLoc(p.config, d) + result = byRefLoc(p, d) else: result = rdLoc(d) @@ -1110,7 +1110,7 @@ proc genStrAppend(p: BProc, e: PNode, d: var TLoc) = strLoc(p, dest), rdLoc(a))) if p.config.selectedGC == gcDestructors: linefmt(p, cpsStmts, "#prepareAdd($1, $2$3);$n", - addrLoc(p.config, dest), lens, rope(L)) + byRefLoc(p, dest), lens, rope(L)) else: initLoc(call, locCall, e, OnHeap) call.r = ropecg(p.module, "#resizeString($1, $2$3)", [rdLoc(dest), lens, rope(L)]) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 2d9814621d..d020b1bd7d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -264,6 +264,12 @@ proc addrLoc(conf: ConfigRef; a: TLoc): Rope = if lfIndirect notin a.flags and mapType(conf, a.t) != ctArray: result = "(&" & result & ")" +proc byRefLoc(p: BProc; a: TLoc): Rope = + result = a.r + if lfIndirect notin a.flags and mapType(p.config, a.t) != ctArray and not + p.module.compileToCpp: + result = "(&" & result & ")" + proc rdCharLoc(a: TLoc): Rope = # read a location that may need a char-cast: result = rdLoc(a) diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index e21d532ea9..22ace36349 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -244,7 +244,10 @@ proc patchHead(n: PNode) = proc patchHead(s: PSym) = if sfFromGeneric in s.flags: - patchHead(s.ast[bodyPos]) + # do not patch the builtin type bound operators for seqs: + let dest = s.typ.sons[1].skipTypes(abstractVar) + if dest.kind != tySequence: + patchHead(s.ast[bodyPos]) proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) = var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">" @@ -267,7 +270,8 @@ template genOp(opr, opname, ri) = globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t)) elif op.ast[genericParamsPos].kind != nkEmpty: - globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator is generic") + globalError(c.graph.config, dest.info, "internal error: '" & opname & + "' operator is generic") patchHead op if sfError in op.flags: checkForErrorPragma(c, t, ri, opname) let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ)) @@ -275,6 +279,12 @@ template genOp(opr, opname, ri) = result = newTree(nkCall, newSymNode(op), addrExp) proc genSink(c: Con; t: PType; dest, ri: PNode): PNode = + when false: + if t.kind != tyString: + echo "this one ", c.graph.config$dest.info, " for ", typeToString(t, preferDesc) + debug t.sink.typ.sons[2] + echo t.sink.id, " owner ", t.id + quit 1 let t = t.skipTypes({tyGenericInst, tyAlias, tySink}) genOp(if t.sink != nil: t.sink else: t.assignment, "=sink", ri) diff --git a/compiler/parser.nim b/compiler/parser.nim index c9626c527c..01a3ce4d00 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -724,6 +724,14 @@ const tkTypeClasses = {tkRef, tkPtr, tkVar, tkStatic, tkType, tkEnum, tkTuple, tkObject, tkProc} +proc commandExpr(p: var TParser; r: PNode; mode: TPrimaryMode): PNode = + result = newNodeP(nkCommand, p) + addSon(result, r) + var isFirstParam = true + # progress NOT guaranteed + p.hasProgress = false + addSon result, commandParam(p, isFirstParam, mode) + proc primarySuffix(p: var TParser, r: PNode, baseIndent: int, mode: TPrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? @@ -734,8 +742,6 @@ proc primarySuffix(p: var TParser, r: PNode, #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax result = r - template somePar() = - if p.tok.strongSpaceA > 0: break # progress guaranteed while p.tok.indent < 0 or (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): @@ -749,6 +755,8 @@ proc primarySuffix(p: var TParser, r: PNode, result = newNodeP(nkCommand, p) result.addSon r result.addSon primary(p, pmNormal) + else: + result = commandExpr(p, result, mode) break result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result.sons[1].kind == nkExprColonExpr: @@ -759,39 +767,27 @@ proc primarySuffix(p: var TParser, r: PNode, result = parseGStrLit(p, result) of tkBracketLe: # progress guaranteed - somePar() + if p.tok.strongSpaceA > 0: + result = commandExpr(p, result, mode) + break result = namedParams(p, result, nkBracketExpr, tkBracketRi) of tkCurlyLe: # progress guaranteed - somePar() + if p.tok.strongSpaceA > 0: + result = commandExpr(p, result, mode) + break result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkOpr, tkDotDot, tkTypeClasses - {tkRef, tkPtr}: - # XXX: In type sections we allow the free application of the - # command syntax, with the exception of expressions such as - # `foo ref` or `foo ptr`. Unfortunately, these two are also - # used as infix operators for the memory regions feature and - # the current parsing rules don't play well here. + # XXX: In type sections we allow the free application of the + # command syntax, with the exception of expressions such as + # `foo ref` or `foo ptr`. Unfortunately, these two are also + # used as infix operators for the memory regions feature and + # the current parsing rules don't play well here. if p.inPragma == 0 and (isUnary(p) or p.tok.tokType notin {tkOpr, tkDotDot}): # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet # solution, but pragmas.nim can't handle that - let a = result - result = newNodeP(nkCommand, p) - addSon(result, a) - var isFirstParam = true - when true: - # progress NOT guaranteed - p.hasProgress = false - addSon result, commandParam(p, isFirstParam, mode) - if not p.hasProgress: break - else: - while p.tok.tokType != tkEof: - let x = parseExpr(p) - addSon(result, x) - if p.tok.tokType != tkComma: break - getTok(p) - optInd(p, x) - result = postExprBlocks(p, result) + result = commandExpr(p, result, mode) break else: break diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 9f1ef313b7..41b0879e66 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -316,6 +316,11 @@ proc liftBody(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym = if typ.kind == tyDistinct: return liftBodyDistinctType(g, typ, kind, info) + when false: + var typ = typ + if c.config.selectedGC == gcDestructors and typ.kind == tySequence: + # use the canonical type to access the =sink and =destroy etc. + typ = c.graph.sysTypes[tySequence] var a: TLiftCtx a.info = info diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 5e9d5d9c53..f1778e8161 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -168,7 +168,7 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode = else: illFormedAst(it, c.config) if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or (not hasElse and efInTypeof notin flags): - for it in n: + for it in n: it.sons[^1] = discardCheck(c, it.sons[^1], flags) result.kind = nkIfStmt # propagate any enforced VoidContext: @@ -1563,6 +1563,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = if obj.kind in {tyObject, tyDistinct, tySequence, tyString} and sameType(obj, objB): # attach these ops to the canonical tySequence obj = canonType(c, obj) + #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj) let opr = if s.name.s == "=": addr(obj.assignment) else: addr(obj.sink) if opr[].isNil: opr[] = s diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index fbf3638347..7447463238 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1159,7 +1159,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, # compiler only checks for 'nil': if skipTypes(r, {tyGenericInst, tyAlias, tySink}).kind != tyVoid: if kind notin {skMacro, skTemplate} and r.kind in {tyStmt, tyExpr}: - localError(c.config, n.sons[0].info, "return type '" & typeToString(r) & + localError(c.config, n.sons[0].info, "return type '" & typeToString(r) & "' is only valid for macros and templates") # 'auto' as a return type does not imply a generic: elif r.kind == tyAnything: @@ -1577,11 +1577,16 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = assert s != nil assert prev == nil result = copyType(s, s.owner, keepId=false) - # XXX figure out why this has children already... + # Remove the 'T' parameter from tySequence: result.sons.setLen 0 result.n = nil result.flags = {tfHasAsgn} semContainerArg(c, n, "seq", result) + if result.len > 0: + var base = result[0] + if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base) + if base.kind != tyGenericParam: + c.typesWithOps.add((result, result)) else: result = semContainer(c, n, tySequence, "seq", prev) if c.config.selectedGc == gcDestructors: @@ -1714,11 +1719,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = newOrPrevType(tyError, prev, c) n.typ = result dec c.inTypeContext - if c.inTypeContext == 0: instAllTypeBoundOp(c, n.info) - -when false: - proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = - result = semTypeNodeInner(c, n, prev) + if c.inTypeContext == 0: + #if $n == "var seq[StackTraceEntry]": + # echo "begin ", n instAllTypeBoundOp(c, n.info) proc setMagicType(conf: ConfigRef; m: PSym, kind: TTypeKind, size: int) = diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 027ffd4aa8..ebe822cdfc 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -297,12 +297,6 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType = #result.destructor = nil result.sink = nil -template typeBound(c, newty, oldty, field, info) = - let opr = newty.field - if opr != nil and sfFromGeneric notin opr.flags: - # '=' needs to be instantiated for generics when the type is constructed: - newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1) - proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = # tyGenericInvocation[A, tyGenericInvocation[A, B]] # is difficult to handle: @@ -317,7 +311,10 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = else: result = searchInstTypes(t) - if result != nil and eqFlags*result.flags == eqFlags*t.flags: return + if result != nil and eqFlags*result.flags == eqFlags*t.flags: + when defined(reportCacheHits): + echo "Generic instantiation cached ", typeToString(result), " for ", typeToString(t) + return for i in countup(1, sonsLen(t) - 1): var x = t.sons[i] if x.kind in {tyGenericParam}: @@ -332,7 +329,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = if header != t: # search again after first pass: result = searchInstTypes(header) - if result != nil and eqFlags*result.flags == eqFlags*t.flags: return + if result != nil and eqFlags*result.flags == eqFlags*t.flags: + when defined(reportCacheHits): + echo "Generic instantiation cached ", typeToString(result), " for ", + typeToString(t), " header ", typeToString(header) + return else: header = instCopyType(cl, t) @@ -384,7 +385,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = rawAddSon(result, newbody) checkPartialConstructedType(cl.c.config, cl.info, newbody) let dc = newbody.deepCopy - if cl.allowMetaTypes == false: + if not cl.allowMetaTypes: if dc != nil and sfFromGeneric notin newbody.deepCopy.flags: # 'deepCopy' needs to be instantiated for # generics *when the type is constructed*: @@ -402,6 +403,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = discard else: newbody.lastSon.typeInst = result + # DESTROY: adding object|opt for opt[topttree.Tree] + # sigmatch: Formal opt[=destroy.T] real opt[topttree.Tree] + # adding myseq for myseq[system.int] + # sigmatch: Formal myseq[=destroy.T] real myseq[system.int] + #echo "DESTROY: adding ", typeToString(newbody), " for ", typeToString(result, preferDesc) cl.c.typesWithOps.add((newbody, result)) let mm = skipTypes(bbody, abstractPtrs) if tfFromGeneric notin mm.flags: @@ -432,7 +438,7 @@ proc eraseVoidParams*(t: PType) = inc pos setLen t.sons, pos setLen t.n.sons, pos - return + break proc skipIntLiteralParams*(t: PType) = for i in 0 ..< t.sonsLen: @@ -561,9 +567,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = for i in countup(0, sonsLen(result) - 1): if result.sons[i] != nil: if result.sons[i].kind == tyGenericBody: - localError( - cl.c.config, - t.sym.info, + localError(cl.c.config, t.sym.info, "cannot instantiate '" & typeToString(result.sons[i], preferDesc) & "' inside of type definition: '" & @@ -603,6 +607,13 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = result.size = -1 result.n = replaceObjBranches(cl, result.n) +template typeBound(c, newty, oldty, field, info) = + let opr = newty.field + if opr != nil and sfFromGeneric notin opr.flags: + # '=' needs to be instantiated for generics when the type is constructed: + #echo "DESTROY: instantiating ", astToStr(field), " for ", typeToString(oldty) + newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1) + proc instAllTypeBoundOp*(c: PContext, info: TLineInfo) = var i = 0 while i < c.typesWithOps.len: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index fa4ab3703f..3eaac06e51 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2505,6 +2505,11 @@ proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; if f.kind in {tyRef, tyPtr}: f = f.lastSon else: if f.kind == tyVar: f = f.lastSon + #if c.config.selectedGC == gcDestructors and f.kind == tySequence: + # use the canonical type to access the =sink and =destroy etc. + # f = c.graph.sysTypes[tySequence] + #echo "YUP_---------Formal ", typeToString(f, preferDesc), " real ", typeToString(t, preferDesc), " ", f.id, " ", t.id + if typeRel(m, f, t) == isNone: localError(c.config, info, "cannot instantiate: '" & dc.name.s & "'") else: diff --git a/lib/core/seqs.nim b/lib/core/seqs.nim index 977b23b26a..1a81b89ea0 100644 --- a/lib/core/seqs.nim +++ b/lib/core/seqs.nim @@ -15,7 +15,7 @@ proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} ## Default seq implementation used by Nim's core. type - NimSeqPayload {.core.}[T] = object + NimSeqPayload[T] = object cap: int region: Allocator data: UncheckedArray[T] @@ -40,6 +40,7 @@ proc `=destroy`[T](s: var seq[T]) = var x = cast[ptr NimSeqV2[T]](addr s) var p = x.p if p != nil: + mixin `=destroy` when not supportsCopyMem(T): for i in 0.. 0: # also copy the \0 terminator: copyMem(unsafeAddr dest.p.data[dest.len], unsafeAddr src.p.data[0], src.len+1) + inc dest.len, src.len proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} = dest.p.data[dest.len] = c @@ -166,7 +164,6 @@ proc mnewString(len: int): NimStringV2 {.compilerProc.} = proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} = if newLen > s.len: prepareAdd(s, newLen - s.len) - else: - s.len = newLen - # this also only works because the destructor - # looks at s.p and not s.len + s.len = newLen + # this also only works because the destructor + # looks at s.p and not s.len diff --git a/lib/system.nim b/lib/system.nim index 4951961cac..a7cf251f64 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3049,6 +3049,19 @@ else: if x < 0: -x else: x {.pop.} +when defined(nimNewRoof): + iterator `..<`*[T](a, b: T): T = + var i = T(a) + while i < b: + yield i + inc i +else: + iterator `..<`*[S, T](a: S, b: T): T = + var i = T(a) + while i < b: + yield i + inc i + when not defined(JS): proc likelyProc(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} proc unlikelyProc(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} @@ -3144,7 +3157,7 @@ when not defined(JS): #and not defined(nimscript): # ----------------- IO Part ------------------------------------------------ type CFile {.importc: "FILE", header: "", - final, incompletestruct.} = object + incompletestruct.} = object File* = ptr CFile ## The type representing a file handle. FileMode* = enum ## The file mode when opening a file. @@ -3392,6 +3405,10 @@ when not defined(JS): #and not defined(nimscript): ## returns the OS file handle of the file ``f``. This is only useful for ## platform specific programming. + when defined(gcDestructors) and not defined(nimscript): + include "core/strs" + include "core/seqs" + when declared(newSeq): proc cstringArrayToSeq*(a: cstringArray, len: Natural): seq[string] = ## converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be @@ -3483,10 +3500,6 @@ when not defined(JS): #and not defined(nimscript): when defined(memtracker): include "system/memtracker" - when defined(gcDestructors): - include "core/strs" - include "core/seqs" - when hostOS == "standalone": include "system/embedded" else: @@ -3716,19 +3729,6 @@ template `..<`*(a, b: untyped): untyped = ## a shortcut for 'a .. (when b is BackwardsIndex: succ(b) else: pred(b))'. a .. (when b is BackwardsIndex: succ(b) else: pred(b)) -when defined(nimNewRoof): - iterator `..<`*[T](a, b: T): T = - var i = T(a) - while i < b: - yield i - inc i -else: - iterator `..<`*[S, T](a: S, b: T): T = - var i = T(a) - while i < b: - yield i - inc i - template spliceImpl(s, a, L, b: untyped): untyped = # make room for additional elements or cut: var shift = b.len - max(0,L) # ignore negative slice size diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index cc0c1f54bd..f2f82c3b8d 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -220,11 +220,12 @@ proc auxWriteStackTrace(f: PFrame; s: var seq[StackTraceEntry]) = inc(i) it = it.prev var last = i-1 - if s.len == 0: - s = newSeq[StackTraceEntry](i) - else: - last = s.len + i - 1 - s.setLen(last+1) + when true: # not defined(gcDestructors): + if s.len == 0: + s = newSeq[StackTraceEntry](i) + else: + last = s.len + i - 1 + s.setLen(last+1) it = f while it != nil: s[last] = StackTraceEntry(procname: it.procname, @@ -440,11 +441,13 @@ proc getStackTrace(e: ref Exception): string = else: result = "" -when not defined(gcDestructors): - proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = - ## Returns the attached stack trace to the exception ``e`` as - ## a ``seq``. This is not yet available for the JS backend. +proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = + ## Returns the attached stack trace to the exception ``e`` as + ## a ``seq``. This is not yet available for the JS backend. + when not defined(gcDestructors): shallowCopy(result, e.trace) + else: + result = move(e.trace) const nimCallDepthLimit {.intdefine.} = 2000 diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 59f68918fd..797eeeebf5 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -195,6 +195,19 @@ proc runFinalizers(c: Chunk) = (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) it = it.nextFinal +proc runFinalizers(c: Chunk; newbump: pointer) = + var it = c.head + var prev: ptr ObjHeader = nil + while it != nil: + let nxt = it.nextFinal + if it >= newbump: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + elif prev != nil: + prev.nextFinal = nil + prev = it + it = nxt + proc dealloc(r: var MemRegion; p: pointer; size: int) = let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) if it.typ != nil and it.typ.finalizer != nil: @@ -237,16 +250,15 @@ template computeRemaining(r): untyped = proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = # free everything after 'sp': - if sp.current.next != nil: + if sp.current != nil and sp.current.next != nil: deallocAll(r, sp.current.next) sp.current.next = nil when false: # better leak this memory than be sorry: for i in 0..high(r.freeLists): r.freeLists[i] = nil r.holes = nil - #else: - # deallocAll(r, r.head) - # r.head = nil + if r.tail != nil: runFinalizers(r.tail, sp.bump) + r.bump = sp.bump r.tail = sp.current r.remaining = sp.remaining diff --git a/lib/system/helpers2.nim b/lib/system/helpers2.nim index c67a2c2785..8bd69ad711 100644 --- a/lib/system/helpers2.nim +++ b/lib/system/helpers2.nim @@ -1,7 +1,7 @@ # imported by other modules, unlike helpers.nim which is included template formatErrorIndexBound*[T](i, a, b: T): string = - "index out of bounds: (a:" & $a & ") <= (i:" & $i & ") <= (b:" & $b & ") " + "index out of bounds: (a: " & $a & ") <= (i: " & $i & ") <= (b: " & $b & ") " template formatErrorIndexBound*[T](i, n: T): string = - "index out of bounds: (i:" & $i & ") <= (n:" & $n & ") " + "index out of bounds: (i: " & $i & ") <= (n: " & $n & ") " diff --git a/tests/misc/tinvalidarrayaccess.nim b/tests/misc/tinvalidarrayaccess.nim index 57ad38b85e..ab44d98e88 100644 --- a/tests/misc/tinvalidarrayaccess.nim +++ b/tests/misc/tinvalidarrayaccess.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "index out of bounds: (a:0) <= (i:2) <= (b:1) " + errormsg: "index out of bounds: (a: 0) <= (i: 2) <= (b: 1) " line: 18 """ diff --git a/tests/misc/tinvalidarrayaccess2.nim b/tests/misc/tinvalidarrayaccess2.nim index 86d3494577..a791dc4e7f 100644 --- a/tests/misc/tinvalidarrayaccess2.nim +++ b/tests/misc/tinvalidarrayaccess2.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "index out of bounds: (a:0) <= (i:3) <= (b:1) " + errormsg: "index out of bounds: (a: 0) <= (i: 3) <= (b: 1) " line: 9 """ diff --git a/tests/parser/tprecedence.nim b/tests/parser/tprecedence.nim index aff7c6aca4..3e1c03dd18 100644 --- a/tests/parser/tprecedence.nim +++ b/tests/parser/tprecedence.nim @@ -40,3 +40,12 @@ proc getX(x: MyObject): lent MyField {.inline.} = let a = MyObject() echo a.getX.b.len + + +# bug #10458 +template t(x: untyped): untyped = "x" + +let + aaa = t 2 + 4 + ccc = t (1, 1) + 6 + ddd = t [0, 1, 2] + 5 From b2a5195e92f8852a47ca70136b103bc374b73393 Mon Sep 17 00:00:00 2001 From: c-blake Date: Tue, 29 Jan 2019 15:25:08 -0500 Subject: [PATCH 264/350] Add a standard Rusage type definition and wait4, getrusage declarations (#10484) --- changelog.md | 2 ++ lib/posix/posix.nim | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/changelog.md b/changelog.md index 85e06112bb..9382a194f9 100644 --- a/changelog.md +++ b/changelog.md @@ -124,6 +124,8 @@ proc enumToString*(enums: openArray[enum]): string = - Added `xmltree.toXmlAttributes`. +- Added `Rusage`, `getrusage`, `wait4` to posix interface. + ### Library changes diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 175f6a61dd..800188b8fc 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -663,6 +663,26 @@ proc waitid*(a1: cint, a2: Id, a3: var SigInfo, a4: cint): cint {. proc waitpid*(a1: Pid, a2: var cint, a3: cint): Pid {. importc, header: "".} +type Rusage* {.importc: "struct rusage", header: "", + bycopy.} = object + ru_utime*, ru_stime*: Timeval # User and system time + ru_maxrss*, ru_ixrss*, ru_idrss*, ru_isrss*, # memory sizes + ru_minflt*, ru_majflt*, ru_nswap*, # paging activity + ru_inblock*, ru_oublock*, ru_msgsnd*, ru_msgrcv*, # IO activity + ru_nsignals*, ru_nvcsw*, ru_nivcsw*: clong # switching activity + +proc wait4*(pid: Pid, status: ptr cint, options: cint, rusage: ptr Rusage): Pid + {.importc, header: "".} + +const + RUSAGE_SELF* = cint(0) + RUSAGE_CHILDREN* = cint(-1) + RUSAGE_THREAD* = cint(1) # This one is less std; Linux, BSD agree though. + +# This can only fail if `who` is invalid or `rusage` ptr is invalid. +proc getrusage*(who: cint, rusage: ptr Rusage): cint + {.importc, header: "", discardable.} + proc bsd_signal*(a1: cint, a2: proc (x: pointer) {.noconv.}) {. importc, header: "".} proc kill*(a1: Pid, a2: cint): cint {.importc, header: "".} From 7295468516efedb18fcd28ee34e2c7830395dfe9 Mon Sep 17 00:00:00 2001 From: deansher Date: Tue, 29 Jan 2019 19:43:01 -0500 Subject: [PATCH 265/350] Finished this terminology update. --- doc/manual.rst | 88 +++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 953f89e7ff..b9d5499b22 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -46,8 +46,8 @@ and ``a ^* b`` is short for ``(a (b a)*)?``. Example:: arrayConstructor = '[' expr ^* ',' ']' -Other parts of Nim - like scoping rules or execution semantics are only -described in the, more easily comprehensible, informal manner for now. +Other parts of Nim, like scoping rules or execution semantics, are +described informally. @@ -76,17 +76,19 @@ the context. A Nim `program`:idx: consists of one or more text `source files`:idx: containing Nim code. It is processed by a Nim `compiler`:idx: into an `executable`:idx:. The nature of this executable depends on the compiler implementation; it may, -for example, be a native binary or JavaScript source code. In addition to the -code that is compiled into the executable and executed at `runtime`:idx:, -a Nim program can contain Nim code that will be executed at `compile time`:idx:. -This can include constant expressions, macro definitions, and Nim procedures -used by macro definitions. Most of the Nim language is supported at -compile time, but there are some restrictions -- see `Restrictions on -Compile-Time Execution `_ for details. +for example, be a native binary or JavaScript source code. + +In a typical Nim program, most of the code is compiled into the executable for +execution at `runtime`:idx:. However, some of the code may be executed at +`compile time`:idx:. This can include constant expressions, macro definitions, +and Nim procedures used by macro definitions. Most of the Nim language is +supported at compile time, but there are some restrictions -- see `Restrictions +on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for +details. The compiler parses Nim source code into an internal data structure called the -`abstract syntax tree`:idx: (`AST`:idx). Then, before executing the code or -processing it into the executable, it transforms the AST through `semantic +`abstract syntax tree`:idx: (`AST`:idx:). Then, before executing the code or +compiling it into the executable, it transforms the AST through `semantic analysis`:idx:. This adds semantic information such as expression types, identifier meanings, and in some cases expression values. An error detected during semantic analysis is called a `static error`:idx:. Errors described in @@ -128,11 +130,9 @@ analysis and compile-time code execution. It is mostly accurate to picture semantic analysis proceeding top to bottom and left to right in the source code, with compile-time code execution interleaved when necessary to compute values that are required for subsequent semantic -analysis. We saw just above that this interleaving is necessary for handling -constant expressions. We will see much later in this document that macro -invocation not only requires this interleaving, but also creates a situation -where semantic analyis does not entirely proceed top to bottom and left to -right. +analysis. We will see much later in this document that macro invocation not only +requires this interleaving, but also creates a situation where semantic analyis +does not entirely proceed top to bottom and left to right. Lexical Analysis @@ -709,8 +709,8 @@ Rationale: Consistency with overloaded assignment or assignment-like operations, Constants and Constant Expressions ================================== -A `constant`:idx: is a symbol that is bound to the value of a `constant -expression`. This is an expression whose value can be computed during +A `constant`:idx: is a symbol that is bound to the value of a constant +expression. This is an expression whose value can be computed during semantic analysis of the code in which it appears. However, constant expressions are not limited to the capabilities of semantic analysis; they can use all Nim language features that are supported for @@ -718,9 +718,9 @@ compile-time execution. Compile-time execution is interleaved with semantic analysis as necessary. A constant's value cannot change after it is first computed. -Constant expressions can only depend on the following values and operations -that are either built into the language or available during compilation of -the constant expression: +Constant expressions are restricted to depend only on the following categories +of values and operations, because these are either built into the language or +declared and evaluated before semantic analysis of the constant expression: * literals * built-in operators @@ -730,8 +730,12 @@ the constant expression: possibly modifying compile-time variables A constant expression can contain code blocks that may internally use all Nim -features supported at compile time (as detailed in the next section below), -but that cannot refer to any external values beyond those listed above. +features supported at compile time (as detailed in the next section below). +Within such a code block, it is possible to declare variables and then later +read and update them, or declare variables and pass them to procedures that +modify them. However, the code in such a block must still adhere to the +retrictions listed above for referencing values and operations outside the +block. The ability to access and modify compile-time variables adds flexibility to constant expressions that may be surprising to those coming from other @@ -774,7 +778,7 @@ problem!) Restrictions on Compile-Time Execution ====================================== -Nim code that will be executed at compile-time cannot use the following +Nim code that will be executed at compile time cannot use the following language features: * methods @@ -910,8 +914,8 @@ For further details, see `Convertible relation Subrange types -------------- A subrange type is a range of values from an ordinal or floating point type (the base -type). To define a subrange type, one must specify it's limiting values: the -lowest and highest value of the type: +type). To define a subrange type, one must specify its limiting values -- the +lowest and highest value of the type. For example: .. code-block:: nim type @@ -1543,8 +1547,8 @@ contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does not yet know the size of the object. To complete an incomplete object the ``package`` pragma has to be used. ``package`` implies ``byref``. -As long as a type ``T`` is incomplete ``sizeof(T)`` or "execution-time type -information" for ``T`` is not available. +As long as a type ``T`` is incomplete, neither ``sizeof(T)`` nor execution-time +type information for ``T`` is available. Example: @@ -2363,6 +2367,10 @@ In a call ``p(args)`` the routine ``p`` that matches best is selected. If multiple routines match equally well, the ambiguity is reported during semantic analysis. +Every arg in args needs to match. There are multiple different categories how an +argument can match. Let ``f`` be the formal parameter's type and ``a`` the type +of the argument. + 1. Exact match: ``a`` and ``f`` are of the same type. 2. Literal match: ``a`` is an integer literal of value ``v`` and ``f`` is a signed or unsigned integer type and ``v`` is in ``f``'s @@ -2764,7 +2772,7 @@ Even some code that has side effects is permitted in a static block: There are limitations on what Nim code can be executed at compile time; see `Restrictions on Compile-Time Execution -<#restrictions-on-compile-time-execution>`_ for details. +<#restrictions-on-compileminustime-execution>`_ for details. It's a static error if the compiler cannot execute the block at compile time. @@ -3767,7 +3775,7 @@ Invocation of a multi-method cannot be ambiguous: collide 2 is preferred over collide 1 because the resolution works from left to right. In the example ``Unit, Thing`` is preferred over ``Thing, Unit``. -**Note**: Compile time execution is not (yet) supported for methods. +**Note**: Compile-time execution is not (yet) supported for methods. Inhibit dynamic method resolution via procCall @@ -5443,7 +5451,7 @@ chance to convert it into a sequence. Macros ====== -A macro is a special function that is executed at compile-time. +A macro is a special function that is executed at compile time. Normally the input for a macro is an abstract syntax tree (AST) of the code that is passed to it. The macro can then do transformations on it and return the transformed AST. The @@ -5452,8 +5460,8 @@ invocation would have been replaced by its result in the source code. This can be used to implement `domain specific languages`:idx:. -Macro invocation leads to a case where semantic analyis does **not** entirely -proceed top to bottom and left to right. The compiler must +Macro invocation is a case where semantic analyis does **not** entirely proceed +top to bottom and left to right. The compiler must * perform semantic analysis through the end of the macro invocation, * execute the macro body, @@ -6869,7 +6877,7 @@ pragma block can be used: compileTime pragma ------------------ The ``compileTime`` pragma is used to mark a proc or variable to be used only -during compile-time execution. No code will be generated for it. Compile time +during compile-time execution. No code will be generated for it. Compile-time procs are useful as helpers for macros. Since version 0.12.0 of the language, a proc that uses ``system.NimNode`` within its parameter types is implicitly declared ``compileTime``: @@ -8331,11 +8339,11 @@ Example: echo formatFloat(pi(5000)) -The parallel statement is the preferred mechanism to introduce parallelism -in a Nim program. A subset of the Nim language is valid within a -``parallel`` section. This subset is checked to be free of data races during -semantic analysis. A sophisticated `disjoint checker`:idx: ensures that no data -races are possible even though shared memory is extensively supported! +The parallel statement is the preferred mechanism to introduce parallelism in a +Nim program. A subset of the Nim language is valid within a ``parallel`` +section. This subset is checked during semantic analysis to be free of data +races. A sophisticated `disjoint checker`:idx: ensures that no data races are +possible even though shared memory is extensively supported! The subset is in fact the full language with the following restrictions / changes: @@ -8368,7 +8376,7 @@ pragmas: 2) Every access of a guarded memory location needs to happen in an appropriate `locks`:idx: statement. 3) Locks and routines can be annotated with `lock levels`:idx: to allow - potential deadlocks to be detected during static analysis. + potential deadlocks to be detected during semantic analysis. Guards and the locks section From e98bd3acebb21b14894e7f12c0b7f9e79a3f4c9c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 30 Jan 2019 00:26:33 -0800 Subject: [PATCH 266/350] fix #10488 GC memory leak regression (#10498) * fix #10488 GC memory leak regression * re-enable gch.stack.bottom.repr but only inside when defined(logGC) --- lib/system/gc.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 416827f21f..018197c1eb 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -864,7 +864,10 @@ when not defined(useNimRtl): for stack in items(gch.stack): result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & cast[pointer](stack.maxStackSize).repr & "\n" else: - result.add "[GC] stack bottom: " & gch.stack.bottom.repr + # this caused memory leaks, see #10488 ; find a way without `repr` + # maybe using a local copy of strutils.toHex or snprintf + when defined(logGC): + result.add "[GC] stack bottom: " & gch.stack.bottom.repr result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" {.pop.} # profiler: off, stackTrace: off From 9031c8fb0faeae35748fa311c22d7b0ee817d356 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 29 Jan 2019 11:51:04 +0100 Subject: [PATCH 267/350] Make sure the test snippet directory exists --- compiler/docgen.nim | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index a7f7d77b5c..90b6d0426f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -149,14 +149,16 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, if filename.len == 0: inc(d.id) let nameOnly = splitFile(d.filename).name - let subdir = getNimcacheDir(conf) / RelativeDir(nameOnly) - createDir(subdir) - outp = subdir / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim") + outp = getNimcacheDir(conf) / RelativeDir(nameOnly) / + RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim") elif isAbsolute(filename): - outp = AbsoluteFile filename + outp = AbsoluteFile(filename) else: # Nim's convention: every path is relative to the file it was written in: - outp = splitFile(d.filename).dir.AbsoluteDir / RelativeFile(filename) + let nameOnly = splitFile(d.filename).name + outp = AbsoluteDir(nameOnly) / RelativeFile(filename) + # Make sure the destination directory exists + createDir(outp.splitFile.dir) # Include the current file if we're parsing a nim file let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename] writeFile(outp, importStmt & content) From 49710b9d14291389529cdc627c21a25a99ff238d Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 29 Jan 2019 11:52:19 +0100 Subject: [PATCH 268/350] The `file` directive is relative to the file it is in --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 90b6d0426f..b70561a1d0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -246,7 +246,7 @@ proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool if n.comment.len > 0: - renderRstToOut(d[], parseRst(n.comment, toFilename(d.conf, n.info), + renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info), toLinenumber(n.info), toColumn(n.info), dummyHasToc, d.options, d.conf), result) From fa7b75782361fb31e74117a7e84ebe8fcda922c0 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 29 Jan 2019 13:04:21 +0100 Subject: [PATCH 269/350] Remove unused variable --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index a7cf251f64..b9be52308d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -4407,7 +4407,7 @@ template doAssertRaises*(exception: typedesc, code: untyped): typed = wrong = true except exception: discard - except Exception as exc: + except Exception: raiseAssert(astToStr(exception) & " wasn't raised, another error was raised instead by:\n"& astToStr(code)) From 7118e1ca01e1892208836411b3e4801f06b028c3 Mon Sep 17 00:00:00 2001 From: Mandeep Singh Date: Wed, 30 Jan 2019 10:35:45 -0500 Subject: [PATCH 270/350] Example for json.pretty (#10466) --- lib/pure/json.nim | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index ffb8f4f351..4a3d5b4323 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -130,9 +130,9 @@ ## { "name": "Susan", "age": herAge } ## ] ## -## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} -## j2["details"] = %* {"age":35, "pi":3.1415} -## echo j2 +## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} +## j2["details"] = %* {"age":35, "pi":3.1415} +## echo j2 runnableExamples: ## Note: for JObject, key ordering is preserved, unlike in some languages, @@ -708,6 +708,25 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, proc pretty*(node: JsonNode, indent = 2): string = ## Returns a JSON Representation of `node`, with indentation and ## on multiple lines. + ## + ## Similar to prettyprint in Python. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## let j = %* {"name": "Isaac", "books": ["Robot Dreams"], + ## "details": {"age":35, "pi":3.1415}} + ## echo pretty(j) + ## # { + ## # "name": "Isaac", + ## # "books": [ + ## # "Robot Dreams" + ## # ], + ## # "details": { + ## # "age": 35, + ## # "pi": 3.1415 + ## # } + ## # } result = "" toPretty(result, node, indent) From 9ac0cbdd51e838bbe4b040ac2df67fa050e6ac07 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 30 Jan 2019 17:31:06 +0100 Subject: [PATCH 271/350] fixes #2796 (#10500) [backport] --- lib/pure/oids.nim | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index d6369b5f9f..3aee3941dd 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -60,23 +60,21 @@ proc `$`*(oid: Oid): string = result = newString(24) oidToString(oid, result) +proc rand(): cint {.importc: "rand", header: "", nodecl.} +proc srand(seed: cint) {.importc: "srand", header: "", nodecl.} + +var t = getTime().toUnix.int32 +srand(t) + var - incr: int - fuzz: int32 + incr: int = rand() + fuzz: int32 = rand() proc genOid*(): Oid = ## generates a new OID. - proc rand(): cint {.importc: "rand", header: "", nodecl.} - proc srand(seed: cint) {.importc: "srand", header: "", nodecl.} - - var t = getTime().toUnix.int32 - + t = getTime().toUnix.int32 var i = int32(atomicInc(incr)) - if fuzz == 0: - # racy, but fine semantically: - srand(t) - fuzz = rand() bigEndian32(addr result.time, addr(t)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) From 0c2c2dca2a8a3bcdb9729021d1f4734b8db9efbd Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 30 Jan 2019 17:35:09 +0100 Subject: [PATCH 272/350] better docs: os (#10492) * better docs: os * remove broken test on osx --- lib/pure/includes/osenv.nim | 42 +- lib/pure/includes/oserr.nim | 30 +- lib/pure/includes/osseps.nim | 32 +- lib/pure/os.nim | 1213 +++++++++++++++++++++++++--------- 4 files changed, 968 insertions(+), 349 deletions(-) diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 945555540f..f9c076158d 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -102,9 +102,18 @@ proc findEnvVar(key: string): int = proc getEnv*(key: string, default = ""): TaintedString {.tags: [ReadEnvEffect].} = ## Returns the value of the `environment variable`:idx: named `key`. ## - ## If the variable does not exist, "" is returned. To distinguish - ## whether a variable exists or it's value is just "", call - ## `existsEnv(key)`. + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc <#existsEnv,string>`_. + ## + ## See also: + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + when nimvm: discard "built into the compiler" else: @@ -119,6 +128,14 @@ proc getEnv*(key: string, default = ""): TaintedString {.tags: [ReadEnvEffect].} proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert not existsEnv("unknownEnv") + when nimvm: discard "built into the compiler" else: @@ -127,7 +144,12 @@ proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `EInvalidEnvVar` is raised. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ # Note: by storing the string in the environment sequence, # we guarantee that we don't free the memory before the program @@ -154,9 +176,15 @@ proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = raiseOSError(osLastError()) iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. In the first component - ## of the tuple is the name of the current variable stored, in the second - ## its value. + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ getEnvVarsC() for i in 0..high(environment): var p = find(environment[i], '=') diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index db7d84c1e0..25e221d3bc 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -18,7 +18,7 @@ proc `$`*(err: OSErrorCode): string {.borrow.} proc osErrorMsg*(errorCode: OSErrorCode): string = ## Converts an OS error code into a human readable string. ## - ## The error code can be retrieved using the ``osLastError`` proc. + ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. ## ## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be ## returned. @@ -26,6 +26,16 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = ## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to ## make this procedure use the non-unicode Win API calls to retrieve the ## message. + ## + ## See also: + ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ + ## * `osLastError proc <#osLastError>`_ + runnableExamples: + when defined(posix): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + result = "" when defined(nimscript): discard @@ -48,13 +58,21 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = result = $c_strerror(errorCode.int32) proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = - ## Raises an ``OSError`` exception. The ``errorCode`` will determine the - ## message, ``osErrorMsg`` will be used to get this message. + ## Raises an `OSError exception `_. ## - ## The error code can be retrieved using the ``osLastError`` proc. + ## The ``errorCode`` will determine the + ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc + ## <#osLastError>`_. ## ## If the error code is ``0`` or an error message could not be retrieved, ## the message ``unknown OS error`` will be used. + ## + ## See also: + ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ + ## * `osLastError proc <#osLastError>`_ var e: ref OSError; new(e) e.errorCode = errorCode.int32 e.msg = osErrorMsg(errorCode) @@ -80,6 +98,10 @@ proc osLastError*(): OSErrorCode {.sideEffect.} = ## On Windows some OS calls can reset the error code to ``0`` causing this ## procedure to return ``0``. It is therefore advised to call this procedure ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ + ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ when defined(nimscript): discard elif defined(windows): diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim index 944ad123ef..859722f6a9 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/pure/includes/osseps.nim @@ -7,44 +7,44 @@ const when defined(Nimdoc): # only for proper documentation: const CurDir* = '.' - ## The constant string used by the operating system to refer to the + ## The constant character used by the operating system to refer to the ## current directory. ## - ## For example: '.' for POSIX or ':' for the classic Macintosh. + ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. ParDir* = ".." ## The constant string used by the operating system to refer to the ## parent directory. ## - ## For example: ".." for POSIX or "::" for the classic Macintosh. + ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. DirSep* = '/' ## The character used by the operating system to separate pathname - ## components, for example, '/' for POSIX or ':' for the classic - ## Macintosh. + ## components, for example: `'/'` for POSIX, `':'` for the classic + ## Macintosh, and `'\\'` on Windows. AltSep* = '/' ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep` if only one separator - ## character exists. This is set to '/' on Windows systems - ## where `DirSep` is a backslash. + ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator + ## character exists. This is set to `'/'` on Windows systems + ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). PathSep* = ':' ## The character conventionally used by the operating system to separate - ## search patch components (as in PATH), such as ':' for POSIX - ## or ';' for Windows. + ## search patch components (as in PATH), such as `':'` for POSIX + ## or `';'` for Windows. FileSystemCaseSensitive* = true - ## true if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths` to compare filenames properly. + ## True if the file system is case sensitive, false otherwise. Used by + ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. ExeExt* = "" ## The file extension of native executables. For example: - ## "" for POSIX, "exe" on Windows. + ## `""` for POSIX, `"exe"` on Windows (without a dot). ScriptExt* = "" - ## The file extension of a script file. For example: "" for POSIX, - ## "bat" on Windows. + ## The file extension of a script file. For example: `""` for POSIX, + ## `"bat"` on Windows. DynlibFormat* = "lib$1.so" ## The format string to turn a filename into a `DLL`:idx: file (also @@ -127,4 +127,4 @@ else: # UNIX-like operating system const ExtSep* = '.' ## The character which separates the base filename from the extension; - ## for example, the '.' in ``os.nim``. + ## for example, the `'.'` in ``os.nim``. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 181bc5728d..53bf880b64 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -10,6 +10,36 @@ ## This module contains basic operating system facilities like ## retrieving environment variables, reading command line arguments, ## working with directories, running shell commands, etc. +## +## .. code-block:: +## import os +## +## let myFile = "/path/to/my/file.nim" +## +## let splittedPath = splitPath(myFile) +## assert splittedPath.head == "/path/to/my" +## assert splittedPath.tail == "file.nim" +## +## assert parentDir(myFile) == "/path/to/my" +## +## let splittedFile = splitFile(myFile) +## assert splittedFile.dir == "/path/to/my" +## assert splittedFile.name == "file" +## assert splittedFile.ext == ".nim" +## +## assert myFile.changeFileExt("c") == "/path/to/my/file.c" +## +## +## **See also:** +## * `osproc module `_ for process communication beyond +## `execShellCmd proc <#execShellCmd,string>`_ +## * `parseopt module `_ for command-line parser beyond +## `parseCmdLine proc <#parseCmdLine,string>`_ +## * `distros module `_ +## * `dynlib module `_ +## * `streams module `_ + + {.deadCodeElim: on.} # dce option deprecated {.push debugger: off.} @@ -39,24 +69,24 @@ else: {.pragma: noNimScript.} type - ReadEnvEffect* = object of ReadIOEffect ## effect that denotes a read - ## from an environment variable - WriteEnvEffect* = object of WriteIOEffect ## effect that denotes a write - ## to an environment variable + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. - ReadDirEffect* = object of ReadIOEffect ## effect that denotes a read + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read ## operation from the directory - ## structure - WriteDirEffect* = object of WriteIOEffect ## effect that denotes a write + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write ## operation to - ## the directory structure + ## the directory structure. OSErrorCode* = distinct int32 ## Specifies an OS Error Code. include "includes/osseps" proc normalizePathEnd(path: var string, trailingSep = false) = - ## ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on ## ``trailingSep``, and taking care of edge cases: it preservers whether ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, ## not `AltSep`. @@ -83,27 +113,24 @@ proc joinPath*(head, tail: string): string {. noSideEffect, rtl, extern: "nos$1".} = ## Joins two directory names to one. ## - ## For example on Unix: + ## If `head` is the empty string, `tail` is returned. If `tail` is the empty + ## string, `head` is returned with a trailing path separator. If `tail` starts + ## with a path separator it will be removed when concatenated to `head`. Other + ## path separators not located on boundaries won't be modified. ## - ## .. code-block:: nim - ## joinPath("usr", "lib") - ## - ## results in: - ## - ## .. code-block:: nim - ## "usr/lib" - ## - ## If head is the empty string, tail is returned. If tail is the empty - ## string, head is returned with a trailing path separator. If tail starts - ## with a path separator it will be removed when concatenated to head. Other - ## path separators not located on boundaries won't be modified. More - ## examples on Unix: - ## - ## .. code-block:: nim - ## assert joinPath("usr", "") == "usr/" - ## assert joinPath("", "lib") == "lib" - ## assert joinPath("", "/lib") == "/lib" - ## assert joinPath("usr/", "/lib") == "usr/lib" + ## See also: + ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ + ## * `/ proc <#/,string,string>`_ + ## * `splitPath proc <#splitPath,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "") == "usr/" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + result = newStringOfCap(head.len + tail.len) var state = 0 addNormalizePath(head, result, state, DirSep) @@ -127,9 +154,23 @@ proc joinPath*(head, tail: string): string {. proc joinPath*(parts: varargs[string]): string {.noSideEffect, rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail)`, but works with any number of - ## directory parts. You need to pass at least one element or the proc + ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ + ## * `/ proc <#/,string,string>`_ + ## * `/../ proc <#/../,string,string>`_ + ## * `splitPath proc <#splitPath,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + var estimatedLen = 0 for p in parts: estimatedLen += p.len result = newStringOfCap(estimatedLen) @@ -138,30 +179,41 @@ proc joinPath*(parts: varargs[string]): string {.noSideEffect, addNormalizePath(parts[i], result, state, DirSep) proc `/`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``joinPath(head, tail)`` + ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. ## - ## Here are some examples for Unix: - ## - ## .. code-block:: nim - ## assert "usr" / "" == "usr/" - ## assert "" / "lib" == "lib" - ## assert "" / "/lib" == "/lib" - ## assert "usr/" / "/lib" == "usr/lib" + ## See also: + ## * `/../ proc <#/../,string,string>`_ + ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ + ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ + ## * `splitPath proc <#splitPath,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr/" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib" == "usr/lib" + assert "usr" / "lib" / "../bin" == "usr/bin" + return joinPath(head, tail) proc splitPath*(path: string): tuple[head, tail: string] {. noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into (head, tail), so that + ## Splits a directory into `(head, tail)` tuple, so that ## ``head / tail == path`` (except for edge cases like "/usr"). ## - ## Examples: - ## - ## .. code-block:: nim - ## splitPath("usr/local/bin") -> ("usr/local", "bin") - ## splitPath("usr/local/bin/") -> ("usr/local/bin", "") - ## splitPath("bin") -> ("", "bin") - ## splitPath("/bin") -> ("", "bin") - ## splitPath("") -> ("", "") + ## See also: + ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ + ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ + ## * `/ proc <#/,string,string>`_ + ## * `/../ proc <#/../,string,string>`_ + ## * `relativePath proc <#relativePath,string,string>`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("bin") == ("", "bin") + assert splitPath("/bin") == ("", "bin") + assert splitPath("") == ("", "") + var sepPos = -1 for i in countdown(len(path)-1, 0): if path[i] in {DirSep, AltSep}: @@ -182,16 +234,21 @@ else: proc relativePath*(path, base: string; sep = DirSep): string {. noSideEffect, rtl, extern: "nos$1", raises: [].} = ## Converts `path` to a path relative to `base`. - ## The `sep` is used for the path normalizations, this can be useful to - ## ensure the relative path only contains '/' so that it can be used for - ## URL constructions. + ## + ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## See also: + ## * `splitPath proc <#splitPath,string>`_ + ## * `parentDir proc <#parentDir,string>`_ + ## * `tailDir proc <#tailDir,string>`_ runnableExamples: - doAssert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - doAssert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" - doAssert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" - doAssert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" - doAssert relativePath("", "/users/moo", '/') == "" - + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" # Todo: If on Windows, path and base do not agree on the drive letter, # return `path` as is. @@ -252,12 +309,21 @@ proc parentDir*(path: string): string {. ## ## This is the same as ``splitPath(path).head`` when ``path`` doesn't end ## in a dir separator. - ## The remainder can be obtained with ``lastPathPart(path)`` + ## The remainder can be obtained with `lastPathPart(path) proc + ## <#lastPathPart,string>`_. + ## + ## See also: + ## * `relativePath proc <#relativePath,string,string>`_ + ## * `splitPath proc <#splitPath,string>`_ + ## * `tailDir proc <#tailDir,string>`_ + ## * `parentDirs iterator <#parentDirs.i,string>`_ runnableExamples: - doAssert parentDir("") == "" + assert parentDir("") == "" when defined(posix): - doAssert parentDir("/usr/local/bin") == "/usr/local" - doAssert parentDir("foo/bar/") == "foo" + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar/") == "foo" + assert parentDir("./foo") == "." + assert parentDir("/foo") == "" let sepPos = parentDirPos(path) if sepPos >= 0: @@ -267,11 +333,18 @@ proc parentDir*(path: string): string {. proc tailDir*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`.. + ## Returns the tail part of `path`. ## - ## | Example: ``tailDir("/usr/local/bin") == "local/bin"``. - ## | Example: ``tailDir("usr/local/bin/") == "local/bin"``. - ## | Example: ``tailDir("bin") == ""``. + ## See also: + ## * `relativePath proc <#relativePath,string,string>`_ + ## * `splitPath proc <#splitPath,string>`_ + ## * `parentDir proc <#parentDir,string>`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + var q = 1 if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 for i in 0..len(path)-q: @@ -281,18 +354,53 @@ proc tailDir*(path: string): string {. proc isRootDir*(path: string): bool {. noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + result = parentDirPos(path) < 0 iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path` + ## Walks over all parent directories of a given `path`. ## - ## If `fromRoot` is set, the traversal will start from the file system root - ## diretory. If `inclusive` is set, the original argument will be included + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root diretory. + ## If `inclusive` is true (default), the original argument will be included ## in the traversal. ## - ## Relative paths won't be expanded by this proc. Instead, it will traverse + ## Relative paths won't be expanded by this iterator. Instead, it will traverse ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc <#parentDir,string>`_ + ## + ## **Examples:** + ## + ## .. code-block:: + ## let g = "a/b/c" + ## + ## for p in g.parentDirs: + ## echo p + ## # a/b/c + ## # a/b + ## # a + ## + ## for p in g.parentDirs(fromRoot=true): + ## echo p + ## # a/ + ## # a/b/ + ## # a/b/c + ## + ## for p in g.parentDirs(inclusive=false): + ## echo p + ## # a/b + ## # a + if not fromRoot: var current = path if inclusive: yield path @@ -310,8 +418,16 @@ iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = if inclusive: yield path proc `/../`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail`` unless there is no parent + ## The same as ``parentDir(head) / tail``, unless there is no parent ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc <#/,string,string>`_ + ## * `parentDir proc <#parentDir,string>`_ + runnableExamples: + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + let sepPos = parentDirPos(head) if sepPos >= 0: result = substr(head, 0, sepPos-1) / tail @@ -323,8 +439,21 @@ proc normExt(ext: string): string = else: result = ExtSep & ext proc searchExtPos*(path: string): int = - ## Returns index of the '.' char in `path` if it signifies the beginning + ## Returns index of the `'.'` char in `path` if it signifies the beginning ## of extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc <#splitFile,string>`_ + ## * `extractFilename proc <#extractFilename,string>`_ + ## * `lastPathPart proc <#lastPathPart,string>`_ + ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `addFileExt proc <#addFileExt,string,string>`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + # BUGFIX: do not search until 0! .DS_Store is no file extension! result = -1 for i in countdown(len(path)-1, 1): @@ -336,21 +465,35 @@ proc searchExtPos*(path: string): int = proc splitFile*(path: string): tuple[dir, name, ext: string] {. noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into (dir, name, extension). - ## `dir` does not end in `DirSep`. + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in `DirSep <#DirSep>`_. ## `extension` includes the leading dot. ## - ## Example: - ## - ## .. code-block:: nim - ## var (dir, name, ext) = splitFile("usr/local/nimc.html") - ## assert dir == "usr/local" - ## assert name == "nimc" - ## assert ext == ".html" - ## ## If `path` has no extension, `ext` is the empty string. ## If `path` has no directory component, `dir` is the empty string. ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc <#searchExtPos,string>`_ + ## * `extractFilename proc <#extractFilename,string>`_ + ## * `lastPathPart proc <#lastPathPart,string>`_ + ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `addFileExt proc <#addFileExt,string,string>`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + if path.len == 0: result = ("", "", "") elif path[^1] in {DirSep, AltSep}: @@ -380,12 +523,22 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. proc extractFilename*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. This is the same as - ## ``name & ext`` from ``splitFile(path)``. See also ``lastPathPart``. + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc + ## <#splitFile,string>`_. + ## + ## See also: + ## * `searchExtPos proc <#searchExtPos,string>`_ + ## * `splitFile proc <#splitFile,string>`_ + ## * `lastPathPart proc <#lastPathPart,string>`_ + ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `addFileExt proc <#addFileExt,string,string>`_ runnableExamples: - when defined(posix): - doAssert extractFilename("foo/bar/") == "" - doAssert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: result = "" else: @@ -393,11 +546,19 @@ proc extractFilename*(path: string): string {. proc lastPathPart*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = - ## like ``extractFilename``, but ignores trailing dir separator; aka: `baseName`:idx: - ## in some other languages. + ## Like `extractFilename proc <#extractFilename,string>`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc <#searchExtPos,string>`_ + ## * `splitFile proc <#splitFile,string>`_ + ## * `extractFilename proc <#extractFilename,string>`_ + ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `addFileExt proc <#addFileExt,string,string>`_ runnableExamples: - when defined(posix): - doAssert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + let path = path.normalizePathEnd(trailingSep = false) result = extractFilename(path) @@ -407,9 +568,22 @@ proc changeFileExt*(filename, ext: string): string {. ## ## If the `filename` has no extension, `ext` will be added. ## If `ext` == "" then any extension is removed. - ## `Ext` should be given without the leading '.', because some + ## + ## `Ext` should be given without the leading `'.'`, because some ## filesystems may use a different character. (Although I know ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc <#searchExtPos,string>`_ + ## * `splitFile proc <#splitFile,string>`_ + ## * `extractFilename proc <#extractFilename,string>`_ + ## * `lastPathPart proc <#lastPathPart,string>`_ + ## * `addFileExt proc <#addFileExt,string,string>`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + var extPos = searchExtPos(filename) if extPos < 0: result = filename & normExt(ext) else: result = substr(filename, 0, extPos-1) & normExt(ext) @@ -419,9 +593,21 @@ proc addFileExt*(filename, ext: string): string {. ## Adds the file extension `ext` to `filename`, unless ## `filename` already has an extension. ## - ## `Ext` should be given without the leading '.', because some + ## `Ext` should be given without the leading `'.'`, because some ## filesystems may use a different character. ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc <#searchExtPos,string>`_ + ## * `splitFile proc <#splitFile,string>`_ + ## * `extractFilename proc <#extractFilename,string>`_ + ## * `lastPathPart proc <#lastPathPart,string>`_ + ## * `changeFileExt proc <#changeFileExt,string,string>`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + var extPos = searchExtPos(filename) if extPos < 0: result = filename & normExt(ext) else: result = filename @@ -438,9 +624,9 @@ proc cmpPaths*(pathA, pathB: string): int {. ## | > 0 iff pathA > pathB runnableExamples: when defined(macosx): - doAssert cmpPaths("foo", "Foo") == 0 + assert cmpPaths("foo", "Foo") == 0 elif defined(posix): - doAssert cmpPaths("foo", "Foo") > 0 + assert cmpPaths("foo", "Foo") > 0 let a = normalizePath(pathA) let b = normalizePath(pathB) @@ -458,11 +644,12 @@ proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} = ## ## On Windows, network paths are considered absolute too. runnableExamples: - doAssert(not "".isAbsolute) - doAssert(not ".".isAbsolute) + assert not "".isAbsolute + assert not ".".isAbsolute when defined(posix): - doAssert "/".isAbsolute - doAssert(not "a/".isAbsolute) + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute if len(path) == 0: return false @@ -483,7 +670,7 @@ proc unixToNativePath*(path: string, drive=""): string {. ## Converts an UNIX-like path to a native one. ## ## On an UNIX system this does nothing. Else it converts - ## '/', '.', '..' to the appropriate things. + ## `'/'`, `'.'`, `'..'` to the appropriate things. ## ## On systems with a concept of "drives", `drive` is used to determine ## which drive label to use during absolute path conversion. @@ -542,8 +729,18 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = ## Returns the home directory of the current user. ## - ## This proc is wrapped by the expandTilde proc for the convenience of - ## processing paths coming from user configuration files. + ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getTempDir proc <#getTempDir>`_ + ## * `expandTilde proc <#expandTilde,string>`_ + ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `setCurrentDir proc <#setCurrentDir,string>`_ + runnableExamples: + assert getHomeDir() == expandTilde("~") + when defined(windows): return string(getEnv("USERPROFILE")) & "\\" else: return string(getEnv("HOME")) & "/" @@ -552,12 +749,19 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## Returns the config directory of the current user for applications. ## ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the XDG_CONFIG_HOME environment - ## variable if it is set, and returns the default configuration directory, - ## "~/.config/", otherwise. + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). ## ## An OS-dependent trailing slash is always present at the end of the - ## returned string; `\` on Windows and `/` on all other OSs. + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc <#getHomeDir>`_ + ## * `getTempDir proc <#getTempDir>`_ + ## * `expandTilde proc <#expandTilde,string>`_ + ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `setCurrentDir proc <#setCurrentDir,string>`_ when defined(windows): result = getEnv("APPDATA").string else: @@ -573,6 +777,13 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", ## returns ``getHomeDir()``, and on other Unix based systems it can cause ## security problems too. That said, you can override this implementation ## by adding ``-d:tempDir=mytempname`` to your compiler invokation. + ## + ## See also: + ## * `getHomeDir proc <#getHomeDir>`_ + ## * `getConfigDir proc <#getConfigDir>`_ + ## * `expandTilde proc <#expandTilde,string>`_ + ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `setCurrentDir proc <#setCurrentDir,string>`_ when defined(tempDir): const tempDir {.strdefine.}: string = nil return tempDir @@ -583,12 +794,22 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with ``getHomeDir()`` (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). ## ## Windows: this is still supported despite Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. + ## + ## See also: + ## * `getHomeDir proc <#getHomeDir>`_ + ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getTempDir proc <#getTempDir>`_ + ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `setCurrentDir proc <#setCurrentDir,string>`_ runnableExamples: - doAssert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" + assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" + assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" + assert expandTilde("/foo/bar") == "/foo/bar" + if len(path) == 0 or path[0] != '~': result = path elif len(path) == 1: @@ -602,11 +823,13 @@ proc expandTilde*(path: string): string {. # TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand # belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to Windows API. - ## Based on Python's subprocess.list2cmdline - ## See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - let needQuote = {' ', '\t'} in s or s.len == 0 + ## Quote `s`, so it can be safely passed to Windows API. + ## + ## Based on Python's `subprocess.list2cmdline`. + ## See `this link `_ + ## for more details. + let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" if needQuote: @@ -631,7 +854,7 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's pipes.quote + ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: @@ -647,18 +870,23 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. + ## + ## When on Windows, it calls `quoteShellWindows proc + ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc + ## <#quoteShellPosix,string>`_. when defined(windows): return quoteShellWindows(s) else: return quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = - ## Concatenates and quotes shell arguments `args` + ## Concatenates and quotes shell arguments `args`. runnableExamples: when defined(posix): assert quoteShellCommand(["aaa", "", "c d"]) == "aaa '' 'c d'" when defined(windows): assert quoteShellCommand(["aaa", "", "c d"]) == "aaa \"\" \"c d\"" + # can't use `map` pending https://github.com/nim-lang/Nim/issues/8303 for i in 0.. 0: result.add " " @@ -705,8 +933,12 @@ when defined(windows) and not weirdTarget: proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], noNimScript.} = ## Returns true if `filename` exists and is a regular file or symlink. - ## (directories, device files, named pipes and sockets return false) - ## This proc is not available for NimScript. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `existsDir proc <#existsDir,string>`_ + ## * `symlinkExists proc <#symlinkExists,string>`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, filename) @@ -722,6 +954,10 @@ proc existsDir*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] noNimScript.} = ## Returns true iff the directory `dir` exists. If `dir` is a file, false ## is returned. Follows symlinks. + ## + ## See also: + ## * `existsFile proc <#existsFile,string>`_ + ## * `symlinkExists proc <#symlinkExists,string>`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, dir) @@ -738,6 +974,10 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", noNimScript.} = ## Returns true iff the symlink `link` exists. Will return true ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `existsFile proc <#existsFile,string>`_ + ## * `existsDir proc <#existsDir,string>`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, link) @@ -750,11 +990,19 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) proc fileExists*(filename: string): bool {.inline, noNimScript.} = - ## Synonym for existsFile + ## Alias for `existsFile proc <#existsFile,string>`_. + ## + ## See also: + ## * `existsDir proc <#existsDir,string>`_ + ## * `symlinkExists proc <#symlinkExists,string>`_ existsFile(filename) proc dirExists*(dir: string): bool {.inline, noNimScript.} = - ## Synonym for existsDir + ## Alias for `existsDir proc <#existsDir,string>`_. + ## + ## See also: + ## * `existsFile proc <#existsFile,string>`_ + ## * `symlinkExists proc <#symlinkExists,string>`_ existsDir(dir) when not defined(windows) and not weirdTarget: @@ -764,19 +1012,23 @@ when not defined(windows) and not weirdTarget: else: result = S_ISLNK(rawInfo.st_mode) const - ExeExts* = when defined(windows): ["exe", "cmd", "bat"] else: [""] ## \ - ## platform specific file extension for executables. On Windows - ## ``["exe", "cmd", "bat"]``, on Posix ``[""]``. + ExeExts* = ## Platform specific file extension for executables. + ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. + when defined(windows): ["exe", "cmd", "bat"] else: [""] proc findExe*(exe: string, followSymlinks: bool = true; extensions: openarray[string]=ExeExts): string {. tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimScript.} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. - ## Returns "" if the `exe` cannot be found. `exe` + ## + ## Returns `""` if the `exe` cannot be found. `exe` ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## ## If the system supports symlinks it also resolves them until it - ## meets the actual file. This behavior can be disabled if desired. + ## meets the actual file. This behavior can be disabled if desired + ## by setting `followSymlinks = false`. + if exe.len == 0: return template checkCurrentDir() = for ext in extensions: @@ -824,6 +1076,11 @@ when weirdTarget: proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScript.} = ## Returns the `file`'s last modification time. + ## + ## See also: + ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ + ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `fileNewer proc <#fileNewer,string,string>`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) @@ -837,6 +1094,11 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScript.} = ## Returns the `file`'s last read or write access time. + ## + ## See also: + ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ + ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `fileNewer proc <#fileNewer,string,string>`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) @@ -854,6 +1116,11 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScr ## **Note:** Under POSIX OS's, the returned time may actually be the time at ## which the file's attribute's were last modified. See ## `here `_ for details. + ## + ## See also: + ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ + ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ + ## * `fileNewer proc <#fileNewer,string,string>`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) @@ -868,6 +1135,11 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScr proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noNimScript.} = ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s ## modification time is later than `b`'s. + ## + ## See also: + ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ + ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ + ## * `getCreationTime proc <#getCreationTime,string>`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -879,6 +1151,12 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noNimScript.} = proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [], noNimScript.} = ## Returns the `current working directory`:idx:. + ## + ## See also: + ## * `getHomeDir proc <#getHomeDir>`_ + ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getTempDir proc <#getTempDir>`_ + ## * `setCurrentDir proc <#setCurrentDir,string>`_ when defined(windows): var bufsize = MAX_PATH.int32 when useWinUnicode: @@ -922,8 +1200,14 @@ proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [], noNimScript.} = raiseOSError(osLastError()) proc setCurrentDir*(newDir: string) {.inline, tags: [], noNimScript.} = - ## Sets the `current working directory`:idx:; `OSError` is raised if - ## `newDir` cannot been set. + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc <#getHomeDir>`_ + ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getTempDir proc <#getTempDir>`_ + ## * `getCurrentDir proc <#getCurrentDir>`_ when defined(Windows): when useWinUnicode: if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: @@ -935,10 +1219,16 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [], noNimScript.} = when not weirdTarget: proc absolutePath*(path: string, root = getCurrentDir()): string {.noNimScript.} = - ## Returns the absolute path of `path`, rooted at `root` (which must be absolute) - ## if `path` is absolute, return it, ignoring `root` + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc <#normalizedPath,string>`_ + ## * `normalizePath proc <#normalizePath,string>`_ runnableExamples: - doAssert absolutePath("a") == getCurrentDir() / "a" + assert absolutePath("a") == getCurrentDir() / "a" + if isAbsolute(path): path else: if not root.isAbsolute: @@ -950,11 +1240,21 @@ proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [], noNimScr ## ## Consecutive directory separators are collapsed, including an initial double slash. ## - ## On relative paths, double dot (..) sequences are collapsed if possible. + ## On relative paths, double dot (`..`) sequences are collapsed if possible. ## On absolute paths they are always collapsed. ## ## Warning: URL-encoded and Unicode attempts at directory traversal are not detected. ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc <#absolutePath,string>`_ + ## * `normalizedPath proc <#normalizedPath,string>`_ for a version which returns + ## a new string + runnableExamples: + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + path = pathnorm.normalizePath(path) when false: let isAbs = isAbsolute(path) @@ -984,7 +1284,13 @@ proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [], noNimScr path = "." proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [], noNimScript.} = - ## Returns a normalized path for the current OS. See `<#normalizePath>`_ + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc <#absolutePath,string>`_ + ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version + runnableExamples: + assert normalizedPath("a///b//..//c///d") == "a/c/d" result = pathnorm.normalizePath(path) when defined(Windows) and not weirdTarget: @@ -1010,11 +1316,16 @@ when defined(Windows) and not weirdTarget: proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], noNimScript.} = ## Returns true if both pathname arguments refer to the same physical - ## file or directory. Raises an exception if any of the files does not + ## file or directory. + ## + ## Raises `OSError` if any of the files does not ## exist or information about it can not be obtained. ## ## This proc will return true if given two alternative hard-linked or ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc <#sameFileContent,string,string>`_ when defined(Windows): var success = true var f1 = openHandle(path1) @@ -1051,6 +1362,9 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = ## Returns true if both pathname arguments refer to files with identical ## binary content. + ## + ## See also: + ## * `sameFile proc <#sameFile,string,string>`_ const bufSize = 8192 # 8K buffer var @@ -1079,7 +1393,12 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", close(b) type - FilePermission* = enum ## file access permission; modelled after UNIX + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions <#getFilePermissions,string>`_ + ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ + ## * `FileInfo object <#FileInfo>`_ fpUserExec, ## execute access for the file owner fpUserWrite, ## write access for the file owner fpUserRead, ## read access for the file owner @@ -1092,9 +1411,15 @@ type proc getFilePermissions*(filename: string): set[FilePermission] {. rtl, extern: "nos$1", tags: [ReadDirEffect], noNimScript.} = - ## retrieves file permissions for `filename`. `OSError` is raised in case of - ## an error. On Windows, only the ``readonly`` flag is checked, every other + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ + ## * `FilePermission enum <#FilePermission>`_ when defined(posix): var a: Stat if stat(filename, a) < 0'i32: raiseOSError(osLastError()) @@ -1124,9 +1449,15 @@ proc getFilePermissions*(filename: string): set[FilePermission] {. proc setFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## sets the file permissions for `filename`. `OSError` is raised in case of - ## an error. On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite``. + ## Sets the file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions <#getFilePermissions,string>`_ + ## * `FilePermission enum <#FilePermission>`_ when defined(posix): var p = 0'i32 if fpUserRead in permissions: p = p or S_IRUSR @@ -1162,14 +1493,29 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = ## Copies a file from `source` to `dest`. ## - ## If this fails, `OSError` is raised. On the Windows platform this proc will - ## copy the source file's attributes into dest. On other platforms you need - ## to use `getFilePermissions() <#getFilePermissions>`_ and - ## `setFilePermissions() <#setFilePermissions>`_ to copy them by hand (or use - ## the convenience `copyFileWithPermissions() <#copyFileWithPermissions>`_ - ## proc), otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. If `dest` already exists, the file attributes + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions <#getFilePermissions,string>`_ and + ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc <#copyFileWithPermissions,string,string>`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes ## will be preserved and the content overwritten. + ## + ## See also: + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `removeFile proc <#removeFile,string>`_ + ## * `moveFile proc <#moveFile,string,string>`_ + when defined(Windows): when useWinUnicode: let s = newWideCString(source) @@ -1221,9 +1567,18 @@ when defined(Windows) and not weirdTarget: setFileAttributesA(file, attrs) proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## Removes the `file`. If this fails, returns `false`. This does not fail + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail ## if the file never existed in the first place. + ## ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `removeFile proc <#removeFile,string>`_ + ## * `moveFile proc <#moveFile,string,string>`_ result = true when defined(Windows): when useWinUnicode: @@ -1244,9 +1599,19 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE result = false proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## Removes the `file`. If this fails, `OSError` is raised. This does not fail + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail ## if the file never existed in the first place. + ## ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <#removeDir,string>`_ + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `moveFile proc <#moveFile,string,string>`_ if not tryRemoveFile(file): when defined(Windows): raiseOSError(osLastError()) @@ -1254,9 +1619,11 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n raiseOSError(osLastError(), $strerror(errno)) proc tryMoveFSObject(source, dest: string): bool {.noNimScript.} = - ## Moves a file or directory from `source` to `dest`. Returns false in case - ## of `EXDEV` error. In case of other errors `OSError` is raised. Returns - ## true in case of success. + ## Moves a file or directory from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error. + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. when defined(Windows): when useWinUnicode: let s = newWideCString(source) @@ -1275,8 +1642,19 @@ proc tryMoveFSObject(source, dest: string): bool {.noNimScript.} = proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = - ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised. - ## Can be used to `rename files`:idx: + ## Moves a file from `source` to `dest`. + ## + ## If this fails, `OSError` is raised. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `removeFile proc <#removeFile,string>`_ + ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + if not tryMoveFSObject(source, dest): when not defined(windows): # Fallback to copy & del @@ -1288,7 +1666,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", raise proc exitStatusLikeShell*(status: cint): cint = - ## converts exit code from `c_system` into a shell exit code + ## Converts exit code from `c_system` into a shell exit code. when defined(posix) and not weirdTarget: if WIFSIGNALED(status): # like the shell! @@ -1304,9 +1682,16 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## ## Command has the form 'program args' where args are the command ## line arguments given to program. The proc returns the error code - ## of the shell when it has finished. The proc does not return until - ## the process has finished. To execute a program without having a - ## shell involved, use `osproc.execProcess`. + ## of the shell when it has finished (zero if there is no error). + ## The proc does not return until the process has finished. + ## + ## To execute a program without having a shell involved, use `osproc.execProcess proc + ## `_. + ## + ## **Examples:** + ## + ## .. code-block:: + ## discard execShellCmd("ls -la") result = exitStatusLikeShell(c_system(command)) # Templates for filtering directories and files @@ -1368,32 +1753,51 @@ template walkCommon(pattern: string, filter) = iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = ## Iterate over all the files and directories that match the `pattern`. - ## On POSIX this uses the `glob`:idx: call. ## - ## `pattern` is OS dependent, but at least the "\*.ext" + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"\*.ext"` ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDir iterator <#walkDir.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ walkCommon(pattern, defaultWalkFilter) iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = - ## Iterate over all the files that match the `pattern`. On POSIX this uses - ## the `glob`:idx: call. + ## Iterate over all the files that match the `pattern`. ## - ## `pattern` is OS dependent, but at least the "\*.ext" + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"\*.ext"` ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDir iterator <#walkDir.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ walkCommon(pattern, isFile) iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = ## Iterate over all the directories that match the `pattern`. - ## On POSIX this uses the `glob`:idx: call. ## - ## `pattern` is OS dependent, but at least the "\*.ext" + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"\*.ext"` ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDir iterator <#walkDir.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ walkCommon(pattern, isDir) proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect], noNimScript.} = - ## Returns the full (`absolute`:idx:) path of an existing file `filename`, - ## raises OSError in case of an error. Follows symlinks. + ## Returns the full (`absolute`:idx:) path of an existing file `filename`. + ## + ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 when useWinUnicode: @@ -1440,13 +1844,19 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", type PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `FileInfo object <#FileInfo>`_ pcFile, ## path refers to a file pcLinkToFile, ## path refers to a symbolic link to a file pcDir, ## path refers to a directory pcLinkToDir ## path refers to a symbolic link to a directory proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## `getAppFilename` at CT; can be used to retrive the currently executing + ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## + ## Can be used to retrive the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary ## inside a nimble program (likewise with other binaries built from ## compiler API). @@ -1467,10 +1877,11 @@ proc staticWalkDir(dir: string; relative: bool): seq[ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. tags: [ReadDirEffect].} = - ## walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item is returned. - ## Walking is not recursive. If ``relative`` is true the resulting path is - ## shortened to be relative to ``dir``. + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. If ``relative`` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``. ## Example: This directory structure:: ## dirA / dirB / fileB1.txt ## / dirC @@ -1483,11 +1894,18 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## for kind, path in walkDir("dirA"): ## echo(path) ## - ## produces this output (but not necessarily in this order!):: + ## produce this output (but not necessarily in this order!):: ## dirA/dirB ## dirA/dirC ## dirA/fileA1.txt ## dirA/fileA2.txt + ## + ## See also: + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + when nimvm: for k, v in items(staticWalkDir(dir, relative)): yield (k, v) @@ -1554,19 +1972,20 @@ iterator walkDirRec*(dir: string, relative = false): string {.tags: [ReadDirEffect].} = ## Recursively walks over the directory `dir` and yields for each file ## or directory in `dir`. - ## If ``relative`` is true the resulting path is + ## + ## If ``relative`` is true (default: false) the resulting path is ## shortened to be relative to ``dir``, otherwise the full path is returned. ## ## **Warning**: ## Modifying the directory structure while the iterator ## is traversing may result in undefined behavior! ## - ## Walking is recursive. `filters` controls the behaviour of the iterator: + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: ## ## --------------------- --------------------------------------------- ## yieldFilter meaning ## --------------------- --------------------------------------------- - ## ``pcFile`` yield real files + ## ``pcFile`` yield real files (default) ## ``pcLinkToFile`` yield symbolic links to files ## ``pcDir`` yield real directories ## ``pcLinkToDir`` yield symbolic links to directories @@ -1575,10 +1994,17 @@ iterator walkDirRec*(dir: string, ## --------------------- --------------------------------------------- ## followFilter meaning ## --------------------- --------------------------------------------- - ## ``pcDir`` follow real directories + ## ``pcDir`` follow real directories (default) ## ``pcLinkToDir`` follow symbolic links to directories ## --------------------- --------------------------------------------- ## + ## + ## See also: + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDir iterator <#walkDir.i,string>`_ + var stack = @[""] while stack.len > 0: let d = stack.pop() @@ -1609,6 +2035,15 @@ proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ ## ## If this fails, `OSError` is raised. This does not fail if the directory never ## existed in the first place. + ## + ## See also: + ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `removeFile proc <#removeFile,string>`_ + ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ + ## * `createDir proc <#createDir,string>`_ + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `moveDir proc <#moveDir,string,string>`_ for kind, path in walkDir(dir): case kind of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) @@ -1666,6 +2101,13 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", ## Does not create parent directories (fails if parent does not exist). ## Returns `true` if the directory already exists, and `false` ## otherwise. + ## + ## See also: + ## * `removeDir proc <#removeDir,string>`_ + ## * `createDir proc <#createDir,string>`_ + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `moveDir proc <#moveDir,string,string>`_ result = not rawCreateDir(dir) if result: # path already exists - need to check that it is indeed a directory @@ -1677,9 +2119,17 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", ## Creates the `directory`:idx: `dir`. ## ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. It does **not** - ## fail if the directory already exists because for most usages this does not - ## indicate an error. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc <#removeDir,string>`_ + ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `moveDir proc <#moveDir,string,string>`_ var omitNext = false when doslikeFileSystem: omitNext = isAbsolute(dir) @@ -1699,11 +2149,24 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", tags: [WriteIOEffect, ReadIOEffect], benign, noNimScript.} = ## Copies a directory from `source` to `dest`. ## - ## If this fails, `OSError` is raised. On the Windows platform this proc will - ## copy the attributes from `source` into `dest`. On other platforms created - ## files and directories will inherit the default permissions of a newly - ## created file/directory for the user. To preserve attributes recursively on - ## these platforms use `copyDirWithPermissions() <#copyDirWithPermissions>`_. + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `removeDir proc <#removeDir,string>`_ + ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ + ## * `createDir proc <#createDir,string>`_ + ## * `moveDir proc <#moveDir,string,string>`_ createDir(dest) for kind, path in walkDir(source): var noSource = splitPath(path).tail @@ -1714,6 +2177,24 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", copyDir(path, dest / noSource) else: discard +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = + ## Moves a directory from `source` to `dest`. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <#moveFile,string,string>`_ + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `removeDir proc <#removeDir,string>`_ + ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ + ## * `createDir proc <#createDir,string>`_ + if not tryMoveFSObject(source, dest): + when not defined(windows): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + proc createSymlink*(src, dest: string) {.noNimScript.} = ## Create a symbolic link at `dest` which points to the item specified ## by `src`. On most operating systems, will fail if a link already exists. @@ -1721,6 +2202,11 @@ proc createSymlink*(src, dest: string) {.noNimScript.} = ## **Warning**: ## Some OS's (such as Microsoft Windows) restrict the creation ## of symlinks to root users (administrators). + ## + ## See also: + ## * `createHardlink proc <#createHardlink,string,string>`_ + ## * `expandSymlink proc <#expandSymlink,string>`_ + when defined(Windows): # 2 is the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE. This allows # anyone with developer mode on to create a link @@ -1743,6 +2229,9 @@ proc createHardlink*(src, dest: string) {.noNimScript.} = ## ## **Warning**: Some OS's restrict the creation of hard links to ## root users (administrators). + ## + ## See also: + ## * `createSymlink proc <#createSymlink,string,string>`_ when defined(Windows): when useWinUnicode: var wSrc = newWideCString(src) @@ -1756,13 +2245,128 @@ proc createHardlink*(src, dest: string) {.noNimScript.} = if link(src, dest) != 0: raiseOSError(osLastError()) +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true) {.noNimScript.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, + ## `getFilePermissions <#getFilePermissions,string>`_ and + ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc + ## <#copyFile,string,string>`_ since that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `removeFile proc <#removeFile,string>`_ + ## * `moveFile proc <#moveFile,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + copyFile(source, dest) + when not defined(Windows): + try: + setFilePermissions(dest, getFilePermissions(source)) + except: + if not ignorePermissionErrors: + raise + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true) {.rtl, extern: "nos$1", + tags: [WriteIOEffect, ReadIOEffect], benign, noNimScript.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir + ## <#copyDir,string,string>`_ and `copyFileWithPermissions + ## <#copyFileWithPermissions,string,string>`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc + ## <#copyDir,string,string>`_ since that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ + ## * `removeDir proc <#removeDir,string>`_ + ## * `moveDir proc <#moveDir,string,string>`_ + ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ + ## * `createDir proc <#createDir,string>`_ + createDir(dest) + when not defined(Windows): + try: + setFilePermissions(dest, getFilePermissions(source)) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source): + var noSource = splitPath(path).tail + case kind + of pcFile: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors) + of pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) + else: discard + +proc inclFilePermissions*(filename: string, + permissions: set[FilePermission]) {. + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = + ## A convenience proc for: + ## + ## .. code-block:: nim + ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + setFilePermissions(filename, getFilePermissions(filename)+permissions) + +proc exclFilePermissions*(filename: string, + permissions: set[FilePermission]) {. + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = + ## A convenience proc for: + ## + ## .. code-block:: nim + ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + setFilePermissions(filename, getFilePermissions(filename)-permissions) + +proc expandSymlink*(symlinkPath: string): string {.noNimScript.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, ``symlinkPath`` is simply returned. + ## + ## See also: + ## * `createSymlink proc <#createSymlink,string,string>`_ + when defined(windows): + result = symlinkPath + else: + result = newString(256) + var len = readlink(symlinkPath, result, 256) + if len < 0: + raiseOSError(osLastError()) + if len > 256: + result = newString(len+1) + len = readlink(symlinkPath, result, len) + setLen(result, len) + proc parseCmdLine*(c: string): seq[string] {. noSideEffect, rtl, extern: "nos$1".} = - ## Splits a `command line`:idx: into several components; - ## This proc is only occasionally useful, better use the `parseopt` module. + ## Splits a `command line`:idx: into several components. ## - ## On Windows, it uses the following parsing rules - ## (see http://msdn.microsoft.com/en-us/library/17w5ykft.aspx ): + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module `_. + ## + ## On Windows, it uses the `following parsing rules + ## `_: ## ## * Arguments are delimited by white space, which is either a space or a tab. ## * The caret character (^) is not recognized as an escape character or @@ -1787,6 +2391,13 @@ proc parseCmdLine*(c: string): seq[string] {. ## On Posix systems, it uses the following parsing rules: ## Components are separated by whitespace unless the whitespace ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module `_ + ## * `paramCount proc <#paramCount>`_ + ## * `paramStr proc <#paramStr,int>`_ + ## * `commandLineParams proc <#commandLineParams>`_ + result = @[] var i = 0 var a = "" @@ -1844,102 +2455,6 @@ proc parseCmdLine*(c: string): seq[string] {. inc(i) add(result, a) -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.noNimScript.} = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## This is a wrapper proc around `copyFile() <#copyFile>`_, - ## `getFilePermissions() <#getFilePermissions>`_ and `setFilePermissions() - ## <#setFilePermissions>`_ on non Windows platform. On Windows this proc is - ## just a wrapper for `copyFile() <#copyFile>`_ since that proc already - ## copies attributes. - ## - ## On non Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true, errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - copyFile(source, dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign, noNimScript.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir() - ## <#copyDir>`_ and `copyFileWithPermissions() <#copyFileWithPermissions>`_ - ## on non Windows platforms. On Windows this proc is just a wrapper for - ## `copyDir() <#copyDir>`_ since that proc already copies attributes. - ## - ## On non Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true, errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - createDir(dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - case kind - of pcFile: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors) - of pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: discard - -proc inclFilePermissions*(filename: string, - permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = - ## a convenience procedure for: - ## - ## .. code-block:: nim - ## setFilePermissions(filename, getFilePermissions(filename)+permissions) - setFilePermissions(filename, getFilePermissions(filename)+permissions) - -proc exclFilePermissions*(filename: string, - permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = - ## a convenience procedure for: - ## - ## .. code-block:: nim - ## setFilePermissions(filename, getFilePermissions(filename)-permissions) - setFilePermissions(filename, getFilePermissions(filename)-permissions) - -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = - ## Moves a directory from `source` to `dest`. If this fails, `OSError` is raised. - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) - -proc expandSymlink*(symlinkPath: string): string {.noNimScript.} = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, ``symlinkPath`` is simply returned. - when defined(windows): - result = symlinkPath - else: - result = newString(256) - var len = readlink(symlinkPath, result, 256) - if len < 0: - raiseOSError(osLastError()) - if len > 256: - result = newString(len+1) - len = readlink(symlinkPath, result, len) - setLen(result, len) - when defined(nimdoc): # Common forward declaration docstring block for parameter retrieval procs. proc paramCount*(): int {.tags: [ReadIOEffect].} = @@ -1948,14 +2463,21 @@ when defined(nimdoc): ## ## Unlike `argc`:idx: in C, if your binary was called without parameters this ## will return zero. - ## You can query each individual paramater with `paramStr() <#paramStr>`_ - ## or retrieve all of them in one go with `commandLineParams() + ## You can query each individual paramater with `paramStr proc <#paramStr,int>`_ + ## or retrieve all of them in one go with `commandLineParams proc ## <#commandLineParams>`_. ## - ## **Availability**: When generating a dynamic library (see --app:lib) on + ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. - ## Test for availability using `declared() `_. - ## Example: + ## Test for availability using `declared() `_. + ## + ## See also: + ## * `parseopt module `_ + ## * `parseCmdLine proc <#parseCmdLine,string>`_ + ## * `paramStr proc <#paramStr,int>`_ + ## * `commandLineParams proc <#commandLineParams>`_ + ## + ## **Examples:** ## ## .. code-block:: nim ## when declared(paramCount): @@ -1976,10 +2498,18 @@ when defined(nimdoc): ## contents (usually the name of the invoked executable). You should avoid ## this and call `getAppFilename() <#getAppFilename>`_ instead. ## - ## **Availability**: When generating a dynamic library (see --app:lib) on + ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. - ## Test for availability using `declared() `_. - ## Example: + ## Test for availability using `declared() `_. + ## + ## See also: + ## * `parseopt module `_ + ## * `parseCmdLine proc <#parseCmdLine,string>`_ + ## * `paramCount proc <#paramCount>`_ + ## * `commandLineParams proc <#commandLineParams>`_ + ## * `getAppFilename proc <#getAppFilename>`_ + ## + ## **Examples:** ## ## .. code-block:: nim ## when declared(paramStr): @@ -2052,8 +2582,17 @@ when declared(paramCount) or defined(nimdoc): ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() `_. - ## Example: + ## can test for its availability with `declared() + ## `_. + ## + ## See also: + ## * `parseopt module `_ + ## * `parseCmdLine proc <#parseCmdLine,string>`_ + ## * `paramCount proc <#paramCount>`_ + ## * `paramStr proc <#paramStr,int>`_ + ## * `getAppFilename proc <#getAppFilename>`_ + ## + ## **Examples:** ## ## .. code-block:: nim ## when declared(commandLineParams): @@ -2151,10 +2690,12 @@ when defined(haiku): result = "" proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = - ## Returns the filename of the application's executable. See also - ## `getCurrentCompilerExe`. + ## Returns the filename of the application's executable. + ## This proc will resolve symlinks. ## - ## This procedure will resolve symlinks. + ## See also: + ## * `getAppDir proc <#getAppDir>`_ + ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ # Linux: /proc//exe # Solaris: @@ -2213,10 +2754,13 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noN proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = ## Returns the directory of the application's executable. + ## + ## See also: + ## * `getAppFilename proc <#getAppFilename>`_ result = splitFile(getAppFilename()).dir proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noNimScript.} = - ## sleeps `milsecs` milliseconds. + ## Sleeps `milsecs` milliseconds. when defined(windows): winlean.sleep(int32(milsecs)) else: @@ -2227,7 +2771,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noNimScrip proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = - ## returns the file size of `file` (in bytes). An ``OSError`` exception is + ## Returns the file size of `file` (in bytes). ``OSError`` is ## raised in case of an error. when defined(windows): var a: WIN32_FIND_DATA @@ -2254,14 +2798,19 @@ else: type FileInfo* = object ## Contains information associated with a file object. - id*: tuple[device: DeviceId, file: FileId] # Device and file id. - kind*: PathComponent # Kind of file object - directory, symlink, etc. - size*: BiggestInt # Size of file. - permissions*: set[FilePermission] # File permissions - linkCount*: BiggestInt # Number of hard links the file object has. - lastAccessTime*: times.Time # Time file was last accessed. - lastWriteTime*: times.Time # Time file was last modified/written to. - creationTime*: times.Time # Time file was created. Not supported on all systems! + ## + ## See also: + ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ + ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + id*: tuple[device: DeviceId, file: FileId] ## Device and file id. + kind*: PathComponent ## Kind of file object - directory, symlink, etc. + size*: BiggestInt ## Size of file. + permissions*: set[FilePermission] ## File permissions + linkCount*: BiggestInt ## Number of hard links the file object has. + lastAccessTime*: times.Time ## Time file was last accessed. + lastWriteTime*: times.Time ## Time file was last modified/written to. + creationTime*: times.Time ## Time file was created. Not supported on all systems! template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. @@ -2333,7 +2882,12 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noNimScript.} = ## handle. ## ## If the information cannot be retrieved, such as when the file handle - ## is invalid, an error will be thrown. + ## is invalid, `OSError` is raised. + ## + ## See also: + ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + # Done: ID, Kind, Size, Permissions, Link Count when defined(Windows): var rawInfo: BY_HANDLE_FILE_INFORMATION @@ -2350,6 +2904,11 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noNimScript.} = rawToFormalFileInfo(rawInfo, "", result) proc getFileInfo*(file: File): FileInfo {.noNimScript.} = + ## Retrieves file information for the file object. + ## + ## See also: + ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ + ## * `getFileInfo(path) proc <#getFileInfo,string>`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) @@ -2358,16 +2917,20 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noNimScript.} ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo` structure will be slightly different - ## across platforms, and in some cases, incomplete or inaccurate. + ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## different across platforms, and in some cases, incomplete or inaccurate. ## - ## When `followSymlink` is true, symlinks are followed and the information - ## retrieved is information related to the symlink's target. Otherwise, - ## information on the symlink itself is retrieved. + ## When `followSymlink` is true (default), symlinks are followed and the + ## information retrieved is information related to the symlink's target. + ## Otherwise, information on the symlink itself is retrieved. ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving - ## file information, an error will be thrown. + ## file information, `OSError` is raised. + ## + ## See also: + ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ + ## * `getFileInfo(file) proc <#getFileInfo,File>`_ when defined(Windows): var handle = openHandle(path, followSymlink) @@ -2389,21 +2952,23 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noNimScript.} rawToFormalFileInfo(rawInfo, path, result) proc isHidden*(path: string): bool {.noNimScript.} = - ## Determines whether ``path`` is hidden or not, using this - ## reference https://en.wikipedia.org/wiki/Hidden_file_and_hidden_directory + ## Determines whether ``path`` is hidden or not, using `this + ## reference `_. ## ## On Windows: returns true if it exists and its "hidden" attribute is set. ## ## On posix: returns true if ``lastPathPart(path)`` starts with ``.`` and is - ## not ``.`` or ``..``. Note: paths are not normalized to determine `isHidden`. + ## not ``.`` or ``..``. + ## + ## **Note**: paths are not normalized to determine `isHidden`. runnableExamples: when defined(posix): - doAssert ".foo".isHidden - doAssert: not ".foo/bar".isHidden - doAssert: not ".".isHidden - doAssert: not "..".isHidden - doAssert: not "".isHidden - doAssert ".foo/".isHidden + assert ".foo".isHidden + assert not ".foo/bar".isHidden + assert not ".".isHidden + assert not "..".isHidden + assert not "".isHidden + assert ".foo/".isHidden when defined(Windows): when useWinUnicode: @@ -2417,7 +2982,10 @@ proc isHidden*(path: string): bool {.noNimScript.} = result = len(fileName) >= 2 and fileName[0] == '.' and fileName != ".." proc getCurrentProcessId*(): int {.noNimScript.} = - ## return current process ID. See also ``osproc.processID(p: Process)``. + ## Return current process ID. + ## + ## See also: + ## * `osproc.processID(p: Process) `_ when defined(windows): proc GetCurrentProcessId(): DWORD {.stdcall, dynlib: "kernel32", importc: "GetCurrentProcessId".} @@ -2444,6 +3012,7 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noNimScript.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError()) + when isMainModule: assert quoteShellWindows("aaa") == "aaa" assert quoteShellWindows("aaa\"") == "aaa\\\"" From 0a628f36f077f4ae1162e17549203ba30a8bf1f2 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Jan 2019 18:17:40 +0100 Subject: [PATCH 273/350] destructors: do not produce strong backrefs in closure environments so that refcounting works --- compiler/lambdalifting.nim | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index ddde1be311..1c12688a33 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -320,17 +320,30 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym; rawAddSon(result, obj) c.ownerToType[owner.id] = result +proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym; + info: TLineInfo): PType = + var r = c.getEnvTypeForOwner(owner, info) + result = newType(tyPtr, owner) + rawAddSon(result, r.base) + proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) = let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ let obj = refObj.lastSon - let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ + # The assumption here is that gcDestructors means we cannot deal + # with cycles properly, so it's better to produce a weak ref (=ptr) here. + # This seems to be generally correct but since it's a bit risky it's only + # enabled for gcDestructors. + let fieldType = if c.graph.config.selectedGc == gcDestructors: + c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ + else: + c.getEnvTypeForOwner(dest, info) if refObj == fieldType: localError(c.graph.config, dep.info, "internal error: invalid up reference computed") let upIdent = getIdent(c.graph.cache, upName) let upField = lookupInRecord(obj.n, upIdent) if upField != nil: - if upField.typ != fieldType: + if upField.typ.base != fieldType.base: localError(c.graph.config, dep.info, "internal error: up references do not agree") else: let result = newSym(skField, upIdent, obj.owner, obj.owner.info) @@ -555,7 +568,7 @@ proc rawClosureCreation(owner: PSym; let upField = lookupInRecord(env.typ.lastSon.n, getIdent(d.graph.cache, upName)) if upField != nil: let up = getUpViaParam(d.graph, owner) - if up != nil and upField.typ == up.typ: + if up != nil and upField.typ.base == up.typ.base: result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), up, env.info)) #elif oldenv != nil and oldenv.typ == upField.typ: @@ -586,7 +599,7 @@ proc closureCreationForIter(iter: PNode; let upField = lookupInRecord(v.typ.lastSon.n, getIdent(d.graph.cache, upName)) if upField != nil: let u = setupEnvVar(owner, d, c) - if u.typ == upField.typ: + if u.typ.base == upField.typ.base: result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info), u, iter.info)) else: From 2ce9845fe4bd2196fd08a2bca0b3c259ed8838d2 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Jan 2019 19:48:21 +0100 Subject: [PATCH 274/350] fixes silly typo causing tons of async regressions --- compiler/lambdalifting.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 1c12688a33..ba67f0d4e7 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -336,7 +336,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) = let fieldType = if c.graph.config.selectedGc == gcDestructors: c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ else: - c.getEnvTypeForOwner(dest, info) + c.getEnvTypeForOwner(dep, info) if refObj == fieldType: localError(c.graph.config, dep.info, "internal error: invalid up reference computed") From 47037ebee32792f5cc0ef292b7f2fc7d4c734936 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 30 Jan 2019 19:49:09 +0100 Subject: [PATCH 275/350] fixes #9471 (#10502) --- doc/regexprs.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/regexprs.txt b/doc/regexprs.txt index 5c6d37e898..83dbd2eeb0 100644 --- a/doc/regexprs.txt +++ b/doc/regexprs.txt @@ -80,13 +80,13 @@ meta character meaning ``|`` start of alternative branch ``(`` start subpattern ``)`` end subpattern -``?`` extends the meaning of ``(`` - also 0 or 1 quantifier - also quantifier minimizer -``*`` 0 or more quantifier -``+`` 1 or more quantifier - also "possessive quantifier" ``{`` start min/max quantifier +``?`` extends the meaning of ``(`` + | also 0 or 1 quantifier (equal to ``{0,1}``) + | also quantifier minimizer +``*`` 0 or more quantifier (equal to ``{0,}``) +``+`` 1 or more quantifier (equal to ``{1,}``) + | also "possessive quantifier" ============== ============================================================ From 3d6f2b77b8494265bda01f82746b427c2574da30 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 30 Jan 2019 19:50:20 +0100 Subject: [PATCH 276/350] fix #9725 (#10503) --- doc/manual.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual.rst b/doc/manual.rst index f8a5887916..f17cab1068 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -251,6 +251,10 @@ Another advantage is that it frees the programmer from remembering the exact spelling of an identifier. The exception with respect to the first letter allows common code like ``var foo: Foo`` to be parsed unambiguously. +Note that this rule also applies to keywords, meaning that ``notin`` is +the same as ``notIn`` and ``not_in`` (all-lowercase version (``notin``, ``isnot``) +is the preferred way of writing keywords). + Historically, Nim was a fully `style-insensitive`:idx: language. This meant that it was not case-sensitive and underscores were ignored and there was not even a distinction between ``foo`` and ``Foo``. From acac107fe7fbef2b55cef3264e9e5e10e1f69ea3 Mon Sep 17 00:00:00 2001 From: alaviss Date: Thu, 31 Jan 2019 01:52:21 +0700 Subject: [PATCH 277/350] times: use clock_gettime() for epochTime() (#10499) * times: use clock_gettime() for epochTime() Fixes #10494 * times: remove gettimeofday() As FreeBSD and OSX support CLOCK_REALTIME, there's no reason to keep this code around. --- lib/pure/times.nim | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 0104a97c16..e2d0b87398 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -224,9 +224,6 @@ elif defined(posix): cpuClockId {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "".}: Clockid - proc gettimeofday(tp: var Timeval, unused: pointer = nil) - {.importc: "gettimeofday", header: "".} - when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "".}: int when not defined(valgrind_workaround_10121): @@ -1110,12 +1107,6 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = let nanos = convert(Milliseconds, Nanoseconds, millis mod convert(Seconds, Milliseconds, 1).int) result = initTime(seconds, nanos) - # I'm not entirely certain if freebsd needs to use `gettimeofday`. - elif defined(macosx) or defined(freebsd): - var a: Timeval - gettimeofday(a) - result = initTime(a.tv_sec.int64, - convert(Microseconds, Nanoseconds, a.tv_usec.int)) elif defined(posix): var ts: Timespec discard clock_gettime(realTimeClockId, ts) @@ -2449,9 +2440,10 @@ when not defined(JS): ## ## ``getTime`` should generally be prefered over this proc. when defined(posix): - var a: Timeval - gettimeofday(a) - result = toBiggestFloat(a.tv_sec.int64) + toFloat(a.tv_usec)*0.00_0001 + var ts: Timespec + discard clock_gettime(realTimeClockId, ts) + result = toBiggestFloat(ts.tv_sec.int64) + + toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 elif defined(windows): var f: winlean.FILETIME getSystemTimeAsFileTime(f) From e16310525c6749ceaefe7c75295038081a1d1e09 Mon Sep 17 00:00:00 2001 From: deansher Date: Wed, 30 Jan 2019 16:22:08 -0500 Subject: [PATCH 278/350] Incorporated Araq's initial feedback. --- doc/manual.rst | 99 ++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index b9d5499b22..09265d3c8d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -46,7 +46,7 @@ and ``a ^* b`` is short for ``(a (b a)*)?``. Example:: arrayConstructor = '[' expr ^* ',' ']' -Other parts of Nim, like scoping rules or execution semantics, are +Other parts of Nim, like scoping rules or runtime semantics, are described informally. @@ -78,13 +78,14 @@ Nim code. It is processed by a Nim `compiler`:idx: into an `executable`:idx:. The nature of this executable depends on the compiler implementation; it may, for example, be a native binary or JavaScript source code. -In a typical Nim program, most of the code is compiled into the executable for -execution at `runtime`:idx:. However, some of the code may be executed at +In a typical Nim program, most of the code is compiled into the executable. +However, some of the code may be executed at `compile time`:idx:. This can include constant expressions, macro definitions, and Nim procedures used by macro definitions. Most of the Nim language is supported at compile time, but there are some restrictions -- see `Restrictions on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for -details. +details. We use the term `runtime`:idx: to cover both compile-time execution +and code execution in the executable. The compiler parses Nim source code into an internal data structure called the `abstract syntax tree`:idx: (`AST`:idx:). Then, before executing the code or @@ -94,16 +95,16 @@ identifier meanings, and in some cases expression values. An error detected during semantic analysis is called a `static error`:idx:. Errors described in this manual are static errors when not otherwise specified. -An error detected during code execution (whether at compile time or at runtime) -is a `checked execution error`:idx:. The method for reporting such errors is via +A `checked runtime error`:idx: is an error that the implementation detects +and reports at runtime. The method for reporting such errors is via *raising exceptions* or *dying with a fatal error*. However, the implementation -provides a means to disable these `execution-time checks`:idx:. See the section +provides a means to disable these `runtime checks`:idx:. See the section pragmas_ for details. -Whether a checked execution error results in an exception or in a fatal error is +Whether a checked runtime error results in an exception or in a fatal error is implementation specific. Thus the following program is invalid; even though the code purports to catch the `IndexError` from an out-of-bounds array access, the -compiler may instead choose to allow execution to die with a fatal error. +compiler may instead choose to allow the program to die with a fatal error. .. code-block:: nim var a: array[0..1, char] @@ -113,10 +114,10 @@ compiler may instead choose to allow execution to die with a fatal error. except IndexError: echo "invalid index" -An `unchecked execution error`:idx: is an error that is not guaranteed to be +An `unchecked runtime error`:idx: is an error that is not guaranteed to be detected, and can cause the subsequent behavior of the computation to -be arbitrary. Unchecked execution errors cannot occur if only `safe`:idx: -language features are used and if no execution-time checks are disabled. +be arbitrary. Unchecked runtime errors cannot occur if only `safe`:idx: +language features are used and if no runtime checks are disabled. A `constant expression`:idx: is an expression whose value can be computed during semantic analysis of the code in which it appears. It is never an l-value and @@ -710,17 +711,10 @@ Constants and Constant Expressions ================================== A `constant`:idx: is a symbol that is bound to the value of a constant -expression. This is an expression whose value can be computed during -semantic analysis of the code in which it appears. However, constant -expressions are not limited to the capabilities of semantic analysis; they -can use all Nim language features that are supported for -compile-time execution. Compile-time execution is interleaved with semantic -analysis as necessary. A constant's value cannot change after it is first -computed. - -Constant expressions are restricted to depend only on the following categories -of values and operations, because these are either built into the language or -declared and evaluated before semantic analysis of the constant expression: +expression. Constant expressions are restricted to depend only on the following +categories of values and operations, because these are either built into the +language or declared and evaluated before semantic analysis of the constant +expression: * literals * built-in operators @@ -817,9 +811,9 @@ Ordinal types have the following characteristics: the operation of functions as ``inc``, ``ord``, ``dec`` on ordinal types to be defined. - Ordinal values have a smallest possible value. Trying to count further - down than the smallest value gives a checked execution or static error. + down than the smallest value gives a checked runtime or static error. - Ordinal values have a largest possible value. Trying to count further - than the largest value gives a checked execution or static error. + than the largest value gives a checked runtime or static error. Integers, bool, characters and enumeration types (and subranges of these types) belong to ordinal types. For reasons of simplicity of implementation @@ -927,7 +921,7 @@ lowest and highest value of the type. For example: to 5. ``PositiveFloat`` defines a subrange of all positive floating point values. NaN does not belong to any subrange of floating point types. Assigning any other value to a variable of type ``Subrange`` is a -checked execution error (or static error if it can be determined during +checked runtime error (or static error if it can be determined during semantic analysis). Assignments from the base type to one of its subrange types (and vice versa) are allowed. @@ -1237,7 +1231,7 @@ inferred from the type of the first element. All other elements need to be implicitly convertable to this type. Sequences are similar to arrays but of dynamic length which may change -during execution (like strings). Sequences are implemented as growable arrays, +during runtime (like strings). Sequences are implemented as growable arrays, allocating pieces of memory as items are added. A sequence ``S`` is always indexed by integers from 0 to ``len(S)-1`` and its bounds are checked. Sequences can be constructed by the array constructor ``[]`` in conjunction @@ -1271,7 +1265,7 @@ operator, and remove (and get) the last element of a sequence with the The notation ``x[i]`` can be used to access the i-th element of ``x``. -Arrays are always bounds checked (statically or during execution). These +Arrays are always bounds checked (statically or at runtime). These checks can be disabled via pragmas or invoking the compiler with the ``--boundChecks:off`` command line switch. @@ -1378,7 +1372,7 @@ is currently not checked. **Future directions**: GC'ed memory should be allowed in unchecked arrays and there should be an explicit annotation of how the GC is to determine the -execution-time size of the array. +runtime size of the array. @@ -1435,7 +1429,7 @@ can also be defined with indentation instead of ``[]``: age: natural # and an age Objects provide many features that tuples do not. Object provide inheritance and -information hiding. Objects have access to their type during execution, so that +information hiding. Objects have access to their type during at runtime, so that the ``of`` operator can be used to determine the object's type. The ``of`` operator is similar to the ``instanceof`` operator in Java. @@ -1547,7 +1541,7 @@ contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does not yet know the size of the object. To complete an incomplete object the ``package`` pragma has to be used. ``package`` implies ``byref``. -As long as a type ``T`` is incomplete, neither ``sizeof(T)`` nor execution-time +As long as a type ``T`` is incomplete, neither ``sizeof(T)`` nor runtime type information for ``T`` is available. @@ -2909,17 +2903,18 @@ When nimvm statement -------------------- ``nimvm`` is a special symbol, that may be used as expression of ``when nimvm`` -statement to differentiate execution path between runtime and compile time. +statement to differentiate execution path between compile time and the +executable. Example: .. code-block:: nim proc someProcThatMayRunInCompileTime(): bool = when nimvm: - # This code executes at compile time + # This branch is taken at compile time. result = true else: - # This code executes at runtime + # This branch is taken in the executable. result = false const ctValue = someProcThatMayRunInCompileTime() let rtValue = someProcThatMayRunInCompileTime() @@ -4215,7 +4210,7 @@ The exception tree is defined in the `system `_ module. Every exception inherits from ``system.Exception``. Exceptions that indicate programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``) and are stricly speaking not catchable as they can also be mapped to an operation -that terminates the whole process. Exceptions that indicate any other execution +that terminates the whole process. Exceptions that indicate any other runtime error that can be caught inherit from ``system.CatchableError`` (which is a subtype of ``Exception``). @@ -5454,19 +5449,19 @@ Macros A macro is a special function that is executed at compile time. Normally the input for a macro is an abstract syntax tree (AST) of the code that is passed to it. The macro can then do -transformations on it and return the transformed AST. The -transformed AST is then passed to the compiler as if the macro -invocation would have been replaced by its result in the source -code. This can be used to implement `domain specific -languages`:idx:. +transformations on it and return the transformed AST. This can be used to +add custom language features and implement `domain specific languages`:idx:. Macro invocation is a case where semantic analyis does **not** entirely proceed -top to bottom and left to right. The compiler must +top to bottom and left to right. Instead, semantic analysis happens at least +twice: -* perform semantic analysis through the end of the macro invocation, -* execute the macro body, -* replace the AST of the macro invocation with the AST returned by the macro, -* and finally repeat semantic analysis of that region of the code. +* Semantic analysis recognizes and resolves the macro invocation. +* The compiler executes the macro body (which may invoke other procs). +* It replaces the AST of the macro invocation with the AST returned by the macro. +* It repeats semantic analysis of that region of the code. +* If the AST returned by the macro contains other macro invocations, + this process iterates. While macros enable advanced compile-time code transformations, they cannot change Nim's syntax. However, this is no real restriction because @@ -6968,7 +6963,7 @@ structure: pure pragma ----------- An object type can be marked with the ``pure`` pragma so that its type field -which is used for execution-time type identification is omitted. This used to be +which is used for runtime type identification is omitted. This used to be necessary for binary compatibility with other compiled languages. An enum type can be marked as ``pure``. Then access of its fields always @@ -7149,7 +7144,7 @@ others may be added later). =============== =============== ============================================ pragma allowed values description =============== =============== ============================================ -checks on|off Turns the code generation for all execution +checks on|off Turns the code generation for all runtime checks on or off. boundChecks on|off Turns the code generation for array bound checks on or off. @@ -7176,7 +7171,7 @@ Example: .. code-block:: nim {.checks: off, optimization: speed.} - # compile without execution-time checks and optimize for speed + # compile without runtime checks and optimize for speed push and pop pragmas @@ -7186,7 +7181,7 @@ but are used to override the settings temporarily. Example: .. code-block:: nim {.push checks: off.} - # compile this section without execution-time checks as it is + # compile this section without runtime checks as it is # speed critical # ... some code ... {.pop.} # restore old settings @@ -8408,9 +8403,9 @@ Top level accesses to ``gdata`` are always allowed so that it can be initialized conveniently. It is *assumed* (but not enforced) that every top level statement is executed before any concurrent action happens. -The ``locks`` section deliberately looks ugly because it has no execution-time +The ``locks`` section deliberately looks ugly because it has no runtime semantics and should not be used directly! It should only be used in templates -that also implement some form of locking during execution: +that also implement some form of locking at runtime: .. code-block:: nim template lock(a: TLock; body: untyped) = @@ -8529,7 +8524,7 @@ single ``locks`` section: Here is how a typical multilock statement can be implemented in Nim. Note how -the execution check is required to ensure a global ordering for two locks ``a`` +the runtime check is required to ensure a global ordering for two locks ``a`` and ``b`` of the same lock level: .. code-block:: nim From abcd9fd2e8120caf6e4588dabe0f22ae0cce07d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 30 Jan 2019 22:19:16 -0800 Subject: [PATCH 279/350] fixup #10466 to use runnableExamples (#10510) --- lib/pure/json.nim | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 4a3d5b4323..176da1d9dc 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -710,23 +710,20 @@ proc pretty*(node: JsonNode, indent = 2): string = ## on multiple lines. ## ## Similar to prettyprint in Python. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## let j = %* {"name": "Isaac", "books": ["Robot Dreams"], - ## "details": {"age":35, "pi":3.1415}} - ## echo pretty(j) - ## # { - ## # "name": "Isaac", - ## # "books": [ - ## # "Robot Dreams" - ## # ], - ## # "details": { - ## # "age": 35, - ## # "pi": 3.1415 - ## # } - ## # } + runnableExamples: + let j = %* {"name": "Isaac", "books": ["Robot Dreams"], + "details": {"age":35, "pi":3.1415}} + doAssert pretty(j) == """ +{ + "name": "Isaac", + "books": [ + "Robot Dreams" + ], + "details": { + "age": 35, + "pi": 3.1415 + } +}""" result = "" toPretty(result, node, indent) From 81e5a35d56de9fa7b3c19b6bf9c3139e6481a0ed Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Thu, 31 Jan 2019 07:34:26 +0100 Subject: [PATCH 280/350] Show correct address for ipv6 sockets, fixes #7634 (#10505) --- lib/pure/net.nim | 2 +- lib/windows/winlean.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index a5bdf3ce6a..43284f8724 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1295,7 +1295,7 @@ proc recvFrom*(socket: Socket, data: var string, length: int, if result != -1: data.setLen(result) - address = $inet_ntoa(sockAddress.sin_addr) + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) port = ntohs(sockAddress.sin_port).Port else: raiseOSError(osLastError()) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 6c480d03a0..d1bfbd4477 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -896,7 +896,7 @@ proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, dynlib: "kernel32", importc: "GetProcessTimes".} type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, - stringBufSize: int32): cstring {.gcsafe, stdcall.} + stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} var inet_ntop_real: inet_ntop_proc = nil From fa058773db018405ff218bc8ff4682a192e9131f Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 31 Jan 2019 08:20:00 +0100 Subject: [PATCH 281/350] fixes #10042 (allow spaces in import) (#10504) This allows spaces in imports, by using the following syntax: * `import "directory with spaces" / subdir / file`, or * `import "directory with spaces/subdir/file"` --- compiler/modulepaths.nim | 1 - tests/dir with space/more spaces/mspace.nim | 1 + tests/dir with space/tspace.nim | 4 ++++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 tests/dir with space/more spaces/mspace.nim diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim index 9e27a2d7d5..129f719e25 100644 --- a/compiler/modulepaths.nim +++ b/compiler/modulepaths.nim @@ -114,7 +114,6 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string = try: result = pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir) - .replace(" ") except ValueError: localError(conf, n.info, "invalid path: " & n.strVal) result = n.strVal diff --git a/tests/dir with space/more spaces/mspace.nim b/tests/dir with space/more spaces/mspace.nim new file mode 100644 index 0000000000..bc2c90f5ea --- /dev/null +++ b/tests/dir with space/more spaces/mspace.nim @@ -0,0 +1 @@ +proc tenTimes*(x: int): int = 10*x diff --git a/tests/dir with space/tspace.nim b/tests/dir with space/tspace.nim index 59237c9a16..87a52c2713 100644 --- a/tests/dir with space/tspace.nim +++ b/tests/dir with space/tspace.nim @@ -2,5 +2,9 @@ discard """ output: "Successful" """ # Test for the compiler to be able to compile a Nim file with spaces in the directory name. +# Also test if import of a directory with a space works. +import "more spaces" / mspace + +assert tenTimes(5) == 50 echo("Successful") From ec6e5681daddde1ff81f9235a6f5348d46ab9eff Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 31 Jan 2019 02:44:11 -0800 Subject: [PATCH 282/350] fix #8063 by adding a testcase for: Deprecation warnings for enum values print twice (#10508) --- tests/deprecated/tdeprecated.nim | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/deprecated/tdeprecated.nim b/tests/deprecated/tdeprecated.nim index 920f350cc6..ba8d579adf 100644 --- a/tests/deprecated/tdeprecated.nim +++ b/tests/deprecated/tdeprecated.nim @@ -1,8 +1,21 @@ discard """ - nimout: '''tdeprecated.nim(10, 3) Warning: a is deprecated [Deprecated] -tdeprecated.nim(17, 11) Warning: asdf; enum 'Foo' which contains field 'a' is deprecated [Deprecated] + nimout: ''' +tdeprecated.nim(23, 3) Warning: a is deprecated [Deprecated] +tdeprecated.nim(30, 11) Warning: asdf; enum 'Foo' which contains field 'a' is deprecated [Deprecated] +tdeprecated.nim(40, 16) Warning: use fooX instead; fooA is deprecated [Deprecated] +end ''' """ + + + + + + +## line 15 + + + block: var a {.deprecated.}: array[0..11, int] @@ -17,3 +30,13 @@ block t10111: var _ = a +block: # issue #8063 + type + Foo = enum + fooX + + {.deprecated: [fooA: fooX].} + let + foo: Foo = fooA + echo foo + static: echo "end" From ec26bc472d3f9f6c3fc299da00c9b13733b114f8 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Thu, 31 Jan 2019 13:47:51 +0100 Subject: [PATCH 283/350] Add `encodeQuery` and `?` to Uri module --- lib/pure/uri.nim | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index e49bfb3c6b..f322718d1a 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -119,6 +119,33 @@ proc decodeUrl*(s: string, decodePlus=true): string = inc(j) setLen(result, j) +proc encodeQuery*(query: openArray[(string, string)], usePlus=true, omitEq=true): string = + ## Encodes a set of (key, value) parameters into a URL query string. + ## + ## Every (key, value) pair is URL-encoded and written as ``key=value``. If the + ## value is an empty string then the ``=`` is omitted, unless ``omitEq`` is + ## false. + ## The pairs are joined together by a ``&`` character. + ## + ## The ``usePlus`` parameter is passed down to the `encodeUrl` function that + ## is used for the URL encoding of the string values. + ## + ## **See also:** + ## * `encodeUrl proc<#encodeUrl,string>`_ + runnableExamples: + assert encodeQuery({:}) == "" + assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" + assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + for elem in query: + # Encode the `key = value` pairs and separate them with a '&' + if result.len > 0: result.add('&') + let (key, val) = elem + result.add(encodeUrl(key, usePlus)) + # Omit the '=' if the value string is empty + if not omitEq or val.len > 0: + result.add('=') + result.add(encodeUrl(val, usePlus)) + proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false @@ -392,6 +419,14 @@ proc `/`*(x: Uri, path: string): Uri = result.path.add '/' result.path.add(path) +proc `?`*(u: Uri, query: openArray[(string, string)]): Uri = + ## Concatenates the query parameters to the specified URI object. + runnableExamples: + let foo = parseUri("https://example.com") / "foo" ? {"bar": "qux"} + assert $foo == "https://example.com/foo?bar=qux" + result = u + result.query = encodeQuery(query) + proc `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. runnableExamples: @@ -676,4 +711,25 @@ when isMainModule: doAssert "https://example.com/about/staff.html?".parseUri().isAbsolute == true doAssert "https://example.com/about/staff.html?parameters".parseUri().isAbsolute == true + # encodeQuery tests + block: + doAssert encodeQuery({:}) == "" + doAssert encodeQuery({"foo": "bar"}) == "foo=bar" + doAssert encodeQuery({"foo": "bar & baz"}) == "foo=bar+%26+baz" + doAssert encodeQuery({"foo": "bar & baz"}, usePlus=false) == "foo=bar%20%26%20baz" + doAssert encodeQuery({"foo": ""}) == "foo" + doAssert encodeQuery({"foo": ""}, omitEq=false) == "foo=" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}) == "a=1&b&c=3" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq=false) == "a=1&b=&c=3" + + block: + var foo = parseUri("http://example.com") / "foo" ? {"bar": "1", "baz": "qux"} + var foo1 = parseUri("http://example.com/foo?bar=1&baz=qux") + doAssert foo == foo1 + + block: + var foo = parseUri("http://example.com") / "foo" ? {"do": "do", "bar": ""} + var foo1 = parseUri("http://example.com/foo?do=do&bar") + doAssert foo == foo1 + echo("All good!") From 0ecec8362a8a110603fc0a7afd4b3f74287074ef Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 31 Jan 2019 15:12:36 +0100 Subject: [PATCH 284/350] distros.nim: brew usually requires 'sudo' --- lib/pure/distros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 0f1ffb1abc..4c531a779e 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -232,7 +232,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = elif defined(haiku): result = ("pkgman install " & p, true) else: - result = ("brew install " & p, false) + result = ("brew install " & p, true) proc foreignDep*(foreignPackageName: string) = ## Registers 'foreignPackageName' to the internal list of foreign deps. From 1d5437e9d2c5800e4b90e87e32acc600c52cf739 Mon Sep 17 00:00:00 2001 From: cooldome Date: Thu, 31 Jan 2019 18:48:39 +0000 Subject: [PATCH 285/350] vm fix for bitwise signed ints (#10507) * fixes #10482 * add missing file * bug fix --- compiler/vmgen.nim | 8 ++++---- tests/vm/tbitops.nim | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 tests/vm/tbitops.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index f87821da4f..4b1551884f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -987,10 +987,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.freeTemp(tmp2) of mShlI: genBinaryABCnarrowU(c, n, dest, opcShlInt) - of mAshrI: genBinaryABCnarrow(c, n, dest, opcAshrInt) - of mBitandI: genBinaryABCnarrowU(c, n, dest, opcBitandInt) - of mBitorI: genBinaryABCnarrowU(c, n, dest, opcBitorInt) - of mBitxorI: genBinaryABCnarrowU(c, n, dest, opcBitxorInt) + of mAshrI: genBinaryABC(c, n, dest, opcAshrInt) + of mBitandI: genBinaryABC(c, n, dest, opcBitandInt) + of mBitorI: genBinaryABC(c, n, dest, opcBitorInt) + of mBitxorI: genBinaryABC(c, n, dest, opcBitxorInt) of mAddU: genBinaryABCnarrowU(c, n, dest, opcAddu) of mSubU: genBinaryABCnarrowU(c, n, dest, opcSubu) of mMulU: genBinaryABCnarrowU(c, n, dest, opcMulu) diff --git a/tests/vm/tbitops.nim b/tests/vm/tbitops.nim new file mode 100644 index 0000000000..3d1a8aa0cd --- /dev/null +++ b/tests/vm/tbitops.nim @@ -0,0 +1,39 @@ +discard """ +output: "" +""" + +import strutils + +const x = [1'i32, -1, -10, 10, -10, 10, -20, 30, -40, 50, 7 shl 28, -(7 shl 28), 7 shl 28, -(7 shl 28)] +const y = [-1'i32, 1, -10, -10, 10, 10, -20, -30, 40, 50, 1 shl 30, 1 shl 30, -(1 shl 30), -(1 shl 30)] + + +const res_xor = block: + var tmp: seq[int64] + for i in 0.. Date: Fri, 1 Feb 2019 12:12:10 +0100 Subject: [PATCH 286/350] Fix vm signed xor (#10519) * fix #10482 * undo changes * fix for bitwise not * remove dead opcode --- compiler/vmgen.nim | 7 +++++-- tests/vm/tbitops.nim | 16 +++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 4b1551884f..7f3ca84ae0 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1009,7 +1009,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu) of mEqProc, mEqRef, mEqUntracedRef: genBinaryABC(c, n, dest, opcEqRef) - of mXor: genBinaryABCnarrowU(c, n, dest, opcXor) + of mXor: genBinaryABC(c, n, dest, opcXor) of mNot: genUnaryABC(c, n, dest, opcNot) of mUnaryMinusI, mUnaryMinusI64: genUnaryABC(c, n, dest, opcUnaryMinusInt) @@ -1018,7 +1018,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mUnaryPlusI, mUnaryPlusF64: gen(c, n.sons[1], dest) of mBitnotI: genUnaryABC(c, n, dest, opcBitnotInt) - genNarrowU(c, n, dest) + #genNarrowU modified, do not narrow signed types + let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) of mToFloat, mToBiggestFloat, mToInt, mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: diff --git a/tests/vm/tbitops.nim b/tests/vm/tbitops.nim index 3d1a8aa0cd..90d463ec96 100644 --- a/tests/vm/tbitops.nim +++ b/tests/vm/tbitops.nim @@ -7,25 +7,29 @@ import strutils const x = [1'i32, -1, -10, 10, -10, 10, -20, 30, -40, 50, 7 shl 28, -(7 shl 28), 7 shl 28, -(7 shl 28)] const y = [-1'i32, 1, -10, -10, 10, 10, -20, -30, 40, 50, 1 shl 30, 1 shl 30, -(1 shl 30), -(1 shl 30)] - const res_xor = block: var tmp: seq[int64] - for i in 0.. Date: Fri, 1 Feb 2019 19:37:24 +0100 Subject: [PATCH 287/350] quick-fix for os.nim documentation --- lib/pure/os.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 53bf880b64..878444e742 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -425,8 +425,9 @@ proc `/../`*(head, tail: string): string {.noSideEffect.} = ## * `/ proc <#/,string,string>`_ ## * `parentDir proc <#parentDir,string>`_ runnableExamples: - assert "a/b/c" /../ "d/e" == "a/b/d/e" - assert "a" /../ "d/e" == "a/d/e" + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" let sepPos = parentDirPos(head) if sepPos >= 0: From 237a36075a0af173fc4680e3e65eda246e097244 Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 1 Feb 2019 19:44:19 +0100 Subject: [PATCH 288/350] another quickfix for os.nim documentation --- lib/pure/os.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 878444e742..e88c3c6e8c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1252,9 +1252,10 @@ proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [], noNimScr ## * `normalizedPath proc <#normalizedPath,string>`_ for a version which returns ## a new string runnableExamples: - var a = "a///b//..//c///d" - a.normalizePath() - assert a == "a/c/d" + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" path = pathnorm.normalizePath(path) when false: @@ -1291,7 +1292,8 @@ proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [], noN ## * `absolutePath proc <#absolutePath,string>`_ ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version runnableExamples: - assert normalizedPath("a///b//..//c///d") == "a/c/d" + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" result = pathnorm.normalizePath(path) when defined(Windows) and not weirdTarget: From 1ac28f6f5e82d8ca1c62ca8d27c60470e8249835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Fri, 1 Feb 2019 20:05:05 +0100 Subject: [PATCH 289/350] Deprecate the times.countX procs (#10522) --- lib/pure/times.nim | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index e2d0b87398..3fb9ccdb70 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -2360,26 +2360,38 @@ proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local -proc countLeapYears*(yearSpan: int): int = +proc countLeapYears*(yearSpan: int): int + {.deprecated.} = ## Returns the number of leap years spanned by a given number of years. ## ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. + ## + ## **Deprecated since v0.20.0**. (yearSpan - 1) div 4 - (yearSpan - 1) div 100 + (yearSpan - 1) div 400 -proc countDays*(yearSpan: int): int = +proc countDays*(yearSpan: int): int + {.deprecated.} = ## Returns the number of days spanned by a given number of years. + ## + ## **Deprecated since v0.20.0**. (yearSpan - 1) * 365 + countLeapYears(yearSpan) -proc countYears*(daySpan: int): int = +proc countYears*(daySpan: int): int + {.deprecated.} = ## Returns the number of years spanned by a given number of days. + ## + ## **Deprecated since v0.20.0**. ((daySpan - countLeapYears(daySpan div 365)) div 365) -proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = +proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] + {.deprecated.} = ## Returns the number of years spanned by a given number of days and the ## remainder as days. + ## + ## **Deprecated since v0.20.0**. let days = daySpan - countLeapYears(daySpan div 365) result.years = days div 365 result.days = days mod 365 From 2b35a516514872409b38729f13b340ede0f1beaf Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 1 Feb 2019 14:56:31 -0800 Subject: [PATCH 290/350] brew almost never requires sudo (#10525) --- lib/pure/distros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 4c531a779e..0f1ffb1abc 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -232,7 +232,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = elif defined(haiku): result = ("pkgman install " & p, true) else: - result = ("brew install " & p, true) + result = ("brew install " & p, false) proc foreignDep*(foreignPackageName: string) = ## Registers 'foreignPackageName' to the internal list of foreign deps. From f4c76bc8e574d998d4ed2052e9349c76015058b7 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 2 Feb 2019 00:18:52 +0100 Subject: [PATCH 291/350] testament: revive ability to test specific important Nimble packages --- testament/categories.nim | 71 ++++++++++++++++++-------------- testament/important_packages.nim | 29 +++++++++++++ 2 files changed, 68 insertions(+), 32 deletions(-) create mode 100644 testament/important_packages.nim diff --git a/testament/categories.nim b/testament/categories.nim index e2b5f0a789..1365690147 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,6 +10,8 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. +import important_packages + const specialCategories = [ "assert", @@ -448,7 +450,7 @@ if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" let nimbleExe = findExe("nimble") #packageDir = nimbleDir / "pkgs" # not used - packageIndex = nimbleDir / "packages.json" + packageIndex = nimbleDir / "packages_official.json" proc waitForExitEx(p: Process): int = var outp = outputStream(p) @@ -463,7 +465,7 @@ proc waitForExitEx(p: Process): int = proc getPackageDir(package: string): string = ## TODO - Replace this with dom's version comparison magic. - var commandOutput = execCmdEx("nimble path $#" % package) + let commandOutput = execCmdEx("nimble path $#" % package) if commandOutput.exitCode != QuitSuccess: return "" else: @@ -471,20 +473,26 @@ proc getPackageDir(package: string): string = iterator listPackages(filter: PackageFilter): tuple[name, url: string] = let packageList = parseFile(packageIndex) - for package in packageList.items(): - let - name = package["name"].str - url = package["url"].str - isCorePackage = "nim-lang" in normalize(url) - case filter: - of pfCoreOnly: - if isCorePackage: - yield (name, url) - of pfExtraOnly: - if not isCorePackage: - yield (name, url) - of pfAll: - yield (name, url) + for package in packageList.items: + if package.hasKey("url"): + let name = package["name"].str + if name notin ["nimble", "compiler"]: + let url = package["url"].str + case filter + of pfCoreOnly: + if "nim-lang" in normalize(url): + yield (name, url) + of pfExtraOnly: + if name in important_packages.packages: + yield (name, url) + of pfAll: + yield (name, url) + +proc makeSupTest(test, options: string, cat: Category): TTest = + result.cat = cat + result.name = test + result.options = options + result.startTime = epochTime() proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = if nimbleExe == "": @@ -495,31 +503,30 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = echo("[Warning] - Cannot run nimble tests: Nimble update failed.") return - let packageFileTest = makeTest("PackageFileParsed", "", cat) + let packageFileTest = makeSupTest("PackageFileParsed", "", cat) try: for name, url in listPackages(filter): - var test = makeTest(name, "", cat) - echo(url) - let - installProcess = startProcess(nimbleExe, "", ["install", "-y", name]) - installStatus = waitForExitEx(installProcess) + var test = makeSupTest(url, "", cat) + let buildPath = "pkgstemp" / name + let installProcess = startProcess("git", "", ["clone", url, buildPath]) + let installStatus = waitForExitEx(installProcess) installProcess.close if installStatus != QuitSuccess: r.addResult(test, targetC, "", "", reInstallFailed) - continue - - let - buildPath = getPackageDir(name).strip - buildProcess = startProcess(nimbleExe, buildPath, ["build"]) - buildStatus = waitForExitEx(buildProcess) - buildProcess.close - if buildStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reBuildFailed) - r.addResult(test, targetC, "", "", reSuccess) + else: + let buildProcess = startProcess(nimbleExe, buildPath, ["test"]) + let buildStatus = waitForExitEx(buildProcess) + buildProcess.close + if buildStatus != QuitSuccess: + r.addResult(test, targetC, "", "", reBuildFailed) + else: + r.addResult(test, targetC, "", "", reSuccess) r.addResult(packageFileTest, targetC, "", "", reSuccess) except JsonParsingError: echo("[Warning] - Cannot run nimble tests: Invalid package file.") r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + finally: + removeDir("pkgstemp") # ---------------------------------------------------------------------------- diff --git a/testament/important_packages.nim b/testament/important_packages.nim new file mode 100644 index 0000000000..1ded00b007 --- /dev/null +++ b/testament/important_packages.nim @@ -0,0 +1,29 @@ +import strutils + +const + packages* = """ + karax + glob + """.splitWhitespace() + +#[ + jester + cligen + libffi + nimongo + nimx + karax + freeimage + regex + nimpy + zero_functional + arraymancer + inim + c2nim + sdl1 + iterutils + gnuplot + nimpb + lazy + choosenim +]# \ No newline at end of file From 17d0ce9c1c32827e5fc40cb486d4e8fb16f377fc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 2 Feb 2019 00:21:13 +0100 Subject: [PATCH 292/350] DOM API: make compatible with Karax's requirements (#10517) * DOM API: make compatible with Karax's requirements * make tools\dochack.nim compile again --- lib/js/dom.nim | 891 ++++++++++++++++++++++++++++++++------ tools/dochack/dochack.nim | 45 +- tools/dochack/karax.nim | 344 --------------- 3 files changed, 798 insertions(+), 482 deletions(-) delete mode 100644 tools/dochack/karax.nim diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 668dee822e..945d6cda1a 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -13,6 +13,9 @@ when not defined(js) and not defined(Nimdoc): {.error: "This module only works on the JavaScript platform".} +const + DomApiVersion* = 3 ## the version of DOM API we try to follow. No guarantees though. + type EventTarget* = ref EventTargetObj EventTargetObj {.importc.} = object of RootObj @@ -37,6 +40,38 @@ type onsubmit*: proc (event: Event) {.nimcall.} onunload*: proc (event: Event) {.nimcall.} + # https://developer.mozilla.org/en-US/docs/Web/Events + DomEvent* {.pure.} = enum + Abort = "abort", + BeforeInput = "beforeinput", + Blur = "blur", + Click = "click", + CompositionEnd = "compositionend", + CompositionStart = "compositionstart", + CompositionUpdate = "compositionupdate", + DblClick = "dblclick", + Error = "error", + Focus = "focus", + FocusIn = "focusin", + FocusOut = "focusout", + Input = "input", + KeyDown = "keydown", + KeyPress = "keypress", + KeyUp = "keyup", + Load = "load", + MouseDown = "mousedown", + MouseEnter = "mouseenter", + MouseLeave = "mouseleave", + MouseMove = "mousemove", + MouseOut = "mouseout", + MouseOver = "mouseover", + MouseUp = "mouseup", + Resize = "resize", + Scroll = "scroll", + Select = "select", + Unload = "unload", + Wheel = "wheel" + Window* = ref WindowObj WindowObj {.importc.} = object of EventTargetObj document*: Document @@ -45,24 +80,19 @@ type location*: Location closed*: bool defaultStatus*: cstring - devicePixelRatio*: float innerHeight*, innerWidth*: int - locationbar*: ref TLocationBar - menubar*: ref TMenuBar + locationbar*: ref LocationBar + menubar*: ref MenuBar name*: cstring outerHeight*, outerWidth*: int pageXOffset*, pageYOffset*: int - personalbar*: ref TPersonalBar - scrollbars*: ref TScrollBars - scrollX*: float - scrollY*: float - statusbar*: ref TStatusBar + personalbar*: ref PersonalBar + scrollbars*: ref ScrollBars + statusbar*: ref StatusBar status*: cstring - toolbar*: ref TToolBar - frames*: seq[TFrame] + toolbar*: ref ToolBar + frames*: seq[Frame] screen*: Screen - performance*: Performance - onpopstate*: proc (event: Event) Frame* = ref FrameObj FrameObj {.importc.} = object of WindowObj @@ -135,10 +165,8 @@ type name*: cstring readOnly*: bool options*: seq[OptionElement] + selectedOptions*: seq[OptionElement] clientWidth*, clientHeight*: int - - # https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement - HtmlElement* = ref object of Element contentEditable*: cstring isContentEditable*: bool dir*: cstring @@ -147,6 +175,87 @@ type offsetLeft*: int offsetTop*: int + # https://developer.mozilla.org/en-US/docs/Web/API/ValidityState + ValidityState* = ref ValidityStateObj + ValidityStateObj {.importc.} = object + badInput*: bool + customError*: bool + patternMismatch*: bool + rangeOverflow*: bool + rangeUnderflow*: bool + stepMismatch*: bool + tooLong*: bool + tooShort*: bool + typeMismatch*: bool + valid*: bool + valueMissing*: bool + + # https://developer.mozilla.org/en-US/docs/Web/API/Blob + Blob* = ref BlobObj + BlobObj {.importc.} = object of RootObj + size*: int + `type`*: cstring + + # https://developer.mozilla.org/en-US/docs/Web/API/File + File* = ref FileObj + FileObj {.importc.} = object of Blob + lastModified*: int + name*: cstring + + # https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement + InputElement* = ref InputElementObj + InputElementObj {.importc.} = object of Element + # Properties related to the parent form + formAction*: cstring + formEncType*: cstring + formMethod*: cstring + formNoValidate*: bool + formTarget*: cstring + + # Properties that apply to any type of input element that is not hidden + `type`*: cstring + autofocus*: bool + required*: bool + value*: cstring + validity*: ValidityState + validationMessage*: cstring + willValidate*: bool + + # Properties that apply only to elements of type "checkbox" or "radio" + indeterminate*: bool + + # Properties that apply only to elements of type "image" + alt*: cstring + height*: cstring + src*: cstring + width*: cstring + + # Properties that apply only to elements of type "file" + accept*: cstring + files*: seq[Blob] + + # Properties that apply only to text/number-containing or elements + autocomplete*: cstring + maxLength*: int + size*: int + pattern*: cstring + placeholder*: cstring + min*: cstring + max*: cstring + selectionStart*: int + selectionEnd*: int + selectionDirection*: cstring + + # Properties not yet categorized + dirName*: cstring + accessKey*: cstring + list*: Element + multiple*: bool + labels*: seq[Element] + step*: cstring + valueAsDate*: cstring + valueAsNumber*: float + LinkElement* = ref LinkObj LinkObj {.importc.} = object of ElementObj target*: cstring @@ -176,19 +285,19 @@ type text*: cstring value*: cstring - TextAreaElement* = ref object of ElementObj - value*: cstring - selectionStart*, selectionEnd*: int - selectionDirection*: cstring - rows*, cols*: int - + # https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement FormElement* = ref FormObj FormObj {.importc.} = object of ElementObj + acceptCharset*: cstring action*: cstring - encoding*: cstring - `method`*: cstring - target*: cstring + autocomplete*: cstring elements*: seq[Element] + encoding*: cstring + enctype*: cstring + length*: int + `method`*: cstring + noValidate*: bool + target*: cstring ImageElement* = ref ImageObj ImageObj {.importc.} = object of ElementObj @@ -264,8 +373,6 @@ type minHeight*: cstring minWidth*: cstring overflow*: cstring - overflowX*: cstring - overflowY*: cstring padding*: cstring paddingBottom*: cstring paddingLeft*: cstring @@ -273,6 +380,7 @@ type paddingTop*: cstring pageBreakAfter*: cstring pageBreakBefore*: cstring + pointerEvents*: cstring position*: cstring right*: cstring scrollbar3dLightColor*: cstring @@ -288,6 +396,7 @@ type textDecoration*: cstring textIndent*: cstring textTransform*: cstring + transform*: cstring top*: cstring verticalAlign*: cstring visibility*: cstring @@ -295,55 +404,467 @@ type wordSpacing*: cstring zIndex*: int - # TODO: A lot of the fields in Event belong to a more specific type of event. - # TODO: Should we clean this up? + EventPhase* = enum + None = 0, + CapturingPhase, + AtTarget, + BubblingPhase + + # https://developer.mozilla.org/en-US/docs/Web/API/Event Event* = ref EventObj EventObj {.importc.} = object of RootObj + bubbles*: bool + cancelBubble*: bool + cancelable*: bool + composed*: bool + currentTarget*: Node + defaultPrevented*: bool + eventPhase*: int target*: Node - altKey*, ctrlKey*, shiftKey*: bool - button*: int - clientX*, clientY*: int + `type`*: cstring + isTrusted*: bool + + # https://developer.mozilla.org/en-US/docs/Web/API/UIEvent + UIEvent* = ref UIEventObj + UIEventObj {.importc.} = object of Event + detail*: int64 + view*: Window + + # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent + KeyboardEvent* = ref KeyboardEventObj + KeyboardEventObj {.importc.} = object of UIEvent + altKey*, ctrlKey*, metaKey*, shiftKey*: bool + code*: cstring + isComposing*: bool + key*: cstring keyCode*: int - layerX*, layerY*: int - modifiers*: int - ALT_MASK*, CONTROL_MASK*, SHIFT_MASK*, META_MASK*: int + location*: int + + # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values + KeyboardEventKey* {.pure.} = enum + # Modifier keys + Alt, + AltGraph, + CapsLock, + Control, + Fn, + FnLock, + Hyper, + Meta, + NumLock, + ScrollLock, + Shift, + Super, + Symbol, + SymbolLock, + + # Whitespace keys + ArrowDown, + ArrowLeft, + ArrowRight, + ArrowUp, + End, + Home, + PageDown, + PageUp, + + # Editing keys + Backspace, + Clear, + Copy, + CrSel, + Cut, + Delete, + EraseEof, + ExSel, + Insert, + Paste, + Redo, + Undo, + + # UI keys + Accept, + Again, + Attn, + Cancel, + ContextMenu, + Escape, + Execute, + Find, + Finish, + Help, + Pause, + Play, + Props, + Select, + ZoomIn, + ZoomOut, + + # Device keys + BrigtnessDown, + BrigtnessUp, + Eject, + LogOff, + Power, + PowerOff, + PrintScreen, + Hibernate, + Standby, + WakeUp, + + # Common IME keys + AllCandidates, + Alphanumeric, + CodeInput, + Compose, + Convert, + Dead, + FinalMode, + GroupFirst, + GroupLast, + GroupNext, + GroupPrevious, + ModeChange, + NextCandidate, + NonConvert, + PreviousCandidate, + Process, + SingleCandidate, + + # Korean keyboards only + HangulMode, + HanjaMode, + JunjaMode, + + # Japanese keyboards only + Eisu, + Hankaku, + Hiragana, + HiraganaKatakana, + KanaMode, + KanjiMode, + Katakana, + Romaji, + Zenkaku, + ZenkakuHanaku, + + # Function keys + F1, + F2, + F3, + F4, + F5, + F6, + F7, + F8, + F9, + F10, + F11, + F12, + F13, + F14, + F15, + F16, + F17, + F18, + F19, + F20, + Soft1, + Soft2, + Soft3, + Soft4, + + # Phone keys + AppSwitch, + Call, + Camera, + CameraFocus, + EndCall, + GoBack, + GoHome, + HeadsetHook, + LastNumberRedial, + Notification, + MannerMode, + VoiceDial, + + # Multimedia keys + ChannelDown, + ChannelUp, + MediaFastForward, + MediaPause, + MediaPlay, + MediaPlayPause, + MediaRecord, + MediaRewind, + MediaStop, + MediaTrackNext, + MediaTrackPrevious, + + # Audio control keys + AudioBalanceLeft, + AudioBalanceRight, + AudioBassDown, + AudioBassBoostDown, + AudioBassBoostToggle, + AudioBassBoostUp, + AudioBassUp, + AudioFaderFront, + AudioFaderRear, + AudioSurroundModeNext, + AudioTrebleDown, + AudioTrebleUp, + AudioVolumeDown, + AUdioVolumeMute, + AudioVolumeUp, + MicrophoneToggle, + MicrophoneVolumeDown, + MicrophoneVolumeMute, + MicrophoneVolumeUp, + + # TV control keys + TV, + TV3DMode, + TVAntennaCable, + TVAudioDescription, + TVAudioDescriptionMixDown, + TVAudioDescriptionMixUp, + TVContentsMenu, + TVDataService, + TVInput, + TVInputComponent1, + TVInputComponent2, + TVInputComposite1, + TVInputComposite2, + TVInputHDMI1, + TVInputHDMI2, + TVInputHDMI3, + TVInputHDMI4, + TVInputVGA1, + TVMediaContext, + TVNetwork, + TVNumberEntry, + TVPower, + TVRadioService, + TVSatellite, + TVSatelliteBS, + TVSatelliteCS, + TVSatelliteToggle, + TVTerrestrialAnalog, + TVTerrestrialDigital, + TVTimer, + + # Media controller keys + AVRInput, + AVRPower, + ColorF0Red, + ColorF1Green, + ColorF2Yellow, + ColorF3Blue, + ColorF4Grey, + ColorF5Brown, + ClosedCaptionToggle, + Dimmer, + DisplaySwap, + DVR, + Exit, + FavoriteClear0, + FavoriteClear1, + FavoriteClear2, + FavoriteClear3, + FavoriteRecall0, + FavoriteRecall1, + FavoriteRecall2, + FavoriteRecall3, + FavoriteStore0, + FavoriteStore1, + FavoriteStore2, + FavoriteStore3, + Guide, + GuideNextDay, + GuidePreviousDay, + Info, + InstantReplay, + Link, + ListProgram, + LiveContent, + Lock, + MediaApps, + MediaAudioTrack, + MediaLast, + MediaSkipBackward, + MediaSkipForward, + MediaStepBackward, + MediaStepForward, + MediaTopMenu, + NavigateIn, + NavigateNext, + NavigateOut, + NavigatePrevious, + NextFavoriteChannel, + NextUserProfile, + OnDemand, + Pairing, + PinPDown, + PinPMove, + PinPUp, + PlaySpeedDown, + PlaySpeedReset, + PlaySpeedUp, + RandomToggle, + RcLowBattery, + RecordSpeedNext, + RfBypass, + ScanChannelsToggle, + ScreenModeNext, + Settings, + SplitScreenToggle, + STBInput, + STBPower, + Subtitle, + Teletext, + VideoModeNext, + Wink, + ZoomToggle, + + # Speech recognition keys + SpeechCorrectionList, + SpeechInputToggle, + + # Document keys + Close, + New, + Open, + Print, + Save, + SpellCheck, + MailForward, + MailReply, + MailSend, + + # Application selector keys + LaunchCalculator, + LaunchCalendar, + LaunchContacts, + LaunchMail, + LaunchMediaPlayer, + LaunchMusicPlayer, + LaunchMyComputer, + LaunchPhone, + LaunchScreenSaver, + LaunchSpreadsheet, + LaunchWebBrowser, + LaunchWebCam, + LaunchWordProcessor, + LaunchApplication1, + LaunchApplication2, + LaunchApplication3, + LaunchApplication4, + LaunchApplication5, + LaunchApplication6, + LaunchApplication7, + LaunchApplication8, + LaunchApplication9, + LaunchApplication10, + LaunchApplication11, + LaunchApplication12, + LaunchApplication13, + LaunchApplication14, + LaunchApplication15, + LaunchApplication16, + + # Browser control keys + BrowserBack, + BrowserFavorites, + BrowserForward, + BrowserHome, + BrowserRefresh, + BrowserSearch, + BrowserStop, + + # Numeric keypad keys + Key11, + Key12, + Separator + + MouseButtons* = enum + NoButton = 0, + PrimaryButton = 1, + SecondaryButton = 2, + AuxilaryButton = 4, + FourthButton = 8, + FifthButton = 16 + + # https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent + MouseEvent* = ref MouseEventObj + MouseEventObj {.importc.} = object of UIEvent + altKey*, ctrlKey*, metaKey*, shiftKey*: bool + button*: int + buttons*: int + clientX*, clientY*: int + movementX*, movementY*: int offsetX*, offsetY*: int pageX*, pageY*: int + relatedTarget*: EventTarget + #region*: cstring screenX*, screenY*: int - which*: int - `type`*: cstring x*, y*: int - ABORT*: int - BLUR*: int - CHANGE*: int - CLICK*: int - DBLCLICK*: int - DRAGDROP*: int - ERROR*: int - FOCUS*: int - KEYDOWN*: int - KEYPRESS*: int - KEYUP*: int - LOAD*: int - MOUSEDOWN*: int - MOUSEMOVE*: int - MOUSEOUT*: int - MOUSEOVER*: int - MOUSEUP*: int - MOVE*: int - RESET*: int - RESIZE*: int - SELECT*: int - SUBMIT*: int - UNLOAD*: int + + DataTransferItemKind* {.pure.} = enum + File = "file", + String = "string" + + # https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem + DataTransferItem* = ref DataTransferItemObj + DataTransferItemObj {.importc.} = object of RootObj + kind*: cstring + `type`*: cstring + + # https://developer.mozilla.org/en-US/docs/Web/API/DataTransfer + DataTransfer* = ref DataTransferObj + DataTransferObj {.importc.} = object of RootObj + dropEffect*: cstring + effectAllowed*: cstring + files*: seq[Element] + items*: seq[DataTransferItem] + types*: seq[cstring] + + DataTransferDropEffect* {.pure.} = enum + None = "none", + Copy = "copy", + Link = "link", + Move = "move" + + DataTransferEffectAllowed* {.pure.} = enum + None = "none", + Copy = "copy", + CopyLink = "copyLink", + CopyMove = "copyMove", + Link = "link", + LinkMove = "linkMove", + Move = "move", + All = "all", + Uninitialized = "uninitialized" + + DragEventTypes* = enum + Drag = "drag", + DragEnd = "dragend", + DragEnter = "dragenter", + DragExit = "dragexit", + DragLeave = "dragleave", + DragOver = "dragover", + DragStart = "dragstart", + Drop = "drop" + + # https://developer.mozilla.org/en-US/docs/Web/API/DragEvent + DragEvent* {.importc.} = object of MouseEvent + dataTransfer*: DataTransfer TouchList* {.importc.} = ref object of RootObj length*: int - TouchEvent* {.importc.} = ref object of Event - changedTouches*, targetTouches*, touches*: TouchList - - Touch* {.importc.} = ref object of RootObj + Touch* = ref TouchObj + TouchObj {.importc.} = object of RootObj identifier*: int screenX*, screenY*, clientX*, clientY*, pageX*, pageY*: int target*: Element @@ -351,6 +872,10 @@ type rotationAngle*: int force*: float + TouchEvent* = ref TouchEventObj + TouchEventObj {.importc.} = object of UIEvent + changedTouches*, targetTouches*, touches*: seq[Touch] + Location* = ref LocationObj LocationObj {.importc.} = object of RootObj hash*: cstring @@ -375,26 +900,26 @@ type language*: cstring platform*: cstring userAgent*: cstring - mimeTypes*: seq[ref TMimeType] + mimeTypes*: seq[ref MimeType] - TPlugin* {.importc.} = object of RootObj + Plugin* {.importc.} = object of RootObj description*: cstring filename*: cstring name*: cstring - TMimeType* {.importc.} = object of RootObj + MimeType* {.importc.} = object of RootObj description*: cstring - enabledPlugin*: ref TPlugin + enabledPlugin*: ref Plugin suffixes*: seq[cstring] `type`*: cstring - TLocationBar* {.importc.} = object of RootObj + LocationBar* {.importc.} = object of RootObj visible*: bool - TMenuBar* = TLocationBar - TPersonalBar* = TLocationBar - TScrollBars* = TLocationBar - TToolBar* = TLocationBar - TStatusBar* = TLocationBar + MenuBar* = LocationBar + PersonalBar* = LocationBar + ScrollBars* = LocationBar + ToolBar* = LocationBar + StatusBar* = LocationBar Screen = ref ScreenObj ScreenObj {.importc.} = object of RootObj @@ -405,62 +930,120 @@ type pixelDepth*: int width*: int - TTimeOut* {.importc.} = object of RootObj - TInterval* {.importc.} = object of RootObj + TimeOut* {.importc.} = ref object of RootObj + Interval* {.importc.} = object of RootObj AddEventListenerOptions* = object capture*: bool once*: bool passive*: bool - BoundingRect* {.importc.} = ref object - top*, bottom*, left*, right*, x*, y*, width*, height*: float +proc id*(n: Node): cstring {.importcpp: "#.id", nodecl.} +proc `id=`*(n: Node; x: cstring) {.importcpp: "#.id = #", nodecl.} +proc class*(n: Node): cstring {.importcpp: "#.className", nodecl.} +proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} - PerformanceMemory* {.importc.} = ref object - jsHeapSizeLimit*: float - totalJSHeapSize*: float - usedJSHeapSize*: float +proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.} +proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.} - PerformanceTiming* {.importc.} = ref object - connectStart*: float - domComplete*: float - domContentLoadedEventEnd*: float - domContentLoadedEventStart*: float - domInteractive*: float - domLoading*: float - domainLookupEnd*: float - domainLookupStart*: float - fetchStart*: float - loadEventEnd*: float - loadEventStart*: float - navigationStart*: float - redirectEnd*: float - redirectStart*: float - requestStart*: float - responseEnd*: float - responseStart*: float - secureConnectionStart*: float - unloadEventEnd*: float - unloadEventStart*: float +proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.} - Performance* {.importc.} = ref object - memory*: PerformanceMemory - timing*: PerformanceTiming +when defined(nodejs): + # we provide a dummy DOM for nodejs for testing purposes + proc len*(x: Node): int = x.childNodes.len + proc `[]`*(x: Node; idx: int): Element = + assert idx >= 0 and idx < x.childNodes.len + result = cast[Element](x.childNodes[idx]) + + var document* = Document(nodeType: DocumentNode) + + proc getElem(x: Element; id: cstring): Element = + if x.id == id: return x + for i in 0..= 0 and rect.left >= 0 and + rect.bottom <= clientHeight().float and + rect.right <= clientWidth().float + +proc scrollTop*(e: Node): int {.importcpp: "#.scrollTop", nodecl.} +proc offsetHeight*(e: Node): int {.importcpp: "#.offsetHeight", nodecl.} +proc offsetTop*(e: Node): int {.importcpp: "#.offsetTop", nodecl.} +proc offsetLeft*(e: Node): int {.importcpp: "#.offsetLeft", nodecl.} diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 61c61225d9..2f8465a636 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,6 +1,38 @@ -import karax +import dom import fuzzysearch +proc textContent(e: Element): cstring {. + importcpp: "#.textContent", nodecl.} + +proc textContent(e: Node): cstring {. + importcpp: "#.textContent", nodecl.} + +proc tree(tag: string; kids: varargs[Element]): Element = + result = document.createElement tag + for k in kids: + result.appendChild k + +proc add(parent, kid: Element) = + if parent.nodeName == cstring"TR" and ( + kid.nodeName == cstring"TD" or kid.nodeName == cstring"TH"): + let k = document.createElement("TD") + appendChild(k, kid) + appendChild(parent, k) + else: + appendChild(parent, kid) + +proc setClass(e: Element; value: string) = + e.setAttribute("class", value) +proc text(s: string): Element = cast[Element](document.createTextNode(s)) +proc text(s: cstring): Element = cast[Element](document.createTextNode(s)) + +proc getElementById(id: cstring): Element {.importc: "document.getElementById", nodecl.} + +proc replaceById(id: cstring; newTree: Node) = + let x = getElementById(id) + x.parentNode.replaceChild(newTree, x) + newTree.id = id + proc findNodeWith(x: Element; tag, content: cstring): Element = if x.nodeName == tag and x.textContent == content: return x @@ -182,7 +214,7 @@ proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = t.markElement() for p in procs: if not isMarked(p): - let xx = karax.getElementsByClass(p.parent, cstring"attachedType") + let xx = getElementsByClass(p.parent, cstring"attachedType") if xx.len == 1 and xx[0].textContent == t.textContent: #kout(cstring"found ", p.nodeName) let q = tree("A", text(p.title)) @@ -230,7 +262,7 @@ proc groupBy*(value: cstring) {.exportc.} = togglevis(getElementById"toc-list") var - db: seq[Element] + db: seq[Node] contents: seq[cstring] template normalize(x: cstring): cstring = x.toLower.replace("_", "") @@ -258,7 +290,7 @@ proc dosearch(value: cstring): Element = let ul = tree("UL") result = tree("DIV") result.setClass"search_results" - var matches: seq[(Element, int)] = @[] + var matches: seq[(Node, int)] = @[] for i in 0.. int: - b[1] - a[1] + matches.sort(proc(a, b: auto): int = b[1] - a[1]) for i in 0 ..< min(matches.len, 19): matches[i][0].innerHTML = matches[i][0].getAttribute("data-doc-search-tag") - ul.add(tree("LI", matches[i][0])) + ul.add(tree("LI", cast[Element](matches[i][0]))) if ul.len == 0: result.add tree("B", text"no search results") else: diff --git a/tools/dochack/karax.nim b/tools/dochack/karax.nim deleted file mode 100644 index 020fd37d11..0000000000 --- a/tools/dochack/karax.nim +++ /dev/null @@ -1,344 +0,0 @@ -# Simple lib to write JS UIs - -import dom - -export dom.Element, dom.Event, dom.cloneNode, dom - -proc kout*[T](x: T) {.importc: "console.log", varargs.} - ## the preferred way of debugging karax applications. - -proc id*(e: Node): cstring {.importcpp: "#.id", nodecl.} -proc `id=`*(e: Node; x: cstring) {.importcpp: "#.id = #", nodecl.} -proc className*(e: Node): cstring {.importcpp: "#.className", nodecl.} -proc `className=`*(e: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} - -proc value*(e: Element): cstring {.importcpp: "#.value", nodecl.} -proc `value=`*(e: Element; v: cstring) {.importcpp: "#.value = #", nodecl.} - -proc getElementsByClass*(e: Element; name: cstring): seq[Element] {.importcpp: "#.getElementsByClassName(#)", nodecl.} - -proc toLower*(x: cstring): cstring {. - importcpp: "#.toLowerCase()", nodecl.} -proc replace*(x: cstring; search, by: cstring): cstring {. - importcpp: "#.replace(#, #)", nodecl.} - -type - EventHandler* = proc(ev: Event) - EventHandlerId* = proc(ev: Event; id: int) - - Timeout* = ref object - -var document* {.importc.}: Document - -var - dorender: proc (): Element {.closure.} - drawTimeout: Timeout - currentTree: Element - -proc setRenderer*(renderer: proc (): Element) = - dorender = renderer - -proc setTimeout*(action: proc(); ms: int): Timeout {.importc, nodecl.} -proc clearTimeout*(t: Timeout) {.importc, nodecl.} -proc targetElem*(e: Event): Element = cast[Element](e.target) - -proc getElementById*(id: cstring): Element {.importc: "document.getElementById", nodecl.} - -proc getElementsByClassName*(cls: cstring): seq[Element] {.importc: - "document.getElementsByClassName", nodecl.} - -proc textContent*(e: Element): cstring {. - importcpp: "#.textContent", nodecl.} - -proc replaceById*(id: cstring; newTree: Node) = - let x = getElementById(id) - x.parentNode.replaceChild(newTree, x) - newTree.id = id - -proc equals(a, b: Node): bool = - if a.nodeType != b.nodeType: return false - if a.id != b.id: return false - if a.nodeName != b.nodeName: return false - if a.nodeType == TextNode: - if a.data != b.data: return false - elif a.childNodes.len != b.childNodes.len: - return false - if a.className != b.className: - # style differences are updated in place and we pretend - # it's still the same node - a.className = b.className - #return false - return true - -proc diffTree(parent, a, b: Node) = - if equals(a, b): - if a.nodeType != TextNode: - # we need to do this correctly in the presence of asyncronous - # DOM updates: - var i = 0 - while i < a.childNodes.len and a.childNodes.len == b.childNodes.len: - diffTree(a, a.childNodes[i], b.childNodes[i]) - inc i - elif parent == nil: - replaceById("ROOT", b) - else: - parent.replaceChild(b, a) - -proc dodraw() = - let newtree = dorender() - newtree.id = "ROOT" - if currentTree == nil: - currentTree = newtree - replaceById("ROOT", currentTree) - else: - diffTree(nil, currentTree, newtree) - -proc redraw*() = - # we buffer redraw requests: - if drawTimeout != nil: - clearTimeout(drawTimeout) - drawTimeout = setTimeout(dodraw, 30) - -proc tree*(tag: string; kids: varargs[Element]): Element = - result = document.createElement tag - for k in kids: - result.appendChild k - -proc tree*(tag: string; attrs: openarray[(string, string)]; - kids: varargs[Element]): Element = - result = tree(tag, kids) - for a in attrs: result.setAttribute(a[0], a[1]) - -proc text*(s: string): Element = cast[Element](document.createTextNode(s)) -proc text*(s: cstring): Element = cast[Element](document.createTextNode(s)) -proc add*(parent, kid: Element) = - if parent.nodeName == cstring"TR" and ( - kid.nodeName == cstring"TD" or kid.nodeName == cstring"TH"): - let k = document.createElement("TD") - appendChild(k, kid) - appendChild(parent, k) - else: - appendChild(parent, kid) - -proc len*(x: Element): int {.importcpp: "#.childNodes.length".} -proc `[]`*(x: Element; idx: int): Element {.importcpp: "#.childNodes[#]".} - -proc isInt*(s: cstring): bool {.asmNoStackFrame.} = - asm """ - return s.match(/^[0-9]+$/); - """ - -var - linkCounter: int - -proc link*(id: int): Element = - result = document.createElement("a") - result.setAttribute("href", "#") - inc linkCounter - result.setAttribute("id", $linkCounter & ":" & $id) - -proc link*(action: EventHandler): Element = - result = document.createElement("a") - result.setAttribute("href", "#") - addEventListener(result, "click", action) - -proc parseInt*(s: cstring): int {.importc, nodecl.} -proc parseFloat*(s: cstring): float {.importc, nodecl.} -proc split*(s, sep: cstring): seq[cstring] {.importcpp, nodecl.} - -proc startsWith*(a, b: cstring): bool {.importcpp: "startsWith", nodecl.} -proc contains*(a, b: cstring): bool {.importcpp: "(#.indexOf(#)>=0)", nodecl.} -proc substr*(s: cstring; start: int): cstring {.importcpp: "substr", nodecl.} -proc substr*(s: cstring; start, length: int): cstring {.importcpp: "substr", nodecl.} - -#proc len*(s: cstring): int {.importcpp: "#.length", nodecl.} -proc `&`*(a, b: cstring): cstring {.importcpp: "(# + #)", nodecl.} -proc toCstr*(s: int): cstring {.importcpp: "((#)+'')", nodecl.} - -proc suffix*(s, prefix: cstring): cstring = - if s.startsWith(prefix): - result = s.substr(prefix.len) - else: - kout(cstring"bug! " & s & cstring" does not start with " & prefix) - -proc valueAsInt*(e: Element): int = parseInt(e.value) -proc suffixAsInt*(s, prefix: cstring): int = parseInt(suffix(s, prefix)) - -proc scrollTop*(e: Element): int {.importcpp: "#.scrollTop", nodecl.} -proc offsetHeight*(e: Element): int {.importcpp: "#.offsetHeight", nodecl.} -proc offsetTop*(e: Element): int {.importcpp: "#.offsetTop", nodecl.} - -template onImpl(s) {.dirty} = - proc wrapper(ev: Event) = - action(ev) - redraw() - addEventListener(e, s, wrapper) - -proc setOnclick*(e: Element; action: proc(ev: Event)) = - onImpl "click" - -proc setOnclick*(e: Element; action: proc(ev: Event; id: int)) = - proc wrapper(ev: Event) = - let id = ev.target.id - let a = id.split(":") - if a.len == 2: - action(ev, parseInt(a[1])) - redraw() - else: - kout(cstring("cannot deal with id "), id) - addEventListener(e, "click", wrapper) - -proc setOnfocuslost*(e: Element; action: EventHandler) = - onImpl "blur" - -proc setOnchanged*(e: Element; action: EventHandler) = - onImpl "change" - -proc setOnscroll*(e: Element; action: EventHandler) = - onImpl "scroll" - -proc select*(choices: openarray[string]): Element = - result = document.createElement("select") - var i = 0 - for c in choices: - result.add tree("option", [("value", $i)], text(c)) - inc i - -proc select*(choices: openarray[(int, string)]): Element = - result = document.createElement("select") - for c in choices: - result.add tree("option", [("value", $c[0])], text(c[1])) - -var radioCounter: int - -proc radio*(choices: openarray[(int, string)]): Element = - result = document.createElement("fieldset") - var i = 0 - inc radioCounter - for c in choices: - let id = "radio_" & c[1] & $i - var kid = tree("input", [("type", "radio"), - ("id", id), ("name", "radio" & $radioCounter), - ("value", $c[0])]) - if i == 0: - kid.setAttribute("checked", "checked") - var lab = tree("label", [("for", id)], text(c[1])) - kid.add lab - result.add kid - inc i - -proc tag*(name: string; id="", class=""): Element = - result = document.createElement(name) - if id.len > 0: - result.setAttribute("id", id) - if class.len > 0: - result.setAttribute("class", class) - -proc tdiv*(id="", class=""): Element = tag("div", id, class) -proc span*(id="", class=""): Element = tag("span", id, class) - -proc th*(s: string): Element = - result = tag("th") - result.add text(s) - -proc td*(s: string): Element = - result = tag("td") - result.add text(s) - -proc td*(s: Element): Element = - result = tag("td") - result.add s - -proc td*(class: string; s: Element): Element = - result = tag("td") - result.add s - result.setAttribute("class", class) - -proc table*(class="", kids: varargs[Element]): Element = - result = tag("table", "", class) - for k in kids: result.add k - -proc tr*(kids: varargs[Element]): Element = - result = tag("tr") - for k in kids: - if k.nodeName == cstring"TD" or k.nodeName == cstring"TH": - result.add k - else: - result.add td(k) - -proc setClass*(e: Element; value: string) = - e.setAttribute("class", value) - -proc setAttr*(e: Element; key, value: cstring) = - e.setAttribute(key, value) - -proc getAttr*(e: Element; key: cstring): cstring {. - importcpp: "#.getAttribute(#)", nodecl.} - -proc realtimeInput*(id, val: string; changed: proc(value: cstring)): Element = - let oldElem = getElementById(id) - #if oldElem != nil: return oldElem - let newVal = if oldElem.isNil: val else: $oldElem.value - var timer: Timeout - proc wrapper() = - changed(getElementById(id).value) - redraw() - proc onkeyup(ev: Event) = - if timer != nil: clearTimeout(timer) - timer = setTimeout(wrapper, 400) - result = tree("input", [("type", "text"), - ("value", newVal), - ("id", id)]) - result.addEventListener("keyup", onkeyup) - -proc ajax(meth, url: cstring; headers: openarray[(string, string)]; - data: cstring; - cont: proc (httpStatus: int; response: cstring)) = - proc setRequestHeader(a, b: cstring) {.importc: "ajax.setRequestHeader".} - {.emit: """ - var ajax = new XMLHttpRequest(); - ajax.open(`meth`,`url`,true);""".} - for a, b in items(headers): - setRequestHeader(a, b) - {.emit: """ - ajax.onreadystatechange = function(){ - if(this.readyState == 4){ - if(this.status == 200){ - `cont`(this.status, this.responseText); - } else { - `cont`(this.status, this.statusText); - } - } - } - ajax.send(`data`); - """.} - -proc ajaxPut*(url: string; headers: openarray[(string, string)]; - data: cstring; - cont: proc (httpStatus: int, response: cstring)) = - ajax("PUT", url, headers, data, cont) - -proc ajaxGet*(url: string; headers: openarray[(string, string)]; - cont: proc (httpStatus: int, response: cstring)) = - ajax("GET", url, headers, nil, cont) - -{.push stackTrace:off.} - -proc setupErrorHandler*(useAlert=false) = - ## Installs an error handler that transforms native JS unhandled - ## exceptions into Nim based stack traces. If `useAlert` is false, - ## the error message it put into the console, otherwise `alert` - ## is called. - proc stackTraceAsCstring(): cstring = cstring(getStackTrace()) - {.emit: """ - window.onerror = function(msg, url, line, col, error) { - var x = "Error: " + msg + "\n" + `stackTraceAsCstring`() - if (`useAlert`) - alert(x); - else - console.log(x); - var suppressErrorAlert = true; - return suppressErrorAlert; - };""".} - -{.pop.} From ad4e3fd28c03a8429a3b4bc0068af77018e81724 Mon Sep 17 00:00:00 2001 From: Miran Date: Sat, 2 Feb 2019 13:23:10 +0100 Subject: [PATCH 293/350] devel docs should point to devel src (#10529) Currently, devel docs (https://nim-lang.github.io/Nim/lib.html) source links point to master branch, leading to outdated source code and showing the wrong line. --- compiler/docgen.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index b70561a1d0..5af4c464e1 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -656,7 +656,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = path = path[cwd.len+1 .. ^1].replace('\\', '/') let gitUrl = getConfigVar(d.conf, "git.url") if gitUrl.len > 0: - let commit = getConfigVar(d.conf, "git.commit", "master") + let defaultBranch = if NimPatch mod 2 == 1: "devel" else: "master" + let commit = getConfigVar(d.conf, "git.commit", defaultBranch) let develBranch = getConfigVar(d.conf, "git.devel", "devel") dispA(d.conf, seeSrcRope, "$1", "", [ropeFormatNamedVars(d.conf, docItemSeeSrc, ["path", "line", "url", "commit", "devel"], [rope path.string, From 07553034de5bd9ffa89df303bf789664f9cb8ed8 Mon Sep 17 00:00:00 2001 From: genotrance Date: Sat, 2 Feb 2019 06:26:32 -0600 Subject: [PATCH 294/350] Enable Travis folding in winrelease (#10528) --- koch.nim | 16 ++++++++++------ tools/kochdocs.nim | 17 +++++++++++------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/koch.nim b/koch.nim index 89ee28d5a7..6b799449e2 100644 --- a/koch.nim +++ b/koch.nim @@ -373,9 +373,10 @@ proc winReleaseArch(arch: string) = withMingw r"..\mingw" & arch & r"\bin": # Rebuilding koch is necessary because it uses its pointer size to # determine which mingw link to put in the NSIS installer. - nimexec "c --cpu:$# koch" % cpu - kochExec "boot -d:release --cpu:$#" % cpu - kochExec "--latest zip -d:release" + inFold "winrelease koch": + nimexec "c --cpu:$# koch" % cpu + kochExecFold("winrelease boot", "boot -d:release --cpu:$#" % cpu) + kochExecFold("winrelease zip", "--latest zip -d:release") overwriteFile r"build\nim-$#.zip" % VersionAsString, r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch] @@ -384,13 +385,16 @@ proc winRelease*() = # anymore! # Build -docs file: when true: - buildDocs(gaCode) + inFold "winrelease buildDocs": + buildDocs(gaCode) withDir "web/upload/" & VersionAsString: - exec "7z a -tzip docs-$#.zip *.html" % VersionAsString + inFold "winrelease zipdocs": + exec "7z a -tzip docs-$#.zip *.html" % VersionAsString overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString, "web/upload/download/docs-$1.zip" % VersionAsString when true: - csource("-d:release") + inFold "winrelease csource": + csource("-d:release") when sizeof(pointer) == 4: winReleaseArch "32" when sizeof(pointer) == 8: diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index c0e7ce66b7..e13ad3501d 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -25,7 +25,6 @@ proc findNim*(): string = # assume there is a symlink to the exe or something: return nim - proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = let prevPath = getEnv("PATH") if additionalPath.len > 0: @@ -38,14 +37,20 @@ proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) +template inFold*(desc, body) = + if existsEnv("TRAVIS"): + echo "travis_fold:start:" & desc.replace(" ", "_") + + body + + if existsEnv("TRAVIS"): + echo "travis_fold:end:" & desc.replace(" ", "_") + proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = ## Execute shell command. Add log folding on Travis CI. # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - exec(cmd, errorcode, additionalPath) - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") + inFold(desc): + exec(cmd, errorcode, additionalPath) proc execCleanPath*(cmd: string, additionalPath = ""; errorcode: int = QuitFailure) = From 0091f2ad3bfa57aa1d5bdd1a1062975135a2a6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Sun, 3 Feb 2019 09:06:00 +0100 Subject: [PATCH 295/350] Implement {.booldefine.} (#10533) --- compiler/ast.nim | 2 +- compiler/condsyms.nim | 11 +++-------- compiler/options.nim | 2 +- compiler/pragmas.nim | 4 +++- compiler/semfold.nim | 12 +++++++++++- compiler/wordrecg.nim | 5 +++-- doc/manual.rst | 6 ++++-- lib/pure/strtabs.nim | 28 +++++++++++++++++++++++++++- tests/misc/tdefine.nim | 18 ++++++++++++++++++ 9 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 tests/misc/tdefine.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index fc470b7a89..3146722cb9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -655,7 +655,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, - mNimvm, mIntDefine, mStrDefine, mRunnableExamples, + mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, mSymIsInstantiationOf diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 9a4c1701c2..5e7ce3a081 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -15,26 +15,21 @@ import from options import Feature from lineinfos import HintsToStr, WarningsToStr -const - catNone = "false" - proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") = symbols[symbol] = value proc undefSymbol*(symbols: StringTableRef; symbol: string) = - symbols[symbol] = catNone + symbols.del(symbol) #proc lookupSymbol*(symbols: StringTableRef; symbol: string): string = # result = if isDefined(symbol): gSymbols[symbol] else: nil iterator definedSymbolNames*(symbols: StringTableRef): string = for key, val in pairs(symbols): - if val != catNone: yield key + yield key proc countDefinedSymbols*(symbols: StringTableRef): int = - result = 0 - for key, val in pairs(symbols): - if val != catNone: inc(result) + symbols.len proc initDefines*(symbols: StringTableRef) = # for bootstrapping purposes and old code: diff --git a/compiler/options.nim b/compiler/options.nim index 54276f99d9..0a25b1b969 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -360,7 +360,7 @@ proc cppDefine*(c: ConfigRef; define: string) = proc isDefined*(conf: ConfigRef; symbol: string): bool = if conf.symbols.hasKey(symbol): - result = conf.symbols[symbol] != "false" + result = true elif cmpIgnoreStyle(symbol, CPU[conf.target.targetCPU].name) == 0: result = true elif cmpIgnoreStyle(symbol, platform.OS[conf.target.targetOS].name) == 0: diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 3967fa22dc..03c9127af0 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -67,7 +67,7 @@ const wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims, - wIntDefine, wStrDefine, wUsed, wCompilerProc, wCore} + wIntDefine, wStrDefine, wBoolDefine, wUsed, wCompilerProc, wCore} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, wThread, wRaises, wLocks, wTags, wGcSafe} @@ -1106,6 +1106,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, sym.magic = mIntDefine of wStrDefine: sym.magic = mStrDefine + of wBoolDefine: + sym.magic = mBoolDefine of wUsed: noVal(c, it) if sym == nil: invalidPragma(c, it) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 0bdd0b64c4..2a29421916 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -569,10 +569,20 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode = try: result = newIntNodeT(g.config.symbols[s.name.s].parseInt, n, g) except ValueError: - localError(g.config, n.info, "expression is not an integer literal") + localError(g.config, s.info, + "{.intdefine.} const was set to an invalid integer: '" & + g.config.symbols[s.name.s] & "'") of mStrDefine: if isDefined(g.config, s.name.s): result = newStrNodeT(g.config.symbols[s.name.s], n, g) + of mBoolDefine: + if isDefined(g.config, s.name.s): + try: + result = newIntNodeT(g.config.symbols[s.name.s].parseBool.int, n, g) + except ValueError: + localError(g.config, s.info, + "{.booldefine.} const was set to an invalid bool: '" & + g.config.symbols[s.name.s] & "'") else: result = copyTree(s.ast) of skProc, skFunc, skMethod: diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 41bdc9fcbc..6f78c9d6fb 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -35,7 +35,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks, - wIntDefine, wStrDefine, + wIntDefine, wStrDefine, wBoolDefine wDestroy, @@ -122,7 +122,8 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine", + "magic", "thread", "final", "profiler", "memtracker", "objchecks", + "intdefine", "strdefine", "booldefine", "destroy", diff --git a/doc/manual.rst b/doc/manual.rst index 170f0d5505..bcb1581dd0 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7851,6 +7851,7 @@ pragma description ================= ============================================ `intdefine`:idx: Reads in a build-time define as an integer `strdefine`:idx: Reads in a build-time define as a string +`booldefine`:idx: Reads in a build-time define as a bool ================= ============================================ .. code-block:: nim @@ -7858,13 +7859,14 @@ pragma description echo FooBar :: - nim c -d:FooBar=42 foobar.c + nim c -d:FooBar=42 foobar.nim In the above example, providing the -d flag causes the symbol ``FooBar`` to be overwritten at compile time, printing out 42. If the ``-d:FooBar=42`` were to be omitted, the default value of 5 would be -used. +used. To see if a value was provided, `defined(FooBar)` can be used. +The syntax `-d:flag` is actually just a shortcut for `-d:flag=true`. Custom annotations ------------------ diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 7bafe16756..cff5293c9b 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -28,7 +28,7 @@ runnableExamples: ## When using the style insensitive mode ``modeStyleInsensitive``, -## all letters are compared case insensitively within the ASCII range +## all letters are compared case insensitively within the ASCII range ## and underscores are ignored. runnableExamples: @@ -272,6 +272,32 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) +proc del*(t: StringTableRef, key: string) = + ## Removes `key` from `t`. + # Impl adapted from `tableimpl.delImplIdx` + var i = rawGet(t, key) + let msk = high(t.data) + if i >= 0: + dec(t.counter) + block outer: + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + t.data[i].hasValue = false # mark current EMPTY + t.data[i].key = "" + t.data[i].val = "" + while true: + i = (i + 1) and msk # increment mod table size + if not t.data[i].hasValue: # end of collision cluster; So all done + break outer + r = t.myhash(t.data[i].key) and msk # "home" location of key@i + if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + break + when defined(js): + t.data[j] = t.data[i] + else: + shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop + proc `$`*(t: StringTableRef): string {.rtlFunc, extern: "nstDollar".} = ## The `$` operator for string tables. if t.len == 0: diff --git a/tests/misc/tdefine.nim b/tests/misc/tdefine.nim new file mode 100644 index 0000000000..1378b89014 --- /dev/null +++ b/tests/misc/tdefine.nim @@ -0,0 +1,18 @@ +discard """ +joinable: false +cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -r $file" +""" + +const booldef {.booldefine.} = false +const booldef2 {.booldefine.} = true +const intdef {.intdefine.} = 0 +const strdef {.strdefine.} = "" + +doAssert defined(booldef) +doAssert defined(booldef2) +doAssert defined(intdef) +doAssert defined(strdef) +doAssert booldef +doAssert not booldef2 +doAssert intdef == 2 +doAssert strdef == "foobar" From 315abd7163586778a03626c371d724cd138e7cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= Date: Sun, 3 Feb 2019 09:37:11 +0100 Subject: [PATCH 296/350] Change strtabs.nextTry to match the assumption in strtabs.del --- lib/pure/strtabs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index cff5293c9b..377178f92b 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -117,7 +117,7 @@ proc mustRehash(length, counter: int): bool = result = (length * 2 < counter * 3) or (length - counter < 4) proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash + result = (h + 1) and maxHash proc rawGet(t: StringTableRef, key: string): int = var h: Hash = myhash(t, key) and high(t.data) # start with real hash value From 8cb80238c4eedc3dbabdadd6a996bb905de8309d Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 4 Feb 2019 12:27:02 +0100 Subject: [PATCH 297/350] update documentation CSS (#10543) * update documentation CSS * update the expected htmls --- config/nimdoc.cfg | 929 +++++------------- .../expected/subdir/subdir_b/utils.html | 927 +++++------------ nimdoc/testproject/expected/testproject.html | 927 +++++------------ nimdoc/testproject/expected/theindex.html | 927 +++++------------ 4 files changed, 965 insertions(+), 2745 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index f534264a0a..8423545762 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -100,7 +100,7 @@ doc.body_toc_group = """