From 83de3a85e7144b5e222fbd89858a03dba95c9fd4 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Fri, 24 Nov 2017 22:34:29 +0000 Subject: [PATCH 01/92] Add TCP_NODELAY support #6795 --- lib/pure/net.nim | 14 +++++++++++++- lib/windows/winlean.nim | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b8d05642b6..15f2c1228e 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -145,7 +145,7 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr, OptReusePort + OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone @@ -869,6 +869,11 @@ proc close*(socket: Socket) = socket.fd.close() +when defined(posix): + from posix import TCP_NODELAY +else: + from winlean import TCP_NODELAY + proc toCInt*(opt: SOBool): cint = ## Converts a ``SOBool`` into its Socket Option cint representation. case opt @@ -880,6 +885,7 @@ proc toCInt*(opt: SOBool): cint = of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR of OptReusePort: SO_REUSEPORT + of OptNoDelay: TCP_NODELAY proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = @@ -902,6 +908,12 @@ proc getPeerAddr*(socket: Socket): (string, Port) = proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. tags: [WriteIOEffect].} = ## Sets option ``opt`` to a boolean value specified by ``value``. + ## + ## .. code-block:: Nim + ## var socket = newSocket() + ## socket.setSockOpt(OptReusePort, true) + ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) + ## var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index c3229cc7bb..7eb268a9a4 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -541,6 +541,7 @@ var SO_DONTLINGER* {.importc, header: "winsock2.h".}: cint SO_EXCLUSIVEADDRUSE* {.importc, header: "winsock2.h".}: cint # disallow local address reuse SO_ERROR* {.importc, header: "winsock2.h".}: cint + TCP_NODELAY* {.importc, header: "winsock2.h".}: cint proc `==`*(x, y: SocketHandle): bool {.borrow.} From 908677a313a04fdda683d09791be276fa110e715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Sat, 25 Nov 2017 16:08:27 +0100 Subject: [PATCH 02/92] remove goFree() (#6808) __go_free() was removed from gcc-7.2.0 so we stop trying to help the garbage collector by marking no longer used memory regions --- lib/system/mmdisp.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 8249349669..9af36c7b8c 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -343,7 +343,6 @@ elif defined(gogc): const goFlagNoZero: uint32 = 1 shl 3 proc goRuntimeMallocGC(size: uint, typ: uint, flag: uint32): pointer {.importc: "runtime_mallocgc", dynlib: goLib.} - proc goFree(v: pointer) {.importc: "__go_free", dynlib: goLib.} proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.} @@ -376,7 +375,6 @@ elif defined(gogc): result = goRuntimeMallocGC(roundup(newsize, sizeof(pointer)).uint, 0.uint, goFlagNoZero) copyMem(result, old, oldsize) zeroMem(cast[pointer](cast[ByteAddress](result) +% oldsize), newsize - oldsize) - goFree(old) proc nimGCref(p: pointer) {.compilerproc, inline.} = discard proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard From 27ea1750e53c7d0280a6cb447e9cd057d219190e Mon Sep 17 00:00:00 2001 From: Veladus Date: Sat, 25 Nov 2017 16:55:10 +0100 Subject: [PATCH 03/92] Moved encodeUrl and decodeUrl from cgi to url --- lib/pure/cgi.nim | 44 ++------------------------------------------ lib/pure/uri.nim | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index fcf2cf99f5..5de6aa4870 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -29,21 +29,8 @@ ## writeLine(stdout, "your password: " & myData["password"]) ## writeLine(stdout, "") -import strutils, os, strtabs, cookies - -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. - result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') - else: - add(result, '%') - add(result, toHex(ord(s[i]), 2)) +import strutils, os, strtabs, cookies, uri +export uri.encodeUrl, uri.decodeUrl proc handleHexChar(c: char, x: var int) {.inline.} = case c @@ -52,28 +39,6 @@ proc handleHexChar(c: char, x: var int) {.inline.} = of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) else: assert(false) -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and - ## and every other character is carried over. - result = newString(s.len) - var i = 0 - var j = 0 - while i < s.len: - case s[i] - of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) - of '+': result[j] = ' ' - else: result[j] = s[i] - inc(i) - inc(j) - setLen(result, j) - proc addXmlChar(dest: var string, c: char) {.inline.} = case c of '&': add(dest, "&") @@ -390,8 +355,3 @@ proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) result = hasKey(gcookies, name) - -when isMainModule: - const test1 = "abc\L+def xyz" - assert encodeUrl(test1) == "abc%0A%2Bdef+xyz" - assert decodeUrl(encodeUrl(test1)) == test1 diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 164a57ecfa..a651530c3d 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -47,6 +47,49 @@ proc add*(url: var Url, a: Url) {.deprecated.} = url = url / a {.pop.} +proc encodeUrl*(s: string): string = + ## Encodes a value to be HTTP safe: This means that characters in the set + ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, + ## a space is converted to ``'+'`` and every other character is encoded as + ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. + result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars + for i in 0..s.len-1: + case s[i] + of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) + of ' ': add(result, '+') + else: + add(result, '%') + add(result, toHex(ord(s[i]), 2)) + +proc decodeUrl*(s: string): string = + ## Decodes a value from its HTTP representation: This means that a ``'+'`` + ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## value) is converted to the character with ordinal number ``xx``, and + ## and every other character is carried over. + proc handleHexChar(c: char, x: var int) {.inline.} = + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: assert(false) + + result = newString(s.len) + var i = 0 + var j = 0 + while i < s.len: + case s[i] + of '%': + var x = 0 + handleHexChar(s[i+1], x) + handleHexChar(s[i+2], x) + inc(i, 2) + result[j] = chr(x) + of '+': result[j] = ' ' + else: result[j] = s[i] + inc(i) + inc(j) + setLen(result, j) + proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false @@ -327,6 +370,11 @@ proc `$`*(u: Uri): string = result.add(u.anchor) when isMainModule: + block: + const test1 = "abc\L+def xyz" + doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" + doAssert decodeUrl(encodeUrl(test1)) == test1 + block: let str = "http://localhost" let test = parseUri(str) From c1782fac2195bf9e82ee38d1e52ae981e2f78229 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 26 Nov 2017 01:07:01 +0100 Subject: [PATCH 04/92] cleaned up tutorial 1 --- doc/tut1.rst | 63 +++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 6731efde97..df42c858f8 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1388,27 +1388,31 @@ slice's bounds can hold any value supported by their type, but it is the proc using the slice object which defines what values are accepted. - To understand some of the different ways of specifying the indices of strings, arrays, sequences, etc., - it must be remembered that Nim uses zero-based indices. +To understand some of the different ways of specifying the indices of +strings, arrays, sequences, etc., it must be remembered that Nim uses +zero-based indices. - So the string ``b`` is of length 19, and two different ways of specifying the indices are +So the string ``b`` is of length 19, and two different ways of specifying the +indices are - .. code-block:: nim +.. code-block:: nim "Slices are useless." | | | 0 11 17 using indices ^19 ^8 ^2 using ^ syntax - where ``b[0..^1]`` is equivalent to ``b[0..b.len-1]`` and ``b[0..`_. Distinct type ------------- -A Distinct type allows for the creation of new type that "does not imply a subtype relationship between it and its base type". +A Distinct type allows for the creation of new type that "does not imply a +subtype relationship between it and its base type". You must **explicitly** define all behaviour for the distinct type. -To help with this, both the distinct type and its base type can cast from one type to the other. +To help with this, both the distinct type and its base type can cast from one +type to the other. Examples are provided in the `manual `_. Modules @@ -1592,39 +1598,6 @@ Each module has a special magic constant ``isMainModule`` that is true if the module is compiled as the main file. This is very useful to embed tests within the module as shown by the above example. -Modules that depend on each other are possible, but strongly discouraged, -because then one module cannot be reused without the other. - -The algorithm for compiling modules is: - -- Compile the whole module as usual, following import statements recursively. -- If there is a cycle only import the already parsed symbols (that are - exported); if an unknown identifier occurs then abort. - -This is best illustrated by an example: - -.. code-block:: nim - # Module A - type - T1* = int # Module A exports the type ``T1`` - import B # the compiler starts parsing B - - proc main() = - var i = p(3) # works because B has been parsed completely here - - main() - -.. code-block:: nim - # Module B - import A # A is not parsed here! Only the already known symbols - # of A are imported. - - proc p*(x: A.T1): A.T1 = - # this works because the compiler has already - # added T1 to A's interface symbol table - result = x + 1 - - A symbol of a module *can* be *qualified* with the ``module.symbol`` syntax. And if a symbol is ambiguous, it *must* be qualified. A symbol is ambiguous if it is defined in two (or more) different modules and both modules are From 8d1a5dc8e7b10d5980dc1ce06dce0739caaa7d06 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 26 Nov 2017 02:51:11 +0100 Subject: [PATCH 05/92] the documentation generator now supports system.runnableExamples --- changelog.md | 4 + compiler/ast.nim | 2 +- compiler/condsyms.nim | 1 + compiler/docgen.nim | 139 ++++++++++++++++++------------- compiler/sem.nim | 13 +++ compiler/semdata.nim | 1 + compiler/semexprs.nim | 11 +++ lib/packages/docutils/rstgen.nim | 2 +- lib/system.nim | 17 ++++ 9 files changed, 132 insertions(+), 58 deletions(-) diff --git a/changelog.md b/changelog.md index 49cd4123ab..14352374c9 100644 --- a/changelog.md +++ b/changelog.md @@ -94,3 +94,7 @@ This now needs to be written as: - [``poly``](https://github.com/lcrees/polynumeric) - [``pdcurses``](https://github.com/lcrees/pdcurses) - [``romans``](https://github.com/lcrees/romans) + +- Added ``system.runnableExamples`` to make examples in Nim's documentation easier + to write and test. The examples are tested as the last step of + ``nim doc``. diff --git a/compiler/ast.nim b/compiler/ast.nim index 787cb49977..5bf4184c95 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -639,7 +639,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, - mNimvm, mIntDefine, mStrDefine + mNimvm, mIntDefine, mStrDefine, mRunnableExamples # things that we can evaluate safely at compile time, even if not asked for it: const diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 2050a746b4..4879ce5c34 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -110,3 +110,4 @@ proc initDefines*() = when false: defineSymbol("nimHasOpt") defineSymbol("nimNoArrayToCstringConversion") defineSymbol("nimNewRoof") + defineSymbol("nimHasRunnableExamples") diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8978052e2e..4a3674812e 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -204,10 +204,85 @@ proc getPlainDocstring(n: PNode): string = if n.comment != nil and startsWith(n.comment, "##"): result = n.comment if result.len < 1: - if n.kind notin {nkEmpty..nkNilLit}: - for i in countup(0, len(n)-1): - result = getPlainDocstring(n.sons[i]) - if result.len > 0: return + for i in countup(0, safeLen(n)-1): + result = getPlainDocstring(n.sons[i]) + if result.len > 0: return + +proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) = + var r: TSrcGen + var literal = "" + initTokRender(r, n, renderFlags) + var kind = tkEof + while true: + getNextTok(r, kind, literal) + case kind + of tkEof: + break + of tkComment: + dispA(result, "$1", "\\spanComment{$1}", + [rope(esc(d.target, literal))]) + of tokKeywordLow..tokKeywordHigh: + dispA(result, "$1", "\\spanKeyword{$1}", + [rope(literal)]) + of tkOpr: + dispA(result, "$1", "\\spanOperator{$1}", + [rope(esc(d.target, literal))]) + of tkStrLit..tkTripleStrLit: + dispA(result, "$1", + "\\spanStringLit{$1}", [rope(esc(d.target, literal))]) + of tkCharLit: + dispA(result, "$1", "\\spanCharLit{$1}", + [rope(esc(d.target, literal))]) + of tkIntLit..tkUInt64Lit: + dispA(result, "$1", + "\\spanDecNumber{$1}", [rope(esc(d.target, literal))]) + of tkFloatLit..tkFloat128Lit: + dispA(result, "$1", + "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))]) + of tkSymbol: + dispA(result, "$1", + "\\spanIdentifier{$1}", [rope(esc(d.target, literal))]) + of tkSpaces, tkInvalid: + add(result, literal) + of tkCurlyDotLe: + dispA(result, """$1
""", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + of tkCurlyDotRi: + dispA(result, "
$1", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, + tkBracketDotLe, tkBracketDotRi, tkParDotLe, + tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, + tkAccent, tkColonColon, + tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: + dispA(result, "$1", "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + +proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) = + case n.kind + of nkCallKinds: + if n[0].kind == nkSym and n[0].sym.magic == mRunnableExamples and + n.len >= 2 and n.lastSon.kind == nkStmtList: + dispA(dest, "\n$1\n", + "\n\\textbf{$1}\n", [rope"Examples:"]) + inc d.listingCounter + let id = $d.listingCounter + dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"]) + # this is a rather hacky way to get rid of the initial indentation + # that the renderer currently produces: + var i = 0 + var body = n.lastSon + if body.len == 1 and body.kind == nkStmtList: body = body.lastSon + for b in body: + if i > 0: dest.add "\n" + inc i + nodeToHighlightedHtml(d, b, dest, {}) + dest.add(d.config.getOrDefault"doc.listing_end" % id) + else: discard + for i in 0 ..< n.safeLen: + getAllRunnableExamples(d, n[i], dest) when false: proc findDocComment(n: PNode): PNode = @@ -379,11 +454,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = let name = getName(d, nameNode) nameRope = name.rope - plainDocstring = getPlainDocstring(n) # call here before genRecComment! + var plainDocstring = getPlainDocstring(n) # call here before genRecComment! var result: Rope = nil var literal, plainName = "" var kind = tkEof var comm = genRecComment(d, n) # call this here for the side-effect! + getAllRunnableExamples(d, n, comm) var r: TSrcGen # Obtain the plain rendered string for hyperlink titles. initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, @@ -395,53 +471,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = plainName.add(literal) # Render the HTML hyperlink. - initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - while true: - getNextTok(r, kind, literal) - case kind - of tkEof: - break - of tkComment: - dispA(result, "$1", "\\spanComment{$1}", - [rope(esc(d.target, literal))]) - of tokKeywordLow..tokKeywordHigh: - dispA(result, "$1", "\\spanKeyword{$1}", - [rope(literal)]) - of tkOpr: - dispA(result, "$1", "\\spanOperator{$1}", - [rope(esc(d.target, literal))]) - of tkStrLit..tkTripleStrLit: - dispA(result, "$1", - "\\spanStringLit{$1}", [rope(esc(d.target, literal))]) - of tkCharLit: - dispA(result, "$1", "\\spanCharLit{$1}", - [rope(esc(d.target, literal))]) - of tkIntLit..tkUInt64Lit: - dispA(result, "$1", - "\\spanDecNumber{$1}", [rope(esc(d.target, literal))]) - of tkFloatLit..tkFloat128Lit: - dispA(result, "$1", - "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))]) - of tkSymbol: - dispA(result, "$1", - "\\spanIdentifier{$1}", [rope(esc(d.target, literal))]) - of tkSpaces, tkInvalid: - add(result, literal) - of tkCurlyDotLe: - dispA(result, """$1
""", - "\\spanOther{$1}", - [rope(esc(d.target, literal))]) - of tkCurlyDotRi: - dispA(result, "
$1", - "\\spanOther{$1}", - [rope(esc(d.target, literal))]) - of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, - tkBracketDotLe, tkBracketDotRi, tkParDotLe, - tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, - tkAccent, tkColonColon, - tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: - dispA(result, "$1", "\\spanOther{$1}", - [rope(esc(d.target, literal))]) + nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments}) inc(d.id) let @@ -609,10 +639,7 @@ proc generateJson*(d: PDoc, n: PNode) = else: discard proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string = - var - name = getName(d, nameNode) - - result = name & "\n" + result = getName(d, nameNode) & "\n" proc generateTags*(d: PDoc, n: PNode, r: var Rope) = case n.kind diff --git a/compiler/sem.nim b/compiler/sem.nim index 3608bc11c0..495321de45 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -570,6 +570,18 @@ proc myProcess(context: PPassContext, n: PNode): PNode = result = ast.emptyNode #if gCmd == cmdIdeTools: findSuggest(c, n) +proc testExamples(c: PContext) = + let inp = toFullPath(c.module.info) + let outp = inp.changeFileExt"" & "_examples.nim" + renderModule(c.runnableExamples, inp, outp) + let backend = if isDefined("js"): "js" + elif isDefined("cpp"): "cpp" + elif isDefined("objc"): "objc" + else: "c" + if os.execShellCmd("nim " & backend & " -r " & outp) != 0: + quit "[Examples] failed" + removeFile(outp) + proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = var c = PContext(context) if gCmd == cmdIdeTools and not c.suggestionsMade: @@ -584,5 +596,6 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = result.add(c.module.ast) popOwner(c) popProcCon(c) + if c.runnableExamples != nil: testExamples(c) const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 5057260a4b..8affee649f 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -136,6 +136,7 @@ type # the generic type has been constructed completely. See # tests/destructor/topttree.nim for an example that # would otherwise fail. + runnableExamples*: PNode proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair = result.genericSym = s diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index d600b1c486..380b367bc5 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1847,6 +1847,17 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = analyseIfAddressTakenInCall(c, result) if callee.magic != mNone: result = magicsAfterOverloadResolution(c, result, flags) + of mRunnableExamples: + if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList: + if sfMainModule in c.module.flags: + let inp = toFullPath(c.module.info) + if c.runnableExamples == nil: + c.runnableExamples = newTree(nkStmtList, + newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) + c.runnableExamples.add newTree(nkBlockStmt, emptyNode, n.lastSon) + result = n + else: + result = emptyNode else: result = semDirectOp(c, n, flags) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 1272affdc8..f156c440bb 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -46,7 +46,7 @@ type target*: OutputTarget config*: StringTableRef splitAfter*: int # split too long entries in the TOC - listingCounter: int + listingCounter*: int tocPart*: seq[TocEntry] hasToc*: bool theIndex: string # Contents of the index file to be dumped at the end. diff --git a/lib/system.nim b/lib/system.nim index 1b53bf9f57..323ff00e6d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3992,3 +3992,20 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage): proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage + + +when defined(nimHasRunnableExamples): + proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a ``runnableExamples`` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the ``##`` doc comment. As the last step of documentation + ## generation the examples are put into an ``$file_example.nim`` file, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. +else: + template runnableExamples*(body: untyped) = + discard From a720539f5e5b3abe504b31fbf5a0fc85ebac0b0d Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 26 Nov 2017 03:24:59 +0100 Subject: [PATCH 06/92] fixes system.runnableExamples; strutils makes use of runnableExamples --- compiler/semexprs.nim | 4 +- lib/pure/strutils.nim | 141 ++++++++++++++++++++---------------------- 2 files changed, 68 insertions(+), 77 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 380b367bc5..4942ef3856 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1854,8 +1854,8 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = if c.runnableExamples == nil: c.runnableExamples = newTree(nkStmtList, newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) - c.runnableExamples.add newTree(nkBlockStmt, emptyNode, n.lastSon) - result = n + c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon) + result = setMs(n, s) else: result = emptyNode else: diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index d773cc7d8f..4ac40d8b4e 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1305,14 +1305,13 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than ## `startLen`. The following example creates a string describing - ## an array of integers: - ## - ## .. code-block:: nim - ## var arr = "[" - ## for x in items([2, 3, 5, 7, 11]): - ## addSep(arr, startLen=len("[")) - ## add(arr, $x) - ## add(arr, "]") + ## an array of integers. + runnableExamples: + var arr = "[" + for x in items([2, 3, 5, 7, 11]): + addSep(arr, startLen=len("[")) + add(arr, $x) + add(arr, "]") if dest.len > startLen: add(dest, sep) proc allCharsInSet*(s: string, theSet: set[char]): bool = @@ -1730,7 +1729,9 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, ## ## Even though the algorithm works with any string `s`, it is only useful ## if `s` contains a number. - ## Example: ``insertSep("1000000") == "1_000_000"`` + runnableExamples: + doAssert insertSep("1000000") == "1_000_000" + var L = (s.len-1) div digits + s.len result = newString(L) var j = 0 @@ -1818,6 +1819,8 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## ## A valid identifier starts with a character of the set `IdentStartChars` ## and is followed by any number of characters of the set `IdentChars`. + runnableExamples: + doAssert "abc_def08".validIdentifier if s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false @@ -1828,7 +1831,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, ## Returns the edit distance between `a` and `b`. ## ## This uses the `Levenshtein`:idx: distance algorithm with only a linear - ## memory overhead. This implementation is highly optimized! + ## memory overhead. var len1 = a.len var len2 = b.len if len1 > len2: @@ -2007,16 +2010,11 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``float`` type. ## ## If ``precision == -1``, it tries to format it nicely. - ## - ## Examples: - ## - ## .. code-block:: nim - ## - ## let x = 123.456 - ## doAssert x.formatFloat() == "123.4560000000000" - ## doAssert x.formatFloat(ffDecimal, 4) == "123.4560" - ## doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" - ## + runnableExamples: + let x = 123.456 + doAssert x.formatFloat() == "123.4560000000000" + doAssert x.formatFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" result = formatBiggestFloat(f, format, precision, decimalSep) proc trimZeros*(x: var string) {.noSideEffect.} = @@ -2051,18 +2049,13 @@ proc formatSize*(bytes: int64, ## ## `includeSpace` can be set to true to include the (SI preferred) space ## between the number and the unit (e.g. 1 KiB). - ## - ## Examples: - ## - ## .. code-block:: nim - ## - ## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - ## formatSize((2.234*1024*1024).int) == "2.234MiB" - ## formatSize(4096, includeSpace=true) == "4 KiB" - ## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" - ## formatSize(4096) == "4KiB" - ## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" - ## + runnableExamples: + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] var @@ -2156,7 +2149,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" ## - ## `decimalSep` is used as the decimal separator + ## `decimalSep` is used as the decimal separator. var absolute: BiggestFloat significand: BiggestFloat @@ -2369,17 +2362,16 @@ proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. rtl, extern: "nsuRemoveSuffixCharSet".} = ## Removes all characters from `chars` from the end of the string `s` ## (in-place). - ## - ## .. code-block:: nim - ## var userInput = "Hello World!*~\r\n" - ## userInput.removeSuffix - ## doAssert userInput == "Hello World!*~" - ## userInput.removeSuffix({'~', '*'}) - ## doAssert userInput == "Hello World!" - ## - ## var otherInput = "Hello!?!" - ## otherInput.removeSuffix({'!', '?'}) - ## doAssert otherInput == "Hello" + runnableExamples: + var userInput = "Hello World!*~\r\n" + userInput.removeSuffix + doAssert userInput == "Hello World!*~" + userInput.removeSuffix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "Hello!?!" + otherInput.removeSuffix({'!', '?'}) + doAssert otherInput == "Hello" if s.len == 0: return var last = s.high while last > -1 and s[last] in chars: last -= 1 @@ -2390,24 +2382,23 @@ proc removeSuffix*(s: var string, c: char) {. ## Removes all occurrences of a single character (in-place) from the end ## of a string. ## - ## .. code-block:: nim - ## var table = "users" - ## table.removeSuffix('s') - ## doAssert table == "user" - ## - ## var dots = "Trailing dots......." - ## dots.removeSuffix('.') - ## doAssert dots == "Trailing dots" + runnableExamples: + var table = "users" + table.removeSuffix('s') + doAssert table == "user" + + var dots = "Trailing dots......." + dots.removeSuffix('.') + doAssert dots == "Trailing dots" removeSuffix(s, chars = {c}) proc removeSuffix*(s: var string, suffix: string) {. rtl, extern: "nsuRemoveSuffixString".} = ## Remove the first matching suffix (in-place) from a string. - ## - ## .. code-block:: nim - ## var answers = "yeses" - ## answers.removeSuffix("es") - ## doAssert answers == "yes" + runnableExamples: + var answers = "yeses" + answers.removeSuffix("es") + doAssert answers == "yes" var newLen = s.len if s.endsWith(suffix): newLen -= len(suffix) @@ -2418,16 +2409,16 @@ proc removePrefix*(s: var string, chars: set[char] = Newlines) {. ## Removes all characters from `chars` from the start of the string `s` ## (in-place). ## - ## .. code-block:: nim - ## var userInput = "\r\n*~Hello World!" - ## userInput.removePrefix - ## doAssert userInput == "*~Hello World!" - ## userInput.removePrefix({'~', '*'}) - ## doAssert userInput == "Hello World!" - ## - ## var otherInput = "?!?Hello!?!" - ## otherInput.removePrefix({'!', '?'}) - ## doAssert otherInput == "Hello!?!" + runnableExamples: + var userInput = "\r\n*~Hello World!" + userInput.removePrefix + doAssert userInput == "*~Hello World!" + userInput.removePrefix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "?!?Hello!?!" + otherInput.removePrefix({'!', '?'}) + doAssert otherInput == "Hello!?!" var start = 0 while start < s.len and s[start] in chars: start += 1 if start > 0: s.delete(0, start - 1) @@ -2437,20 +2428,20 @@ proc removePrefix*(s: var string, c: char) {. ## Removes all occurrences of a single character (in-place) from the start ## of a string. ## - ## .. code-block:: nim - ## var ident = "pControl" - ## ident.removePrefix('p') - ## doAssert ident == "Control" + runnableExamples: + var ident = "pControl" + ident.removePrefix('p') + doAssert ident == "Control" removePrefix(s, chars = {c}) proc removePrefix*(s: var string, prefix: string) {. rtl, extern: "nsuRemovePrefixString".} = ## Remove the first matching prefix (in-place) from a string. ## - ## .. code-block:: nim - ## var answers = "yesyes" - ## answers.removePrefix("yes") - ## doAssert answers == "yes" + runnableExamples: + var answers = "yesyes" + answers.removePrefix("yes") + doAssert answers == "yes" if s.startsWith(prefix): s.delete(0, prefix.len - 1) From a372363190488d7267e545ce9ec6c81ccb79ed8c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 16:40:24 +0000 Subject: [PATCH 07/92] Don't catch-all in asynchttpserver. It hides bugs. --- lib/pure/asynchttpserver.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 433931c9de..ba16156515 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -275,10 +275,7 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string lineFut.mget() = newStringOfCap(80) while not client.isClosed: - try: - await processRequest(server, request, client, address, lineFut, callback) - except: - asyncCheck request.mget().respondError(Http500) + await processRequest(server, request, client, address, lineFut, callback) proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure,gcsafe.}, From 4d931c62763fd68e3f6e874f977402f3ae5d144b Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 22:52:39 +0000 Subject: [PATCH 08/92] Add an attempted reproduction for #5531. --- tests/async/tasyncfile.nim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/async/tasyncfile.nim b/tests/async/tasyncfile.nim index 592f0ebd88..6c0725c889 100644 --- a/tests/async/tasyncfile.nim +++ b/tests/async/tasyncfile.nim @@ -34,4 +34,19 @@ proc main() {.async.} = doAssert data == "foot\ntest2" file.close() + # Issue #5531 + block: + removeFile(fn) + var file = openAsync(fn, fmWrite) + await file.write("test2") + file.close() + file = openAsync(fn, fmWrite) + await file.write("test3") + file.close() + file = openAsync(fn, fmRead) + let data = await file.readAll() + doAssert data == "test3" + file.close() + + waitFor main() From 821a6ef4c26bfaca02f778b3a8634a27ace6dc2f Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 27 Nov 2017 10:41:55 +0000 Subject: [PATCH 09/92] Add links to documentation (#6780) Related to #4219 --- config/nimdoc.cfg | 72 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 3e656cb8f3..0c3f204a57 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -88,9 +88,27 @@ doc.body_toc = """ """ +@if boot: +# This is enabled with the "boot" directive to generate +# the compiler documentation. +# As a user, tweak the block below instead. +# You can add your own global-links entries doc.body_toc_group = """
+
Search: @@ -112,6 +130,37 @@ doc.body_toc_group = """
""" +else: + +doc.body_toc_group = """ +
+
+ +
+ Search: +
+
+ Group by: + +
+ $tableofcontents +
+
+
+

$moduledesc

+ $content +
+
+""" +@end + doc.body_no_toc = """ $moduledesc $content @@ -1268,15 +1317,15 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator background-repeat: no-repeat; background-image: url("data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="); margin-bottom: -5px; } - div.pragma { - display: none; - } - span.pragmabegin { - cursor: pointer; - } - span.pragmaend { - cursor: pointer; - } +div.pragma { + display: none; +} +span.pragmabegin { + cursor: pointer; +} +span.pragmaend { + cursor: pointer; +} div.search_results { background-color: antiquewhite; @@ -1284,6 +1333,11 @@ div.search_results { padding: 1em; border: 1px solid #4d4d4d; } + +div#global-links ul { + margin-left: 0; + list-style-type: none; +} From 5e93eb9d7a839984c99789a4b9409b7a70f691f7 Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 27 Nov 2017 11:39:02 +0000 Subject: [PATCH 10/92] Add visual studio C4297 warning to ignore list (#6815) --- lib/nimbase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index 76192713b9..ac2cc097c1 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -70,7 +70,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090) +# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297) #endif /* ------------------------------------------------------------------------- */ From 653dcb80277ed0189473991b9d3f429fe709d846 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:10:57 +0100 Subject: [PATCH 11/92] cleaned up strutils.nim --- lib/pure/strutils.nim | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 4ac40d8b4e..6fe2df2168 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -32,10 +32,6 @@ when defined(nimOldSplit): else: {.pragma: deprecatedSplit.} -type - CharSet* {.deprecated.} = set[char] # for compatibility with Nim -{.deprecated: [TCharSet: CharSet].} - const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} ## All the characters that count as whitespace. @@ -78,40 +74,40 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, return c in Letters proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericChar".}= + rtl, extern: "nsuIsAlphaNumericChar".} = ## Checks whether or not `c` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. - return c in Letters or c in Digits + return c in Letters+Digits proc isDigit*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitChar".}= + rtl, extern: "nsuIsDigitChar".} = ## Checks whether or not `c` is a number. ## ## This checks 0-9 ASCII characters only. return c in Digits proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiChar".}= + rtl, extern: "nsuIsSpaceAsciiChar".} = ## Checks whether or not `c` is a whitespace character. return c in Whitespace proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiChar".}= + rtl, extern: "nsuIsLowerAsciiChar".} = ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiChar".}= + rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaAsciiStr".}= + rtl, extern: "nsuIsAlphaAsciiStr".} = ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -123,10 +119,10 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlphaAscii() and result + if not c.isAlphaAscii(): return false proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericStr".}= + rtl, extern: "nsuIsAlphaNumericStr".} = ## Checks whether or not `s` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. @@ -142,7 +138,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, return false proc isDigit*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitStr".}= + rtl, extern: "nsuIsDigitStr".} = ## Checks whether or not `s` is a numeric value. ## ## This checks 0-9 ASCII characters only. @@ -158,7 +154,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, return false proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiStr".}= + rtl, extern: "nsuIsSpaceAsciiStr".} = ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -172,7 +168,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, return false proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiStr".}= + rtl, extern: "nsuIsLowerAsciiStr".} = ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -187,7 +183,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, true proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiStr".}= + rtl, extern: "nsuIsUpperAsciiStr".} = ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. From 69122c9f2d1190c7aff7eacb435ab5b04e85df29 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:11:54 +0100 Subject: [PATCH 12/92] todo.txt update --- todo.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/todo.txt b/todo.txt index a30c23ce3c..bc68c21686 100644 --- a/todo.txt +++ b/todo.txt @@ -1,6 +1,9 @@ version 1.0 battle plan ======================= +- introduce ``nkStmtListExpr`` for template/macro invokations to produce + better stack traces +- let 'doAssert' analyse the expressions and produce more helpful output - fix "high priority" bugs - try to fix as many compiler crashes as reasonable From 21ffb3a7069e1abdbc86826d116ea5c1883c534f Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:13:25 +0100 Subject: [PATCH 13/92] tut1.rst makes use of the new ':test:' feature --- doc/tut1.rst | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index df42c858f8..9e6f1ab3c2 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -30,6 +30,7 @@ The first program We start the tour with a modified "hello world" program: .. code-block:: Nim + :test: "nim c $1" # This is a comment echo "What's your name? " var name: string = readLine(stdin) @@ -72,6 +73,7 @@ you can leave out the type in the declaration (this is called `local type inference`:idx:). So this will work too: .. code-block:: Nim + :test: "nim c $1" var name = readLine(stdin) Note that this is basically the only form of type inference that exists in @@ -116,6 +118,7 @@ Comments start anywhere outside a string or character literal with the hash character ``#``. Documentation comments start with ``##``: .. code-block:: nim + :test: "nim c $1" # A comment. var myVariable: int ## a documentation comment @@ -129,6 +132,7 @@ Multiline comments are started with ``#[`` and terminated with ``]#``. Multilin comments can also be nested. .. code-block:: nim + :test: "nim c $1" #[ You can have any Nim code text commented out inside this with no indentation restrictions. @@ -142,6 +146,7 @@ You can also use the `discard statement <#procedures-discard-statement>`_ togeth literals* to create block comments: .. code-block:: nim + :test: "nim c $1" discard """ You can have any Nim code text commented out inside this with no indentation restrictions. yes("May I ask a pointless question?") """ @@ -169,6 +174,7 @@ Indentation can be used after the ``var`` keyword to list a whole section of variables: .. code-block:: + :test: "nim c $1" var x, y: int # a comment can occur here too @@ -186,10 +192,11 @@ to a storage location: x = "xyz" # assigns a new value to `x` ``=`` is the *assignment operator*. The assignment operator can be -overloaded. You can declare multiple variables with a single assignment +overloaded. You can declare multiple variables with a single assignment statement and all the variables will have the same value: .. code-block:: + :test: "nim c $1" var x, y = 3 # assigns 3 to the variables `x` and `y` echo "x ", x # outputs "x 3" echo "y ", y # outputs "y 3" @@ -212,12 +219,14 @@ cannot change. The compiler must be able to evaluate the expression in a constant declaration at compile time: .. code-block:: nim + :test: "nim c $1" const x = "abc" # the constant x contains the string "abc" Indentation can be used after the ``const`` keyword to list a whole section of constants: .. code-block:: + :test: "nim c $1" const x = 1 # a comment can occur here too @@ -243,6 +252,7 @@ and put it into a data section": const input = readLine(stdin) # Error: constant expression expected .. code-block:: + :test: "nim c $1" let input = readLine(stdin) # works @@ -260,6 +270,7 @@ If statement The if statement is one way to branch the control flow: .. code-block:: nim + :test: "nim c $1" let name = readLine(stdin) if name == "": echo "Poor soul, you lost your name?" @@ -281,6 +292,7 @@ Another way to branch is provided by the case statement. A case statement is a multi-branch: .. code-block:: nim + :test: "nim c $1" let name = readLine(stdin) case name of "": @@ -338,6 +350,7 @@ While statement The while statement is a simple looping construct: .. code-block:: nim + :test: "nim c $1" echo "What's your name? " var name = readLine(stdin) @@ -358,6 +371,7 @@ provides. The example uses the built-in `countup `_ iterator: .. code-block:: nim + :test: "nim c $1" echo "Counting to ten: " for i in countup(1, 10): echo i @@ -409,6 +423,7 @@ Other useful iterators for collections (like arrays and sequences) are * ``pairs`` and ``mpairs`` which provides the element and an index number (immutable and mutable respectively) .. code-block:: nim + :test: "nim c $1" for index, item in ["a","b"].pairs: echo item, " at index ", index # => a at index 0 @@ -421,6 +436,8 @@ new scope. This means that in the following example, ``x`` is not accessible outside the loop: .. code-block:: nim + :test: "nim c $1" + :status: 1 while false: var x = "hi" echo x # does not work @@ -430,6 +447,8 @@ are only visible within the block they have been declared. The ``block`` statement can be used to open a new block explicitly: .. code-block:: nim + :test: "nim c $1" + :status: 1 block myblock: var x = "hi" echo x # does not work either @@ -444,6 +463,7 @@ can leave a ``while``, ``for``, or a ``block`` statement. It leaves the innermost construct, unless a label of a block is given: .. code-block:: nim + :test: "nim c $1" block myblock: echo "entering block" while true: @@ -465,6 +485,7 @@ Like in many other programming languages, a ``continue`` statement starts the next iteration immediately: .. code-block:: nim + :test: "nim c $1" while true: let x = readLine(stdin) if x == "": continue @@ -477,6 +498,7 @@ When statement Example: .. code-block:: nim + :test: "nim c $1" when system.hostOS == "windows": echo "running on Windows!" @@ -549,6 +571,7 @@ an expression is allowed: .. code-block:: nim # computes fac(4) at compile time: + :test: "nim c $1" const fac4 = (var x = 1; for i in 1..4: x *= i; x) @@ -561,6 +584,7 @@ is needed. (Some languages call them *methods* or *functions*.) In Nim new procedures are defined with the ``proc`` keyword: .. code-block:: nim + :test: "nim c $1" proc yes(question: string): bool = echo question, " (y/n)" while true: @@ -597,6 +621,7 @@ automatically at the end of a procedure if there is no ``return`` statement at the exit. .. code-block:: nim + :test: "nim c $1" proc sumTillNegative(x: varargs[int]): int = for i in x: if i < 0: @@ -624,6 +649,7 @@ to be declared with ``var`` in the procedure body. Shadowing the parameter name is possible, and actually an idiom: .. code-block:: nim + :test: "nim c $1" proc printSeq(s: seq, nprinted: int = -1) = var nprinted = if nprinted == -1: s.len else: min(nprinted, s.len) for i in 0 .. `_ returns the lowest valid index for the array `a` and `high(a) `_ the highest valid index. .. code-block:: nim + :test: "nim c $1" type Direction = enum north, east, south, west @@ -1228,6 +1270,7 @@ It is quite common to have arrays start at zero, so there's a shortcut syntax to specify a range from zero to the specified index minus one: .. code-block:: nim + :test: "nim c $1" type IntArray = array[0..5, int] # an array that is indexed with 0..5 QuickArray = array[6, int] # an array that is indexed with 0..5 @@ -1260,6 +1303,7 @@ A sequence may be passed to an openarray parameter. Example: .. code-block:: nim + :test: "nim c $1" var x: seq[int] # a reference to a sequence of integers @@ -1282,6 +1326,7 @@ value. Here the ``for`` statement is looping over the results from the `_ module. Examples: .. code-block:: nim + :test: "nim c $1" for value in @[3, 4, 5]: echo value # --> 3 @@ -1308,6 +1353,7 @@ with a compatible base type can be passed to an openarray parameter, the index type does not matter. .. code-block:: nim + :test: "nim c $1" var fruits: seq[string] # reference to a sequence of strings that is initialized with 'nil' capitals: array[3, string] # array of strings with a fixed size @@ -1337,6 +1383,7 @@ arguments to a procedure. The compiler converts the list of arguments to an array automatically: .. code-block:: nim + :test: "nim c $1" proc myWriteln(f: File, a: varargs[string]) = for s in items(a): write(f, s) @@ -1351,6 +1398,7 @@ last parameter in the procedure header. It is also possible to perform type conversions in this context: .. code-block:: nim + :test: "nim c $1" proc myWriteln(f: File, a: varargs[string, `$`]) = for s in items(a): write(f, s) @@ -1374,6 +1422,7 @@ context. A slice is just an object of type Slice which contains two bounds, define operators which accept Slice objects to define ranges. .. code-block:: nim + :test: "nim c $1" var a = "Nim is a progamming language" @@ -1429,6 +1478,7 @@ The assignment operator for tuples copies each component. The notation integer. .. code-block:: nim + :test: "nim c $1" type Person = tuple[name: string, age: int] # type representing a person: @@ -1474,6 +1524,7 @@ otherwise you will be assigning the same value to all the individual variables! For example: .. code-block:: nim + :test: "nim c $1" import os @@ -1513,6 +1564,7 @@ tuple/object field operator) and ``[]`` (array/string/sequence index operator) operators perform implicit dereferencing operations for reference types: .. code-block:: nim + :test: "nim c $1" type Node = ref object @@ -1542,6 +1594,7 @@ techniques. Example: .. code-block:: nim + :test: "nim c $1" proc echoItem(x: int) = echo x proc forEach(action: proc (x: int)) = From 5d2e86ea1a4c6a0b1e74708f038da5590eeffc48 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:13:54 +0100 Subject: [PATCH 14/92] the documentation generator now supports ':test:' for the testing of test snippets --- changelog.md | 7 +++++++ compiler/docgen.nim | 24 ++++++++++++++++++++++-- compiler/semexprs.nim | 15 ++++++++------- compiler/transf.nim | 2 +- lib/packages/docutils/highlite.nim | 9 +++------ lib/packages/docutils/rst.nim | 13 ------------- lib/packages/docutils/rstgen.nim | 24 +++++++++++++++++++----- 7 files changed, 60 insertions(+), 34 deletions(-) diff --git a/changelog.md b/changelog.md index 14352374c9..aaee99cfbd 100644 --- a/changelog.md +++ b/changelog.md @@ -98,3 +98,10 @@ This now needs to be written as: - Added ``system.runnableExamples`` to make examples in Nim's documentation easier to write and test. The examples are tested as the last step of ``nim doc``. +- Nim's ``rst2html`` command now supports the testing of code snippets via an RST + extension that we called ``:test:``:: + + .. code-block:: nim + :test: + # shows how the 'if' statement works + if true: echo "yes" diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 4a3674812e..94cba4ffd7 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -22,7 +22,6 @@ type TSections = array[TSymKind, Rope] TDocumentor = object of rstgen.RstGenerator modDesc: Rope # module description - id: int # for generating IDs toc, section: TSections indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist @@ -109,6 +108,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc = result.id = 100 result.jArray = newJArray() initStrTable result.types + result.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) = + localError(newLineInfo(d.filename, -1, -1), warnUser, "only 'rst2html' supports the ':test:' attribute") proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) = if gCmd != cmdRst2tex: addf(dest, xml, args) @@ -274,7 +275,9 @@ proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) = # that the renderer currently produces: var i = 0 var body = n.lastSon - if body.len == 1 and body.kind == nkStmtList: body = body.lastSon + if body.len == 1 and body.kind == nkStmtList and + body.lastSon.kind == nkStmtList: + body = body.lastSon for b in body: if i > 0: dest.add "\n" inc i @@ -785,6 +788,23 @@ proc commandDoc*() = proc commandRstAux(filename, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen, options.gConfigVars) + d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; + status: int; content: string) = + var outp: string + if filename.len == 0: + inc(d.id) + outp = getNimcacheDir() / splitFile(d.filename).name & "_snippet_" & $d.id & ".nim" + elif isAbsolute(filename): + outp = filename + else: + # Nim's convention: every path is relative to the file it was written in: + outp = splitFile(d.filename).dir / filename + writeFile(outp, content) + let cmd = cmd % outp + rawMessage(hintExecuting, cmd) + if execShellCmd(cmd) != status: + rawMessage(errExecutionOfProgramFailed, cmd) + d.isPureRst = true var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc, {roSupportRawDirective}) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 4942ef3856..1598d1909e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1849,13 +1849,14 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = magicsAfterOverloadResolution(c, result, flags) of mRunnableExamples: if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList: - if sfMainModule in c.module.flags: - let inp = toFullPath(c.module.info) - if c.runnableExamples == nil: - c.runnableExamples = newTree(nkStmtList, - newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) - c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon) - result = setMs(n, s) + if n.sons[0].kind == nkIdent: + if sfMainModule in c.module.flags: + let inp = toFullPath(c.module.info) + if c.runnableExamples == nil: + c.runnableExamples = newTree(nkStmtList, + newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp)))) + c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon) + result = setMs(n, s) else: result = emptyNode else: diff --git a/compiler/transf.nim b/compiler/transf.nim index 69c5269510..8e4bb935b0 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -693,7 +693,7 @@ proc transformCall(c: PTransf, n: PNode): PTransNode = inc(j) add(result, a.PTransNode) if len(result) == 2: result = result[1] - elif magic in {mNBindSym, mTypeOf}: + elif magic in {mNBindSym, mTypeOf, mRunnableExamples}: # for bindSym(myconst) we MUST NOT perform constant folding: result = n.PTransNode elif magic == mProcCall: diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 70369b001d..2a58854a68 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -31,14 +31,12 @@ type state: TokenClass SourceLanguage* = enum - langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava, + langNone, langNim, langCpp, langCsharp, langC, langJava, langYaml -{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass, - TGeneralTokenizer: GeneralTokenizer].} const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"] + "Nim", "C++", "C#", "C", "Java", "Yaml"] tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", @@ -398,7 +396,6 @@ type TokenizerFlag = enum hasPreprocessor, hasNestedComments TokenizerFlags = set[TokenizerFlag] -{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].} proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], flags: TokenizerFlags) = @@ -888,7 +885,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = case lang of langNone: assert false - of langNim, langNimrod: nimNextToken(g) + of langNim: nimNextToken(g) of langCpp: cppNextToken(g) of langCsharp: csharpNextToken(g) of langC: cNextToken(g) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 53699166fb..223fc836a3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -45,8 +45,6 @@ type MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.nimcall.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.nimcall.} -{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption, - TMsgKind: MsgKind].} const messages: array[MsgKind, string] = [ @@ -127,8 +125,6 @@ type bufpos*: int line*, col*, baseIndent*: int skipPounds*: bool -{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq, - TLexer: Lexer].} proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.kind = tkWord @@ -288,10 +284,6 @@ type hasToc*: bool EParseError* = object of ValueError -{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution, - TSharedState: SharedState, TRstParser: RstParser, - TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler, - TMsgClass: MsgClass].} proc whichMsgClass*(k: MsgKind): MsgClass = ## returns which message class `k` belongs to. @@ -341,11 +333,6 @@ proc rstMessage(p: RstParser, msgKind: MsgKind) = p.col + p.tok[p.idx].col, msgKind, p.tok[p.idx].symbol) -when false: - proc corrupt(p: RstParser) = - assert p.indentStack[0] == 0 - for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000 - proc currInd(p: RstParser): int = result = p.indentStack[high(p.indentStack)] diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index f156c440bb..e6c95b59ef 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -61,6 +61,9 @@ type seenIndexTerms: Table[string, int] ## \ ## Keeps count of same text index terms to generate different identifiers ## for hyperlinks. See renderIndexTerm proc for details. + id*: int ## A counter useful for generating IDs. + onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; + content: string) PDoc = var RstGenerator ## Alias to type less. @@ -69,8 +72,9 @@ type startLine: int ## The starting line of the code block, by default 1. langStr: string ## Input string used to specify the language. lang: SourceLanguage ## Type of highlighting, by default none. -{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry, - TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].} + filename: string + testCmd: string + status: int proc init(p: var CodeBlockParams) = ## Default initialisation of CodeBlockParams to sane values. @@ -133,6 +137,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.options = options g.findFile = findFile g.currentSection = "" + g.id = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -368,7 +373,6 @@ type ## ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].} proc cmp(a, b: IndexEntry): int = ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. @@ -823,13 +827,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = var number: int if parseInt(n.getFieldValue, number) > 0: params.startLine = number - of "file": + of "file", "filename": # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This # field is dealt with in ``rst.nim`` which replaces the existing block with # the referenced file, so we only need to ignore it here to avoid incorrect # warning messages. - discard + params.filename = n.getFieldValue.strip + of "test": + params.testCmd = n.getFieldValue.strip + if params.testCmd.len == 0: params.testCmd = "nim c -r $1" + of "status": + var status: int + if parseInt(n.getFieldValue, status) > 0: + params.status = status of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage @@ -901,6 +912,9 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = var m = n.sons[2].sons[0] assert m.kind == rnLeaf + if params.testCmd.len > 0 and d.onTestSnippet != nil: + d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text) + let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text) dispA(d.target, result, blockStart, "\\begin{rstpre}\n", []) From 58c3e5d2f518d10f500dec9627fbe6c4c91dfc64 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:25:59 +0100 Subject: [PATCH 15/92] test the snippets in tut2.rst --- doc/tut2.rst | 52 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/doc/tut2.rst b/doc/tut2.rst index 0636c4ed6a..91cb528341 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -55,6 +55,7 @@ Objects have access to their type at runtime. There is an ``of`` operator that can be used to check the object's type: .. code-block:: nim + :test: "nim c $1" type Person = ref object of RootObj name*: string # the * means that `name` is accessible from other modules @@ -103,6 +104,7 @@ would require arbitrary symbol lookahead which slows down compilation.) Example: .. code-block:: nim + :test: "nim c $1" type Node = ref object # a reference to an object with the following field: le, ri: Node # left and right subtrees @@ -144,6 +146,7 @@ variant types are needed. An example: .. code-block:: nim + :test: "nim c $1" # This is an example how an abstract syntax tree could be modelled in Nim type @@ -201,9 +204,11 @@ This method call syntax is not restricted to objects, it can be used for any type: .. code-block:: nim + :test: "nim c $1" + import strutils echo "abc".len # is the same as echo len("abc") - echo "abc".toUpper() + echo "abc".toUpperAscii() echo({'a', 'b', 'c'}.card) stdout.writeLine("Hallo") # the same as writeLine(stdout, "Hallo") @@ -213,6 +218,7 @@ postfix notation.) So "pure object oriented" code is easy to write: .. code-block:: nim + :test: "nim c $1" import strutils, sequtils stdout.writeLine("Give a list of numbers (separated by spaces): ") @@ -228,6 +234,7 @@ the same. But setting a value is different; for this a special setter syntax is needed: .. code-block:: nim + :test: "nim c $1" type Socket* = ref object of RootObj @@ -252,6 +259,7 @@ The ``[]`` array access operator can be overloaded to provide `array properties`:idx:\ : .. code-block:: nim + :test: "nim c $1" type Vector* = object x, y, z: float @@ -283,23 +291,24 @@ Procedures always use static dispatch. For dynamic dispatch replace the ``proc`` keyword by ``method``: .. code-block:: nim + :test: "nim c $1" type - PExpr = ref object of RootObj ## abstract base class for an expression - PLiteral = ref object of PExpr + Expression = ref object of RootObj ## abstract base class for an expression + Literal = ref object of Expression x: int - PPlusExpr = ref object of PExpr - a, b: PExpr + PlusExpr = ref object of Expression + a, b: Expression # watch out: 'eval' relies on dynamic binding - method eval(e: PExpr): int = + method eval(e: Expression): int = # override this base method quit "to override!" - method eval(e: PLiteral): int = e.x - method eval(e: PPlusExpr): int = eval(e.a) + eval(e.b) + method eval(e: Literal): int = e.x + method eval(e: PlusExpr): int = eval(e.a) + eval(e.b) - proc newLit(x: int): PLiteral = PLiteral(x: x) - proc newPlus(a, b: PExpr): PPlusExpr = PPlusExpr(a: a, b: b) + proc newLit(x: int): Literal = Literal(x: x) + proc newPlus(a, b: Expression): PlusExpr = PlusExpr(a: a, b: b) echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4))) @@ -311,6 +320,7 @@ In a multi-method all parameters that have an object type are used for the dispatching: .. code-block:: nim + :test: "nim c $1" type Thing = ref object of RootObj @@ -365,6 +375,7 @@ Raise statement Raising an exception is done with the ``raise`` statement: .. code-block:: nim + :test: "nim c $1" var e: ref OSError new(e) @@ -385,6 +396,9 @@ Try statement The ``try`` statement handles exceptions: .. code-block:: nim + :test: "nim c $1" + from strutils import parseInt + # read the first two lines of a text file that should contain numbers # and tries to add them var @@ -479,6 +493,7 @@ with `type parameters`:idx:. They are most useful for efficient type safe containers: .. code-block:: nim + :test: "nim c $1" type BinaryTree*[T] = ref object # BinaryTree is a generic type with # generic param ``T`` @@ -573,6 +588,7 @@ Templates are especially useful for lazy evaluation purposes. Consider a simple proc for logging: .. code-block:: nim + :test: "nim c $1" const debug = true @@ -590,6 +606,7 @@ evaluation for procedures is *eager*). Turning the ``log`` proc into a template solves this problem: .. code-block:: nim + :test: "nim c $1" const debug = true @@ -611,6 +628,7 @@ If the template has no explicit return type, To pass a block of statements to a template, use 'untyped' for the last parameter: .. code-block:: nim + :test: "nim c $1" template withFile(f: untyped, filename: string, mode: FileMode, body: untyped): typed = @@ -665,6 +683,7 @@ The following example implements a powerful ``debug`` command that accepts a variable number of arguments: .. code-block:: nim + :test: "nim c $1" # to work with Nim syntax trees, we need an API that is defined in the # ``macros`` module: import macros @@ -744,6 +763,7 @@ dynamic code into something that compiles statically. For the exercise we will use the following snippet of code as the starting point: .. code-block:: nim + :test: "nim c $1" import strutils, tables @@ -863,9 +883,9 @@ variables with ``cfg``. In essence, what the compiler is doing is replacing the line calling the macro with the following snippet of code: .. code-block:: nim - const cfgversion= "1.1" - const cfglicenseOwner= "Hyori Lee" - const cfglicenseKey= "M1Tl3PjBWO2CC48m" + const cfgversion = "1.1" + const cfglicenseOwner = "Hyori Lee" + const cfglicenseKey = "M1Tl3PjBWO2CC48m" You can verify this yourself adding the line ``echo source`` somewhere at the end of the macro and compiling the program. Another difference is that instead @@ -891,12 +911,13 @@ an expression macro. Since we know that we want to generate a bunch of see what the compiler *expects* from us: .. code-block:: nim + :test: "nim c $1" import macros dumpTree: const cfgversion: string = "1.1" - const cfglicenseOwner= "Hyori Lee" - const cfglicenseKey= "M1Tl3PjBWO2CC48m" + const cfglicenseOwner = "Hyori Lee" + const cfglicenseKey = "M1Tl3PjBWO2CC48m" During compilation of the source code we should see the following lines in the output (again, since this is a macro, compilation is enough, you don't have to @@ -996,6 +1017,7 @@ Lifting Procs +++++++++++++ .. code-block:: nim + :test: "nim c $1" import math template liftScalarProc(fname) = From c47ed6c5374c330b892187d26e0360a97367507d Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Mon, 27 Nov 2017 16:52:49 -0800 Subject: [PATCH 16/92] Fixed ospaths compilation on js (#6826) --- lib/pure/ospaths.nim | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index c3bd399db7..0d638abb9b 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -602,14 +602,13 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} else: return "'" & s.replace("'", "'\"'\"'") & "'" -proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote ``s``, so it can be safely passed to shell. - when defined(Windows): - return quoteShellWindows(s) - elif defined(posix): - return quoteShellPosix(s) - else: - {.error:"quoteShell is not supported on your system".} +when defined(windows) or defined(posix): + proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to shell. + when defined(windows): + return quoteShellWindows(s) + else: + return quoteShellPosix(s) when isMainModule: assert quoteShellWindows("aaa") == "aaa" From 06a4dcb17d9b4aef5b98bdac0b512055cc885f08 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 28 Nov 2017 00:57:25 +0000 Subject: [PATCH 17/92] Implement doAssertRaises (#6819) --- lib/system.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 323ff00e6d..d9e315da12 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3752,6 +3752,7 @@ template assert*(cond: bool, msg = "") = ## that ``AssertionError`` is hidden from the effect system, so it doesn't ## produce ``{.raises: [AssertionError].}``. This exception is only supposed ## to be caught by unit testing frameworks. + ## ## The compiler may not generate any code at all for ``assert`` if it is ## advised to do so through the ``-d:release`` or ``--assertions:off`` ## `command line switches `_. @@ -4009,3 +4010,21 @@ when defined(nimHasRunnableExamples): else: template runnableExamples*(body: untyped) = discard + +template doAssertRaises*(exception, code: untyped): typed = + ## Raises ``AssertionError`` if specified ``code`` does not raise the + ## specified exception. + runnableExamples: + doAssertRaises(ValueError): + raise newException(ValueError, "Hello World") + + try: + block: + code + raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) + except exception: + discard + except Exception as exc: + raiseAssert(astToStr(exception) & + " wasn't raised, another error was raised instead by:\n"& + astToStr(code)) \ No newline at end of file From d0d02c2fe32afacad7201a8159152ec60fd5f107 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:38:46 +0100 Subject: [PATCH 18/92] re.nim: Make tests green and deprecate 'parallelReplace'; it should be 'multiReplace' for consistency with strutils.nim --- lib/impure/re.nim | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 24fc83366d..2fbed2479d 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -470,8 +470,8 @@ proc replacef*(s: string, sub: Regex, by: string): string = prev = match.last + 1 add(result, substr(s, prev)) -proc parallelReplace*(s: string, subs: openArray[ - tuple[pattern: Regex, repl: string]]): string = +proc multiReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string = ## Returns a modified copy of ``s`` with the substitutions in ``subs`` ## applied in parallel. result = "" @@ -490,13 +490,20 @@ proc parallelReplace*(s: string, subs: openArray[ # copy the rest: add(result, substr(s, i)) +proc parallelReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string {.deprecated.} = + ## Returns a modified copy of ``s`` with the substitutions in ``subs`` + ## applied in parallel. + ## **Deprecated since version 0.18.0**: Use ``multiReplace`` instead. + result = multiReplace(s, subs) + proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file ``infile``, performs a parallel replacement (calls ## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) + writeFile(outfile, x.multiReplace(subs)) iterator split*(s: string, sep: Regex): string = ## Splits the string ``s`` into substrings. @@ -579,12 +586,12 @@ const ## common regular expressions ## describes an URL when isMainModule: - doAssert match("(a b c)", re"\( .* \)") + doAssert match("(a b c)", rex"\( .* \)") doAssert match("WHiLe", re("while", {reIgnoreCase})) doAssert "0158787".match(re"\d+") doAssert "ABC 0232".match(re"\w+\s+\d+") - doAssert "ABC".match(re"\d+ | \w+") + doAssert "ABC".match(rex"\d+ | \w+") {.push warnings:off.} doAssert matchLen("key", re(reIdentifier)) == 3 From 01304db5b2b05ba7ad06769c2823d49267b06c46 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:49:20 +0100 Subject: [PATCH 19/92] re.nim: removed deprecated symbols --- lib/impure/re.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 2fbed2479d..c7f8f336bd 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -49,9 +49,6 @@ type RegexError* = object of ValueError ## is raised if the pattern is no valid regular expression. -{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex, - EInvalidRegEx: RegexError].} - proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = var e: ref RegexError new(e) From d6d40a2d3cf327c42a9415809a507fe5a9b3507d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 01:51:00 +0100 Subject: [PATCH 20/92] fixes the docgen configuration; refs #6780 --- config/nimdoc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 0c3f204a57..0357730e00 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -130,7 +130,7 @@ doc.body_toc_group = """
""" -else: +@else doc.body_toc_group = """
From 8aebd3851467dbef43c151600f787a8f6b35c71a Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 02:18:23 +0100 Subject: [PATCH 21/92] fixes #6820 --- compiler/extccomp.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index cac3a6e9f9..e6b23aae52 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -728,13 +728,13 @@ proc execCmdsInParallel(cmds: seq[string]; prettyCb: proc (idx: int)) = else: tryExceptOSErrorMessage("invocation of external compiler program failed."): if optListCmd in gGlobalOptions or gVerbosity > 1: - res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, + res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath}, gNumberOfProcessors, afterRunEvent=runCb) elif gVerbosity == 1: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, + res = execProcesses(cmds, {poStdErrToStdOut, poUsePath}, gNumberOfProcessors, prettyCb, afterRunEvent=runCb) else: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, + res = execProcesses(cmds, {poStdErrToStdOut, poUsePath}, gNumberOfProcessors, afterRunEvent=runCb) if res != 0: if gNumberOfProcessors <= 1: From 439b72b40248e35628d19a66cc658718bb94424e Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 02:19:39 +0100 Subject: [PATCH 22/92] osproc improvement: check API consistency in order to prevent bug #6820 --- lib/pure/osproc.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index cc4c261613..9865f114fb 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -321,6 +321,8 @@ when not defined(useNimRtl): elif not running(p): break close(p) +template streamAccess(p) = + assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: @@ -581,12 +583,15 @@ when defined(Windows) and not defined(useNimRtl): return res proc inputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.inHandle) proc outputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.outHandle) proc errorStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.errHandle) proc execCmd(command: string): int = @@ -1152,16 +1157,19 @@ elif not defined(useNimRtl): stream = newFileStream(f) proc inputStream(p: Process): Stream = + streamAccess(p) if p.inStream == nil: createStream(p.inStream, p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = + streamAccess(p) if p.outStream == nil: createStream(p.outStream, p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = + streamAccess(p) if p.errStream == nil: createStream(p.errStream, p.errHandle, fmRead) return p.errStream From 95629acd4d2db5bfb8a7feb902f2b66ee5b2fa73 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Tue, 28 Nov 2017 03:30:49 +0200 Subject: [PATCH 23/92] Exit nodejs with programResult (#6822) --- lib/system.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index d9e315da12..387973f4be 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1439,7 +1439,11 @@ const ## is the value that should be passed to `quit <#quit>`_ to indicate ## failure. -var programResult* {.exportc: "nim_program_result".}: int +when defined(nodejs): + var programResult* {.importc: "process.exitCode".}: int + programResult = 0 +else: + var programResult* {.exportc: "nim_program_result".}: int ## modify this variable to specify the exit code of the program ## under normal circumstances. When the program is terminated ## prematurely using ``quit``, this value is ignored. From 8c634cdb2b12c9dff15ac31820df7ecf0c6c3497 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 02:50:54 +0100 Subject: [PATCH 24/92] make the tests green again --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 94cba4ffd7..9a32636c2f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -793,7 +793,7 @@ proc commandRstAux(filename, outExt: string) = var outp: string if filename.len == 0: inc(d.id) - outp = getNimcacheDir() / splitFile(d.filename).name & "_snippet_" & $d.id & ".nim" + outp = completeGeneratedFilePath(splitFile(d.filename).name & "_snippet_" & $d.id & ".nim") elif isAbsolute(filename): outp = filename else: From 942694d91474a0738460d4cf4c22d4fdfb53a87e Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 11:14:43 +0100 Subject: [PATCH 25/92] fixes the new ':test:' feature --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 9a32636c2f..0861c25b29 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -800,7 +800,7 @@ proc commandRstAux(filename, outExt: string) = # Nim's convention: every path is relative to the file it was written in: outp = splitFile(d.filename).dir / filename writeFile(outp, content) - let cmd = cmd % outp + let cmd = unescape(cmd) % quoteShell(outp) rawMessage(hintExecuting, cmd) if execShellCmd(cmd) != status: rawMessage(errExecutionOfProgramFailed, cmd) From c6c0d28a4f1811c20782d3322f5caf2c8d4b2128 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 28 Nov 2017 14:03:09 +0200 Subject: [PATCH 26/92] Refactored version of execProcesses with test. --- lib/pure/osproc.nim | 137 ++++++++++++++++++++++++--------------- tests/osproc/texecps.nim | 32 +++++++++ 2 files changed, 115 insertions(+), 54 deletions(-) create mode 100644 tests/osproc/texecps.nim diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index cc4c261613..5a22f3a39a 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -231,55 +231,81 @@ proc execProcesses*(cmds: openArray[string], ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes ## is returned. Runs `beforeRunEvent` before running each command. - when false: - # poParentStreams causes problems on Posix, so we simply disable it: - var options = options - {poParentStreams} - + assert n > 0 if n > 1: - var q: seq[Process] - newSeq(q, n) + var i = 0 + var q = newSeq[Process](n) var m = min(n, cmds.len) - for i in 0..m-1: + + when defined(windows): + var w: WOHandleArray + var wcount = m + for c in 0..MAXIMUM_WAIT_OBJECTS - 1: + w[c] = 0 + + while i < m: if beforeRunEvent != nil: beforeRunEvent(i) - q[i] = startProcess(cmds[i], options=options + {poEvalCommand}) - when defined(noBusyWaiting): - var r = 0 - for i in m..high(cmds): - when defined(debugExecProcesses): - var err = "" - var outp = outputStream(q[r]) - while running(q[r]) or not atEnd(outp): - err.add(outp.readLine()) - err.add("\n") - echo(err) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - r = (r + 1) mod n - else: - var i = m - while i <= high(cmds): - sleep(50) - for r in 0..n-1: + q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) + when defined(windows): + w[i] = q[i].fProcessHandle + inc(i) + + var ecount = len(cmds) + while ecount > 0: + when defined(windows): + # waiting for all children, get result if any child exits + var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, + INFINITE) + if ret == WAIT_TIMEOUT: + # must not be happen + discard + elif ret == WAIT_FAILED: + raiseOSError(osLastError()) + else: + var status : cint = 1 + # waiting for all children, get result if any child exits + let res = waitpid(-1, status, 0) + if res > 0: + for r in 0..m-1: + if not isNil(q[r]) and q[r].id == res: + # we updating `exitStatus` manually, so `running()` can work. + if WIFEXITED(status) or WIFSIGNALED(status): + q[r].exitStatus = status + break + else: + let err = osLastError() + if err == OSErrorCode(ECHILD): + # some child exits, we need to check our childs exit codes + discard + elif err == OSErrorCode(EINTR): + # signal interrupted our syscall, lets repeat it + continue + else: + # all other errors are exceptions + raiseOSError(err) + + for r in 0..m-1: + if not isNil(q[r]): if not running(q[r]): - #echo(outputStream(q[r]).readLine()) - result = max(waitForExit(q[r]), result) + result = max(result, q[r].peekExitCode()) if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - inc(i) - if i > high(cmds): break - for j in 0..m-1: - result = max(waitForExit(q[j]), result) - if afterRunEvent != nil: afterRunEvent(j, q[j]) - if q[j] != nil: close(q[j]) + close(q[r]) + if i < len(cmds): + if beforeRunEvent != nil: beforeRunEvent(i) + q[r] = startProcess(cmds[i], + options = options + {poEvalCommand}) + when defined(windows): + w[r] = q[r].fProcessHandle + inc(i) + else: + q[r] = nil + when defined(windows): + for c in r..MAXIMUM_WAIT_OBJECTS - 2: + w[c] = w[c + 1] + dec(wcount) + dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: @@ -939,19 +965,22 @@ elif not defined(useNimRtl): if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) proc running(p: Process): bool = - var ret : int - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) - if ret == int(p.id): - if isExitStatus(status): - p.exitStatus = status - return false - else: - return true - elif ret == 0: - return true # Can't establish status. Assume running. - else: + if p.exitStatus != -3: return false + else: + var ret : int + var status : cint = 1 + ret = waitpid(p.id, status, WNOHANG) + if ret == int(p.id): + if isExitStatus(status): + p.exitStatus = status + return false + else: + return true + elif ret == 0: + return true # Can't establish status. Assume running. + else: + raiseOSError(osLastError()) proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: diff --git a/tests/osproc/texecps.nim b/tests/osproc/texecps.nim new file mode 100644 index 0000000000..887d79bfbf --- /dev/null +++ b/tests/osproc/texecps.nim @@ -0,0 +1,32 @@ +discard """ + file: "texecps.nim" + output: "" +""" + +import osproc, streams, strutils, os + +const NumberOfProcesses = 13 + +var gResults {.threadvar.}: seq[string] + +proc execCb(idx: int, p: Process) = + let exitCode = p.peekExitCode + if exitCode < len(gResults): + gResults[exitCode] = p.outputStream.readAll.strip + +when isMainModule: + + if paramCount() == 0: + gResults = newSeq[string](NumberOfProcesses) + var checks = newSeq[string](NumberOfProcesses) + var commands = newSeq[string](NumberOfProcesses) + for i in 0..len(commands) - 1: + commands[i] = getAppFileName() & " " & $i + checks[i] = $i + let cres = execProcesses(commands, options = {poStdErrToStdOut}, + afterRunEvent = execCb) + doAssert(cres == len(commands) - 1) + doAssert(gResults == checks) + else: + echo paramStr(1) + programResult = parseInt(paramStr(1)) From c4a57e711b154c7fa333ec6daac25059e1fd3c6f Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 28 Nov 2017 14:40:33 +0200 Subject: [PATCH 27/92] Fix nimrtl troubles. --- lib/pure/osproc.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5a22f3a39a..6e0307c8fb 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -167,8 +167,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. - -proc peekExitCode*(p: Process): int {.tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} ## return -1 if the process is still running. Otherwise the process' exit code ## ## On posix, if the process has exited because of a signal, 128 + signal From e758b9408e8fe935117f7f793164f1c9b74cec06 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 28 Nov 2017 13:09:14 +0000 Subject: [PATCH 28/92] Add boot directive in nimweb (#6824) Enables #6780 --- tools/nimweb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nimweb.nim b/tools/nimweb.nim index ffb1ac4e3a..c8b87c1f25 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -52,7 +52,7 @@ proc initConfigData(c: var TConfigData) = c.pdf = @[] c.infile = "" c.outdir = "" - c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off " + c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot " c.authors = "" c.projectTitle = "" c.projectName = "" From 8fbe37b2d813f00ca934cee2fe17bc2e24f82f88 Mon Sep 17 00:00:00 2001 From: "Lynn C. Rees" Date: Tue, 28 Nov 2017 06:16:59 -0700 Subject: [PATCH 29/92] Show nimscript configuration files during compilation (#6750) --- compiler/scriptconfig.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 22377a1e2e..dac2672636 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -143,6 +143,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true; config: ConfigRef=nil) = + rawMessage(hintConf, scriptName) passes.gIncludeFile = includeModule passes.gImportModule = importModule let graph = newModuleGraph(config) From b74a5148a9e7bf646ee6a13cad0ce046d7b9d8b4 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 26 Nov 2017 19:31:59 +0000 Subject: [PATCH 30/92] Fixes #6223. --- changelog.md | 3 +++ lib/pure/unicode.nim | 12 ++++++------ lib/system.nim | 5 ++++- tests/stdlib/tstring.nim | 4 ++++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/changelog.md b/changelog.md index aaee99cfbd..bd40d7e992 100644 --- a/changelog.md +++ b/changelog.md @@ -105,3 +105,6 @@ This now needs to be written as: :test: # shows how the 'if' statement works if true: echo "yes" +- The ``[]`` proc for strings now raises an ``IndexError`` exception when + the specified slice is out of bounds. See issue + [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details. \ No newline at end of file diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 7d9c3108ba..257c620f79 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -293,33 +293,33 @@ proc runeSubStr*(s: string, pos:int, len:int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let e = rl + len if e < 0: result = "" else: - result = s[o.. runeOffset(s, e-(rl+pos) , o)-1] + result = s.substr(o, runeOffset(s, e-(rl+pos) , o)-1) else: - result = s[o.. runeOffset(s, len, o)-1] + result = s.substr(o, runeOffset(s, len, o)-1) else: let o = runeOffset(s, pos) if o < 0: result = "" elif len == int.high: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let (e, rl) = runeReverseOffset(s, -len) discard rl if e <= 0: result = "" else: - result = s[o.. e-1] + result = s.substr(o, e-1) else: var e = runeOffset(s, len, o) if e < 0: e = s.len - result = s[o.. e-1] + result = s.substr(o, e-1) const alphaRanges = [ diff --git a/lib/system.nim b/lib/system.nim index 387973f4be..b9f01c3065 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3529,7 +3529,10 @@ when hasAlloc or defined(nimscript): ## .. code-block:: nim ## var s = "abcdef" ## assert s[1..3] == "bcd" - result = s.substr(s ^^ x.a, s ^^ x.b) + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + result = newString(L) + for i in 0 ..< L: result[i] = s[i + a] proc `[]=`*[T, U](s: var string, x: HSlice[T, U], b: string) = ## slice assignment for strings. If diff --git a/tests/stdlib/tstring.nim b/tests/stdlib/tstring.nim index ddf533a175..904bc462a2 100644 --- a/tests/stdlib/tstring.nim +++ b/tests/stdlib/tstring.nim @@ -50,6 +50,10 @@ proc test_string_slice() = s[2..0] = numbers doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz" + # bug #6223 + doAssertRaises(IndexError): + discard s[0..999] + echo("OK") test_string_slice() From a22dba4a8bcfc04ef444ae9fd6b762ec093ac9e4 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 14:40:27 +0100 Subject: [PATCH 31/92] newruntime: removed old way of writing destructors --- compiler/pragmas.nim | 6 +----- compiler/semdestruct.nim | 15 +++++++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index f1d81f7983..b598cadb20 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -23,7 +23,7 @@ const wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC, - wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl, + wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe, wOverride, wConstructor, wExportNims, wUsed, wLiftLocals} converterPragmas* = procPragmas @@ -759,10 +759,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, incl(sym.loc.flags, lfNoDecl) # implies nodecl, because otherwise header would not make sense if sym.loc.r == nil: sym.loc.r = rope(sym.name.s) - of wDestructor: - sym.flags.incl sfOverriden - if sym.name.s.normalize != "destroy": - localError(n.info, errGenerated, "destructor has to be named 'destroy'") of wOverride: sym.flags.incl sfOverriden of wNosideeffect: diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim index 4b61c6316c..b16bf004ff 100644 --- a/compiler/semdestruct.nim +++ b/compiler/semdestruct.nim @@ -23,7 +23,6 @@ new(destructorIsTrivial) var destructorName = getIdent"destroy_" destructorParam = getIdent"this_" - destructorPragma = newIdentNode(getIdent"destructor", unknownLineInfo()) proc instantiateDestructor(c: PContext, typ: PType): PType @@ -150,19 +149,19 @@ proc instantiateDestructor(c: PContext, typ: PType): PType = let generated = generateDestructor(c, t) if generated != nil: internalAssert t.sym != nil - var i = t.sym.info - let fullDef = newNode(nkProcDef, i, @[ - newIdentNode(destructorName, i), + let info = t.sym.info + let fullDef = newNode(nkProcDef, info, @[ + newIdentNode(destructorName, info), emptyNode, emptyNode, - newNode(nkFormalParams, i, @[ + newNode(nkFormalParams, info, @[ emptyNode, - newNode(nkIdentDefs, i, @[ - newIdentNode(destructorParam, i), + newNode(nkIdentDefs, info, @[ + newIdentNode(destructorParam, info), symNodeFromType(c, makeVarType(c, t), t.sym.info), emptyNode]), ]), - newNode(nkPragma, i, @[destructorPragma]), + emptyNode, emptyNode, generated ]) From e2787c557cdabb45d90fa67fd54e57fab920171b Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 17:34:21 +0100 Subject: [PATCH 32/92] mimetypes improvement: make mimetypes easier to use by allowing the extension to start with a dot which is what splitFile().ext returns --- lib/pure/mimetypes.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 1e315afb4a..b397ef47b5 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -491,6 +491,8 @@ const mimes* = { "vrml": "x-world/x-vrml", "wrl": "x-world/x-vrml"} +from strutils import startsWith + proc newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. @@ -498,8 +500,11 @@ proc newMimetypes*(): MimeDB = proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext`` - ## could not be found. - result = mimedb.mimes.getOrDefault(ext) + ## could not be found. ``ext`` can start with an optional dot which is ignored. + if ext.startsWith("."): + result = mimedb.mimes.getOrDefault(ext.substr(1)) + else: + result = mimedb.mimes.getOrDefault(ext) if result == "": return default From 45821ea2ab081a77b057b6837dae5be52b975cee Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 27 Nov 2017 17:51:23 +0000 Subject: [PATCH 33/92] Fixes #4377. --- changelog.md | 3 ++ lib/pure/htmlgen.nim | 4 +-- lib/pure/strutils.nim | 82 ++++++++++++++++++++++++++----------------- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/changelog.md b/changelog.md index aaee99cfbd..5958085db5 100644 --- a/changelog.md +++ b/changelog.md @@ -105,3 +105,6 @@ This now needs to be written as: :test: # shows how the 'if' statement works if true: echo "yes" +- ``strutils.split`` and ``strutils.rsplit`` with an empty string and a + separator now returns that empty string. + See issue [#4377](https://github.com/nim-lang/Nim/issues/4377). diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index ad199a2156..c0934a45b7 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -59,8 +59,8 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", # copy the attributes; when iterating over them these lists # will be modified, so that each attribute is only given one value - var req = split(reqAttr) - var opt = split(optAttr) + var req = splitWhitespace(reqAttr) + var opt = splitWhitespace(optAttr) result = newNimNode(nnkBracket, e) result.add(newStrLitNode("<")) result.add(newStrLitNode(tag)) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 6fe2df2168..62ceaa2e8b 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -502,16 +502,15 @@ template splitCommon(s, sep, maxsplit, sepLen) = var last = 0 var splits = maxsplit - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and not stringHasSep(s, last, sep): - inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last, sepLen) + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) template oldSplit(s, seps, maxsplit) = var last = 0 @@ -669,30 +668,29 @@ template rsplitCommon(s, sep, maxsplit, sepLen) = splits = maxsplit startPos = 0 - if len(s) > 0: - # go to -1 in order to get separators at the beginning - while first >= -1: - while first >= 0 and not stringHasSep(s, first, sep): - dec(first) - - if splits == 0: - # No more splits means set first to the beginning - first = -1 - - if first == -1: - startPos = 0 - else: - startPos = first + sepLen - - yield substr(s, startPos, last) - - if splits == 0: - break - - dec(splits) + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): dec(first) - last = first + if splits == 0: + # No more splits means set first to the beginning + first = -1 + + if first == -1: + startPos = 0 + else: + startPos = first + sepLen + + yield substr(s, startPos, last) + + if splits == 0: + break + + dec(splits) + dec(first) + + last = first iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = @@ -820,12 +818,18 @@ proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[st noSideEffect, rtl, extern: "nsuSplitCharSet".} = ## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a ## proc that returns a sequence of substrings. + runnableExamples: + doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] + doAssert "".split({' '}) == @[""] accumulateResult(split(s, seps, maxsplit)) proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = ## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc ## that returns a sequence of substrings. + runnableExamples: + doAssert "a,b,c".split(',') == @["a", "b", "c"] + doAssert "".split(' ') == @[""] accumulateResult(split(s, sep, maxsplit)) proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, @@ -834,6 +838,13 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. + runnableExamples: + doAssert "a,b,c".split(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".split("Elon Musk") == @[""] + doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + + doAssert "a largely spaced sentence".split(" ", maxsplit=1) == @["a", " largely spaced sentence"] doAssert(sep.len > 0) accumulateResult(split(s, sep, maxsplit)) @@ -902,6 +913,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] ## .. code-block:: nim ## @["Root#Object#Method", "Index"] ## + runnableExamples: + doAssert "a largely spaced sentence".rsplit(" ", maxsplit=1) == @["a largely spaced", "sentence"] + + doAssert "a,b,c".rsplit(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".rsplit("Elon Musk") == @[""] + doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] accumulateResult(rsplit(s, sep, maxsplit)) result.reverse() From cb8dd0252fc28483e81faaf7f372c37f161c19f5 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 22:27:43 +0100 Subject: [PATCH 34/92] fixes #6831 --- compiler/docgen.nim | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 0861c25b29..2796c77471 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -553,12 +553,24 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 -proc traceDeps(d: PDoc, n: PNode) = +proc traceDeps(d: PDoc, it: PNode) = const k = skModule - if d.section[k] != nil: add(d.section[k], ", ") - dispA(d.section[k], - "$1", - "$1", [rope(getModuleName(n))]) + + if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket: + let sep = it[0] + let dir = it[1] + let a = newNodeI(nkInfix, it.info) + a.add sep + a.add dir + a.add sep # dummy entry, replaced in the loop + for x in it[2]: + a.sons[2] = x + traceDeps(d, a) + else: + if d.section[k] != nil: add(d.section[k], ", ") + dispA(d.section[k], + "$1", + "$1", [rope(getModuleName(it))]) proc generateDoc*(d: PDoc, n: PNode) = case n.kind From 7660e59afe88916b8d0403295dd9dd242fc429d0 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 28 Nov 2017 23:13:46 +0100 Subject: [PATCH 35/92] doc gen :test: feature: created a nested directory in order to keep Nim happy in parallel builds --- compiler/docgen.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2796c77471..65dcb73c95 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -805,7 +805,10 @@ proc commandRstAux(filename, outExt: string) = var outp: string if filename.len == 0: inc(d.id) - outp = completeGeneratedFilePath(splitFile(d.filename).name & "_snippet_" & $d.id & ".nim") + let nameOnly = splitFile(d.filename).name + let subdir = getNimcacheDir() / nameOnly + createDir(subdir) + outp = subdir / (nameOnly & "_snippet_" & $d.id & ".nim") elif isAbsolute(filename): outp = filename else: From c43f718301bb1dbd8d2594159de7b6bb4debc1bf Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 29 Nov 2017 00:01:27 +0100 Subject: [PATCH 36/92] destructors: some improvements for bug #4214: object constructors are moved too --- compiler/destroyer.nim | 4 ++-- tests/destructor/tmove_objconstr.nim | 32 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/destructor/tmove_objconstr.nim diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 36839bf0ba..729480f81b 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -210,7 +210,7 @@ template recurse(n, dest) = dest.add p(n[i], c) proc moveOrCopy(dest, ri: PNode; c: var Con): PNode = - if ri.kind in nkCallKinds: + if ri.kind in nkCallKinds+{nkObjConstr}: result = genSink(ri.typ, dest) # watch out and no not transform 'ri' twice if it's a call: let ri2 = copyNode(ri) @@ -312,7 +312,7 @@ proc injectDestructorCalls*(owner: PSym; n: PNode): PNode = result.add body when defined(nimDebugDestroys): - if owner.name.s == "createSeq": + if owner.name.s == "main" or true: echo "------------------------------------" echo owner.name.s, " transformed to: " echo result diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim new file mode 100644 index 0000000000..3e8a644351 --- /dev/null +++ b/tests/destructor/tmove_objconstr.nim @@ -0,0 +1,32 @@ + +discard """ +output: '''test created +test destroyed 0''' + cmd: '''nim c --newruntime $file''' +""" + +# bug #4214 +type + Data = object + data: string + rc: int + +proc `=destroy`(d: var Data) = + dec d.rc + echo d.data, " destroyed ", d.rc + +proc `=`(dst: var Data, src: Data) = + echo src.data, " copied" + dst.data = src.data & " (copy)" + dec dst.rc + inc dst.rc + +proc initData(s: string): Data = + result = Data(data: s, rc: 1) + echo s, " created" + +proc main = + var x = initData"test" + +when isMainModule: + main() From d0659319913ef25a45ab26491f5ccc688770c70f Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 29 Nov 2017 00:02:49 +0100 Subject: [PATCH 37/92] destructors: harden the test case --- tests/destructor/tmove_objconstr.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim index 3e8a644351..20dc062f9d 100644 --- a/tests/destructor/tmove_objconstr.nim +++ b/tests/destructor/tmove_objconstr.nim @@ -25,8 +25,11 @@ proc initData(s: string): Data = result = Data(data: s, rc: 1) echo s, " created" +proc pointlessWrapper(s: string): Data = + result = initData(s) + proc main = - var x = initData"test" + var x = pointlessWrapper"test" when isMainModule: main() From c00de13e1ff10775ff5979e057031e17cb646dfd Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 29 Nov 2017 00:19:27 +0100 Subject: [PATCH 38/92] closes #985 --- tests/destructor/tmove_objconstr.nim | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim index 20dc062f9d..8aa12ed056 100644 --- a/tests/destructor/tmove_objconstr.nim +++ b/tests/destructor/tmove_objconstr.nim @@ -1,7 +1,12 @@ discard """ output: '''test created -test destroyed 0''' +test destroyed 0 +1 +2 +3 +4 +Pony is dying!''' cmd: '''nim c --newruntime $file''' """ @@ -33,3 +38,22 @@ proc main = when isMainModule: main() + +# bug #985 + +type + Pony = object + name: string + +proc `=destroy`(o: var Pony) = + echo "Pony is dying!" + +proc getPony: Pony = + result.name = "Sparkles" + +iterator items(p: Pony): int = + for i in 1..4: + yield i + +for x in getPony(): + echo x From c343303efeb063102d33bcb7d214b384f3dcd7df Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Wed, 29 Nov 2017 07:34:30 +0700 Subject: [PATCH 39/92] Fix usage of parameters types in templates #6756 (#6768) --- compiler/evaltempl.nim | 2 +- tests/ccgbugs/t6756.nim | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/ccgbugs/t6756.nim diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 2c8abdfced..7fa6df3da4 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -42,7 +42,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam: handleParam actual.sons[s.owner.typ.len + s.position - 1] else: - internalAssert sfGenSym in s.flags + internalAssert sfGenSym in s.flags or s.kind == skType var x = PSym(idTableGet(c.mapping, s)) if x == nil: x = copySym(s, false) diff --git a/tests/ccgbugs/t6756.nim b/tests/ccgbugs/t6756.nim new file mode 100644 index 0000000000..0f08557ebb --- /dev/null +++ b/tests/ccgbugs/t6756.nim @@ -0,0 +1,18 @@ +import typetraits +type + A[T] = ref object + v: T + +template templ(o: A, op: untyped): untyped = + type T = type(o.v) + + var res: A[T] + + block: + var it {.inject.}: T + it = o.v + res = A[T](v: op) + res + +let a = A[int](v: 1) +echo templ(a, it + 2)[] From 5fdc69dfbdcf0134d3f6a97cde1d802ebdeb0a13 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 29 Nov 2017 00:35:26 +0000 Subject: [PATCH 40/92] Update docgen style (#6723) Switch to Lato font for better readability Make text darker Make spacing between paragraph and pre block consistent Fix search input box to prevent overlapping with text --- config/nimdoc.cfg | 63 ++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 0357730e00..2800bc581a 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -109,11 +109,11 @@ doc.body_toc_group = """
-
+
Search:
-
+
Group by:
@@ -184,7 +184,7 @@ doc.file = """ - + @@ -217,18 +217,19 @@ html { /* Where we want fancier font if available */ h1, h2, h3, h4, h5, h6, p.module-desc, table.docinfo + blockquote p, table.docinfo blockquote p, h1 + blockquote p { - font-family: "Raleway", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; } + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; } h1.title { font-weight: 900; } body { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-weight: 400; - font-size: 14px; + font-size: 16px; line-height: 20px; - color: #666; - background-color: rgba(252, 248, 244, 0.75); } + color: #444; + letter-spacing: 0.15px; + background-color: rgba(252, 248, 244, 0.45); } /* Skeleton grid */ .container { @@ -344,8 +345,8 @@ cite { font-style: italic !important; } dt > pre { - border-color: rgba(0, 0, 0, 0.15); - background-color: transparent; + border-color: rgba(0, 0, 0, 0.1); + background-color: rgba(255, 255, 255, 0.3); margin: 15px 0px 5px; } dd > pre { @@ -362,6 +363,17 @@ dd > pre { width: 100%; table-layout: fixed; } +/* Nim search input */ +div#searchInput { + margin-bottom: 8px; +} +div#searchInput input#searchInput { + width: 10em; +} +div.search-groupby { + margin-bottom: 8px; +} + table.line-nums-table { border-radius: 4px; border: 1px solid #cccccc; @@ -505,7 +517,7 @@ img { box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); } p { - margin: 0 0 12px; } + margin: 0 0 8px; } small { font-size: 85%; } @@ -525,7 +537,7 @@ h3, h4, h5, h6 { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-weight: 600; line-height: 20px; color: inherit; @@ -533,6 +545,7 @@ h6 { h1 { font-size: 2em; + font-weight: 400; padding-bottom: .15em; border-bottom: 1px solid #aaaaaa; margin-top: 1.0em; @@ -663,13 +676,13 @@ pre { box-sizing: border-box; min-width: calc(100% - 19.5px); padding: 9.5px; - margin: 0.25em 10px 0.25em 10px; - font-size: 14px; + margin: 0.25em 10px 10px 10px; + font-size: 15px; line-height: 20px; white-space: pre !important; overflow-y: hidden; overflow-x: visible; - background-color: whitesmoke; + background-color: rgba(0, 0, 0, 0.01); border: 1px solid #cccccc; -webkit-border-radius: 4px; -moz-border-radius: 4px; @@ -948,14 +961,14 @@ div.admonition p.admonition-title, div.hint p.admonition-title, div.important p.admonition-title, div.note p.admonition-title, div.tip p.admonition-title { font-weight: bold; - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; } + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; } div.attention p.admonition-title, div.caution p.admonition-title, div.danger p.admonition-title, div.error p.admonition-title, div.warning p.admonition-title, .code .error { color: #b30000; font-weight: bold; - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; } + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; } /* Uncomment (and remove this text!) to get reduced vertical space in compound paragraphs. @@ -1002,7 +1015,7 @@ div.sidebar { clear: right; } div.sidebar p.rubric { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-size: medium; } div.system-messages { @@ -1109,12 +1122,12 @@ p.rubric { text-align: center; } p.sidebar-title { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-weight: bold; font-size: larger; } p.sidebar-subtitle { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-weight: bold; } p.topic-title { @@ -1156,15 +1169,15 @@ pre.code .inserted, code .inserted { background-color: #A3D289; } span.classifier { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-style: oblique; } span.classifier-delimiter { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; font-weight: bold; } span.interpreted { - font-family: "Helvetica Neue", "HelveticaNeue", "Raleway", Helvetica, Arial, sans-serif; } + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif; } span.option { white-space: nowrap; } @@ -1187,7 +1200,7 @@ table.docinfo { margin: 0em; margin-top: 2em; margin-bottom: 2em; - font-family: "Raleway", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; + font-family: "Lato", "Helvetica Neue", "HelveticaNeue", Helvetica, Arial, sans-serif !important; color: #444444; } table.docutils { From fbe0ae74fba5a22bbaea0cc8e68dbf9327c4ce6d Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 29 Nov 2017 03:57:29 +0200 Subject: [PATCH 41/92] Explicit array initialization removed. --- lib/pure/osproc.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6e0307c8fb..29f0380a36 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -240,8 +240,6 @@ proc execProcesses*(cmds: openArray[string], when defined(windows): var w: WOHandleArray var wcount = m - for c in 0..MAXIMUM_WAIT_OBJECTS - 1: - w[c] = 0 while i < m: if beforeRunEvent != nil: From 416a322efb3ce8a90c88f596863cbba89d28bccb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 28 Nov 2017 20:00:20 +0100 Subject: [PATCH 42/92] added lexer.newlineFollows for parser experiments --- compiler/lexer.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 4106494c4a..bca07e5000 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -860,6 +860,23 @@ proc getOperator(L: var TLexer, tok: var TToken) = if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: tok.strongSpaceB = -1 +proc newlineFollows*(L: var TLexer): bool = + var pos = L.bufpos + var buf = L.buf + while true: + case buf[pos] + of ' ', '\t': + inc(pos) + of CR, LF: + result = true + break + of '#': + inc(pos) + if buf[pos] == '#': inc(pos) + if buf[pos] != '[': return true + else: + break + proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; isDoc: bool) = var pos = start From 33814cf63e9cdf3300c2d14df8f611dcb863dfaa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 29 Nov 2017 13:31:31 +0100 Subject: [PATCH 43/92] language change: change how the experimental dot operators work --- changelog.md | 6 ++++++ compiler/condsyms.nim | 1 + compiler/semcall.nim | 9 ++++----- doc/manual/special_ops.txt | 6 +++--- lib/js/jsffi.nim | 22 +++++++++++----------- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/changelog.md b/changelog.md index f0deceb15d..a37089ef2e 100644 --- a/changelog.md +++ b/changelog.md @@ -111,3 +111,9 @@ This now needs to be written as: - ``strutils.split`` and ``strutils.rsplit`` with an empty string and a separator now returns that empty string. See issue [#4377](https://github.com/nim-lang/Nim/issues/4377). +- The experimental overloading of the dot ``.`` operators now take + an ``untyped``` parameter as the field name, it used to be + a ``static[string]``. You can use ``when defined(nimNewDot)`` to make + your code work with both old and new Nim versions. + See [special-operators](https://nim-lang.org/docs/manual.html#special-operators) + for more information. diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 4879ce5c34..a52214e734 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -111,3 +111,4 @@ proc initDefines*() = defineSymbol("nimNoArrayToCstringConversion") defineSymbol("nimNewRoof") defineSymbol("nimHasRunnableExamples") + defineSymbol("nimNewDot") diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 5c0624a77f..5ea34ab1a3 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -235,12 +235,11 @@ proc resolveOverloads(c: PContext, n, orig: PNode, if nfDotField in n.flags: internalAssert f.kind == nkIdent and n.sonsLen >= 2 - let calleeName = newStrNode(nkStrLit, f.ident.s).withInfo(n.info) # leave the op head symbol empty, # we are going to try multiple variants - n.sons[0..1] = [nil, n[1], calleeName] - orig.sons[0..1] = [nil, orig[1], calleeName] + n.sons[0..1] = [nil, n[1], f] + orig.sons[0..1] = [nil, orig[1], f] template tryOp(x) = let op = newIdentNode(getIdent(x), n.info) @@ -255,8 +254,8 @@ proc resolveOverloads(c: PContext, n, orig: PNode, tryOp "." elif nfDotSetter in n.flags and f.kind == nkIdent and n.len == 3: - let calleeName = newStrNode(nkStrLit, - f.ident.s[0..f.ident.s.len-2]).withInfo(n.info) + # we need to strip away the trailing '=' here: + let calleeName = newIdentNode(getIdent(f.ident.s[0..f.ident.s.len-2]), n.info) let callOp = newIdentNode(getIdent".=", n.info) n.sons[0..1] = [callOp, n[1], calleeName] orig.sons[0..1] = [callOp, orig[1], calleeName] diff --git a/doc/manual/special_ops.txt b/doc/manual/special_ops.txt index 1c7136bec7..93977f81b8 100644 --- a/doc/manual/special_ops.txt +++ b/doc/manual/special_ops.txt @@ -17,8 +17,8 @@ or dynamic file formats such as JSON or XML. When Nim encounters an expression that cannot be resolved by the standard overload resolution rules, the current scope will be searched for a dot operator that can be matched against a re-written form of -the expression, where the unknown field or proc name is converted to -an additional static string parameter: +the expression, where the unknown field or proc name is passed to +an ``untyped`` parameter: .. code-block:: nim a.b # becomes `.`(a, "b") @@ -28,7 +28,7 @@ The matched dot operators can be symbols of any callable kind (procs, templates and macros), depending on the desired effect: .. code-block:: nim - proc `.` (js: PJsonNode, field: string): JSON = js[field] + template `.` (js: PJsonNode, field: untyped): JSON = js[astToStr(field)] var js = parseJson("{ x: 1, y: 2}") echo js.x # outputs 1 diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 13eb1e759e..f34efe9a29 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -177,7 +177,7 @@ proc `==`*(x, y: JsRoot): bool {. importcpp: "(# === #)" .} ## and not strings or numbers, this is a *comparison of references*. {. experimental .} -macro `.`*(obj: JsObject, field: static[cstring]): JsObject = +macro `.`*(obj: JsObject, field: untyped): JsObject = ## Experimental dot accessor (get) for type JsObject. ## Returns the value of a property of name `field` from a JsObject `x`. ## @@ -196,14 +196,14 @@ macro `.`*(obj: JsObject, field: static[cstring]): JsObject = helper(`obj`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] result = quote do: proc helper(o: JsObject): JsObject {. importcpp: `importString`, gensym .} helper(`obj`) -macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = +macro `.=`*(obj: JsObject, field, value: untyped): untyped = ## Experimental dot accessor (set) for type JsObject. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. if validJsName($field): @@ -214,7 +214,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = helper(`obj`, `value`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: JsObject, v: auto) @@ -222,7 +222,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = helper(`obj`, `value`) macro `.()`*(obj: JsObject, - field: static[cstring], + field: untyped, args: varargs[JsObject, jsFromAst]): JsObject = ## Experimental "method call" operator for type JsObject. ## Takes the name of a method of the JavaScript object (`field`) and calls @@ -245,7 +245,7 @@ macro `.()`*(obj: JsObject, importString = "#." & $field & "(@)" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" result = quote: proc helper(o: JsObject): JsObject @@ -257,7 +257,7 @@ macro `.()`*(obj: JsObject, result[1].add args[idx].copyNimTree macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], - field: static[cstring]): V = + field: untyped): V = ## Experimental dot accessor (get) for type JsAssoc. ## Returns the value of a property of name `field` from a JsObject `x`. var importString: string @@ -265,7 +265,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], importString = "#." & $field else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] result = quote do: proc helper(o: type(`obj`)): `obj`.V @@ -273,7 +273,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], helper(`obj`) macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], - field: static[cstring], + field: untyped, value: V): untyped = ## Experimental dot accessor (set) for type JsAssoc. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. @@ -282,7 +282,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], importString = "#." & $field & " = #" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: type(`obj`), v: `obj`.V) @@ -290,7 +290,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], helper(`obj`, `value`) macro `.()`*[K: string | cstring, V: proc](obj: JsAssoc[K, V], - field: static[cstring], + field: untyped, args: varargs[untyped]): auto = ## Experimental "method call" operator for type JsAssoc. ## Takes the name of a method of the JavaScript object (`field`) and calls From fcad56c804127cdf8f672a7d0810ecc17d0a0e75 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 29 Nov 2017 14:52:50 +0100 Subject: [PATCH 44/92] make tests green again --- tests/specialops/tdotops.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/specialops/tdotops.nim b/tests/specialops/tdotops.nim index bca9499225..20066a4969 100644 --- a/tests/specialops/tdotops.nim +++ b/tests/specialops/tdotops.nim @@ -23,16 +23,16 @@ type T2 = object x: int -proc `.`*(v: T1, f: string): int = - echo "reading field ", f - return v.x +template `.`*(v: T1, f: untyped): int = + echo "reading field ", astToStr(f) + v.x -proc `.=`(x: var T1, f: string{lit}, v: int) = - echo "assigning ", f, " = ", v - x.x = v +template `.=`(t: var T1, f: untyped, v: int) = + echo "assigning ", astToStr(f), " = ", v + t.x = v -template `.()`(x: T1, f: string, args: varargs[typed]): string = - echo "call to ", f +template `.()`(x: T1, f: untyped, args: varargs[typed]): string = + echo "call to ", astToStr(f) "dot call" echo "" @@ -47,13 +47,13 @@ echo t.y() var d = TD(t) assert(not compiles(d.y)) -proc `.`(v: T2, f: string): int = - echo "no params call to ", f - return v.x +template `.`(v: T2, f: untyped): int = + echo "no params call to ", astToStr(f) + v.x -proc `.`*(v: T2, f: string, a: int): int = - echo "one param call to ", f, " with ", a - return v.x +template `.`*(v: T2, f: untyped, a: int): int = + echo "one param call to ", astToStr(f), " with ", a + v.x var tt = T2(x: 100) From 216119212c256548a3e7557d5190fe61a7b9524a Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 29 Nov 2017 08:07:16 -0800 Subject: [PATCH 45/92] Emit relative object file paths in genScript (#6835) --- compiler/extccomp.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e6b23aae52..42c341651b 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -764,8 +764,9 @@ proc callCCompiler*(projectfile: string) = add(objfiles, quoteShell( addFileExt(objFile, CC[cCompiler].objExt))) for x in toCompile: + let objFile = if noAbsolutePaths(): x.obj.extractFilename else: x.obj add(objfiles, ' ') - add(objfiles, quoteShell(x.obj)) + add(objfiles, quoteShell(objFile)) linkCmd = getLinkCmd(projectfile, objfiles) if optCompileOnly notin gGlobalOptions: From 6e9a98d1e92cabd25b9d1d8dd87bef41216e062f Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 30 Nov 2017 12:06:44 +0100 Subject: [PATCH 46/92] minor code cleanup: remove redundant .final markers --- compiler/cgendata.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 19ab2fe50a..0f8fa760e2 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -54,7 +54,7 @@ type TCProcSections* = array[TCProcSection, Rope] # represents a generated C proc BModule* = ref TCGen BProc* = ref TCProc - TBlock*{.final.} = object + TBlock* = object id*: int # the ID of the label; positive means that it label*: Rope # generated text for the label # nil if label is not used @@ -64,7 +64,7 @@ type nestedExceptStmts*: int16 # how many except statements is it nested into frameLen*: int16 - TCProc{.final.} = object # represents C proc that is currently generated + TCProc = object # represents C proc that is currently generated prc*: PSym # the Nim proc that this C proc belongs to beforeRetNeeded*: bool # true iff 'BeforeRet' label for proc is needed threadVarAccessed*: bool # true if the proc already accessed some threadvar From 34f07d10f2bef8e4ad8b52b1e715abae7b6bad5b Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 30 Nov 2017 12:09:19 +0100 Subject: [PATCH 47/92] renderer.nim: support for outputting symbol magics for debugging --- compiler/renderer.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 03267c53ef..6f80afefa7 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -826,7 +826,10 @@ proc gident(g: var TSrcGen, n: PNode) = t = tkOpr put(g, t, s) if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags): - put(g, tkIntLit, $n.sym.id) + when defined(debugMagics): + put(g, tkIntLit, $n.sym.id & $n.sym.magic) + else: + put(g, tkIntLit, $n.sym.id) proc doParamsAux(g: var TSrcGen, params: PNode) = if params.len > 1: From 49870579ccd089870f9c47995ca7794a6351261c Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 30 Nov 2017 12:11:38 +0100 Subject: [PATCH 48/92] ccgexprs.nim: added support for 'debugMagics' --- compiler/ccgexprs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 571135fbb5..5f107f21f9 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1860,7 +1860,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = initLocExpr(p, e.sons[2], b) genDeepCopy(p, a, b) of mDotDot, mEqCString: genCall(p, e, d) - else: internalError(e.info, "genMagicExpr: " & $op) + else: + when defined(debugMagics): + echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind + internalError(e.info, "genMagicExpr: " & $op) proc genSetConstr(p: BProc, e: PNode, d: var TLoc) = # example: { a..b, c, d, e, f..g } From 34ac04f70509e9f9c3d28b1bb12e3d6e6ed3a947 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 30 Nov 2017 12:55:28 +0100 Subject: [PATCH 49/92] improve the error messages when overloaded '.' operators are involved --- compiler/semcall.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 5ea34ab1a3..a77fcad974 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -179,7 +179,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = add(result, ')') if candidates != "": add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates) - localError(n.info, errGenerated, result) + localError(n.info, errGenerated, result & "\nexpression: " & $n) proc bracketNotFoundError(c: PContext; n: PNode) = var errors: CandidateErrors = @[] @@ -238,6 +238,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode, # leave the op head symbol empty, # we are going to try multiple variants + errors = nil n.sons[0..1] = [nil, n[1], f] orig.sons[0..1] = [nil, orig[1], f] @@ -254,6 +255,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode, tryOp "." elif nfDotSetter in n.flags and f.kind == nkIdent and n.len == 3: + errors = nil # we need to strip away the trailing '=' here: let calleeName = newIdentNode(getIdent(f.ident.s[0..f.ident.s.len-2]), n.info) let callOp = newIdentNode(getIdent".=", n.info) From 255902f9a5a9f92ce2d65996a43626eff4c3b52c Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 30 Nov 2017 15:24:30 +0100 Subject: [PATCH 50/92] added macros.unpackVarargs --- changelog.md | 1 + lib/core/macros.nim | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changelog.md b/changelog.md index a37089ef2e..f061805b14 100644 --- a/changelog.md +++ b/changelog.md @@ -117,3 +117,4 @@ This now needs to be written as: your code work with both old and new Nim versions. See [special-operators](https://nim-lang.org/docs/manual.html#special-operators) for more information. +- Added ``macros.unpackVarargs``. diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ebc9f77142..ee6c1a09ff 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1226,3 +1226,8 @@ when not defined(booting): macro payload: untyped {.gensym.} = result = parseStmt(e) payload() + +macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = + result = newCall(callee) + for i in 0 ..< args.len: + result.add args[i] From 11fcae57052b3c886b4d2b593acb3ac0d717edd1 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 28 Nov 2017 21:49:34 +0000 Subject: [PATCH 51/92] Fixes #5856. Code based on @loloiccl's PR (#5879). --- lib/pure/json.nim | 28 +++++++++++++++++++++++----- tests/stdlib/tjsonmacro.nim | 15 ++++++++++++++- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index cea485c439..1b887a0c54 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1524,6 +1524,27 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = doAssert result.len > 0 +proc processObjFields(obj: NimNode, + jsonNode: NimNode): seq[NimNode] {.compileTime.} = + ## Process all the fields of an ``ObjectTy`` and any of its + ## parent type's fields (via inheritance). + result = @[] + + expectKind(obj[2], nnkRecList) + for field in obj[2]: + let nodes = processObjField(field, jsonNode) + result.add(nodes) + + # process parent type fields + case obj[1].kind + of nnkBracketExpr: + assert $obj[1][0] == "ref" + result.add(processObjFields(getType(obj[1][1]), jsonNode)) + of nnkSym: + result.add(processObjFields(getType(obj[1]), jsonNode)) + else: + discard + proc processType(typeName: NimNode, obj: NimNode, jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} = ## Process a type such as ``Sym "float"`` or ``ObjectTy ...``. @@ -1533,7 +1554,7 @@ proc processType(typeName: NimNode, obj: NimNode, ## .. code-block::plain ## ObjectTy ## Empty - ## Empty + ## InheritanceInformation ## RecList ## Sym "events" case obj.kind @@ -1543,10 +1564,7 @@ proc processType(typeName: NimNode, obj: NimNode, result.add(typeName) # Name of the type to construct. # Process each object field and add it as an exprColonExpr - expectKind(obj[2], nnkRecList) - for field in obj[2]: - let nodes = processObjField(field, jsonNode) - result.add(nodes) + result.add(processObjFields(obj, jsonNode)) # Object might be null. So we need to check for that. if isRef: diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 153cf85563..2d20063ab3 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -246,4 +246,17 @@ when isMainModule: var b = Bird(age: 3, height: 1.734, name: "bardo", colors: [red, blue]) let jnode = %b let data = jnode.to(Bird) - doAssert data == b \ No newline at end of file + doAssert data == b + + block: + type + MsgBase = ref object of RootObj + name*: string + + MsgChallenge = ref object of MsgBase + challenge*: string + + let data = %*{"name": "foo", "challenge": "bar"} + let msg = data.to(MsgChallenge) + doAssert msg.name == "foo" + doAssert msg.challenge == "bar" \ No newline at end of file From e0681715dc9c54135937f54510015f66d384aa29 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 29 Nov 2017 14:48:17 +0000 Subject: [PATCH 52/92] Fixes #6095. --- lib/pure/json.nim | 55 +++++++++++++++++++++++-------------- tests/stdlib/tjsonmacro.nim | 18 +++++++++++- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 1b887a0c54..b9f49f0bdc 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1524,26 +1524,34 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = doAssert result.len > 0 -proc processObjFields(obj: NimNode, - jsonNode: NimNode): seq[NimNode] {.compileTime.} = +proc processFields(obj: NimNode, + jsonNode: NimNode): seq[NimNode] {.compileTime.} = ## Process all the fields of an ``ObjectTy`` and any of its ## parent type's fields (via inheritance). result = @[] + case obj.kind + of nnkObjectTy: + expectKind(obj[2], nnkRecList) + for field in obj[2]: + let nodes = processObjField(field, jsonNode) + result.add(nodes) - expectKind(obj[2], nnkRecList) - for field in obj[2]: - let nodes = processObjField(field, jsonNode) - result.add(nodes) - - # process parent type fields - case obj[1].kind - of nnkBracketExpr: - assert $obj[1][0] == "ref" - result.add(processObjFields(getType(obj[1][1]), jsonNode)) - of nnkSym: - result.add(processObjFields(getType(obj[1]), jsonNode)) + # process parent type fields + case obj[1].kind + of nnkBracketExpr: + assert $obj[1][0] == "ref" + result.add(processFields(getType(obj[1][1]), jsonNode)) + of nnkSym: + result.add(processFields(getType(obj[1]), jsonNode)) + else: + discard + of nnkTupleTy: + for identDefs in obj: + expectKind(identDefs, nnkIdentDefs) + let nodes = processObjField(identDefs[0], jsonNode) + result.add(nodes) else: - discard + doAssert false, "Unable to process field type: " & $obj.kind proc processType(typeName: NimNode, obj: NimNode, jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} = @@ -1558,13 +1566,17 @@ proc processType(typeName: NimNode, obj: NimNode, ## RecList ## Sym "events" case obj.kind - of nnkObjectTy: + of nnkObjectTy, nnkTupleTy: # Create object constructor. - result = newNimNode(nnkObjConstr) - result.add(typeName) # Name of the type to construct. + result = + if obj.kind == nnkObjectTy: newNimNode(nnkObjConstr) + else: newNimNode(nnkPar) - # Process each object field and add it as an exprColonExpr - result.add(processObjFields(obj, jsonNode)) + if obj.kind == nnkObjectTy: + result.add(typeName) # Name of the type to construct. + + # Process each object/tuple field and add it as an exprColonExpr + result.add(processFields(obj, jsonNode)) # Object might be null. So we need to check for that. if isRef: @@ -1687,6 +1699,8 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = result = createConstructor(obj, jsonNode) else: result = processType(typeSym, obj, jsonNode, false) + of nnkTupleTy: + result = processType(typeSym, typeSym, jsonNode, false) else: doAssert false, "Unable to create constructor for: " & $typeSym.kind @@ -1818,6 +1832,7 @@ macro to*(node: JsonNode, T: typedesc): untyped = # TODO: Rename postProcessValue and move it (?) result = postProcessValue(result) + # echo(treeRepr(result)) # echo(toStrLit(result)) when false: diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 2d20063ab3..388d4d534e 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -259,4 +259,20 @@ when isMainModule: let data = %*{"name": "foo", "challenge": "bar"} let msg = data.to(MsgChallenge) doAssert msg.name == "foo" - doAssert msg.challenge == "bar" \ No newline at end of file + doAssert msg.challenge == "bar" + + block: + type + Color = enum Red, Brown + Thing = object + animal: tuple[fur: bool, legs: int] + color: Color + + var j = parseJson(""" + {"animal":{"fur":true,"legs":6},"color":"Red"} + """) + + let parsed = to(j, Thing) + doAssert parsed.animal.fur + doAssert parsed.animal.legs == 6 + doAssert parsed.color == Red \ No newline at end of file From d3c9b58c005e7cd537cbdf3dfd3f69e72fa40722 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 29 Nov 2017 15:56:46 +0000 Subject: [PATCH 53/92] Fixes #6604. Rejects unnamed tuples with error. --- lib/pure/json.nim | 15 +++++++++++++-- tests/stdlib/tjsonmacro.nim | 22 +++++++++++++++++++++- tests/stdlib/tjsonmacro_reject.nim | 18 ++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/stdlib/tjsonmacro_reject.nim diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b9f49f0bdc..90cf7b8c96 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1701,6 +1701,10 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = result = processType(typeSym, obj, jsonNode, false) of nnkTupleTy: result = processType(typeSym, typeSym, jsonNode, false) + of nnkPar: + # TODO: The fact that `jsonNode` here works to give a good line number + # is weird. Specifying typeSym should work but doesn't. + error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode) else: doAssert false, "Unable to create constructor for: " & $typeSym.kind @@ -1828,9 +1832,16 @@ macro to*(node: JsonNode, T: typedesc): untyped = expectKind(typeNode, nnkBracketExpr) doAssert(($typeNode[0]).normalize == "typedesc") - result = createConstructor(typeNode[1], node) + # Create `temp` variable to store the result in case the user calls this + # on `parseJson` (see bug #6604). + result = newNimNode(nnkStmtListExpr) + let temp = genSym(nskLet, "temp") + result.add quote do: + let `temp` = `node` + + let constructor = createConstructor(typeNode[1], temp) # TODO: Rename postProcessValue and move it (?) - result = postProcessValue(result) + result.add(postProcessValue(constructor)) # echo(treeRepr(result)) # echo(toStrLit(result)) diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 388d4d534e..af9633e9ef 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -275,4 +275,24 @@ when isMainModule: let parsed = to(j, Thing) doAssert parsed.animal.fur doAssert parsed.animal.legs == 6 - doAssert parsed.color == Red \ No newline at end of file + doAssert parsed.color == Red + + block: + type + Car = object + engine: tuple[name: string, capacity: float] + model: string + + let j = """ + {"engine": {"name": "V8", "capacity": 5.5}, "model": "Skyline"} + """ + + var i = 0 + proc mulTest: JsonNode = + i.inc() + return parseJson(j) + + let parsed = mulTest().to(Car) + doAssert parsed.engine.name == "V8" + + doAssert i == 1 \ No newline at end of file diff --git a/tests/stdlib/tjsonmacro_reject.nim b/tests/stdlib/tjsonmacro_reject.nim new file mode 100644 index 0000000000..00506449f5 --- /dev/null +++ b/tests/stdlib/tjsonmacro_reject.nim @@ -0,0 +1,18 @@ +discard """ + file: "tjsonmacro_reject.nim" + line: 11 + errormsg: "Use a named tuple instead of: (string, float)" +""" + +import json + +type + Car = object + engine: (string, float) + model: string + +let j = """ + {"engine": {"name": "V8", "capacity": 5.5}, model: "Skyline"} +""" +let parsed = parseJson(j) +echo(to(parsed, Car)) \ No newline at end of file From 8ca41ce637106c734cd819a1f49606db880cf075 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 29 Nov 2017 19:15:25 +0000 Subject: [PATCH 54/92] Implement support for Option[T] in json.to macro. Fixes #5848. --- lib/pure/json.nim | 27 +++++++++++++++++++++++++++ tests/stdlib/tjsonmacro.nim | 25 +++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 90cf7b8c96..b057aa7c6e 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1346,6 +1346,16 @@ proc createJsonIndexer(jsonNode: NimNode, indexNode ) +proc transformJsonIndexer(jsonNode: NimNode): NimNode = + case jsonNode.kind + of nnkBracketExpr: + result = newNimNode(nnkCurlyExpr) + else: + result = jsonNode.copy() + + for child in jsonNode: + result.add(transformJsonIndexer(child)) + template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ast: string) = if node.kind notin kinds: @@ -1637,6 +1647,10 @@ proc processType(typeName: NimNode, obj: NimNode, doAssert(not result.isNil(), "processType not initialised.") +import options +proc workaroundMacroNone[T](): Option[T] = + none(T) + proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc. ## @@ -1650,6 +1664,19 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = of nnkBracketExpr: var bracketName = ($typeSym[0]).normalize case bracketName + of "option": + # TODO: Would be good to verify that this is Option[T] from + # options module I suppose. + let lenientJsonNode = transformJsonIndexer(jsonNode) + + let optionGeneric = typeSym[1] + let value = createConstructor(typeSym[1], jsonNode) + let workaround = bindSym("workaroundMacroNone") # TODO: Nim Bug: This shouldn't be necessary. + + result = quote do: + ( + if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) + ) of "ref": # Ref type. var typeName = $typeSym[1] diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index af9633e9ef..f9f94606cc 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -2,7 +2,7 @@ discard """ file: "tjsonmacro.nim" output: "" """ -import json, strutils +import json, strutils, options when isMainModule: # Tests inspired by own use case (with some additional tests). @@ -295,4 +295,25 @@ when isMainModule: let parsed = mulTest().to(Car) doAssert parsed.engine.name == "V8" - doAssert i == 1 \ No newline at end of file + doAssert i == 1 + + block: + # Option[T] support! + type + Car1 = object # TODO: Codegen bug when `Car` + engine: tuple[name: string, capacity: Option[float]] + model: string + year: Option[int] + + let noYear = """ + {"engine": {"name": "V8", "capacity": 5.5}, "model": "Skyline"} + """ + + let noYearParsed = parseJson(noYear) + let noYearDeser = to(noYearParsed, Car1) + doAssert noYearDeser.engine.capacity == some(5.5) + doAssert noYearDeser.year.isNone + doAssert noYearDeser.engine.name == "V8" + + # TODO: Table[T, Y] support. + # TODO: JsonNode support \ No newline at end of file From 8187e83645bbc9d536eebfab2af3b2437c3485fb Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 29 Nov 2017 20:30:40 +0000 Subject: [PATCH 55/92] Implement Table/OrderedTable support for json.to macro. --- lib/pure/json.nim | 24 ++++++++++++++++++++++++ tests/stdlib/tjsonmacro.nim | 27 +++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b057aa7c6e..6153e2f03a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1677,6 +1677,30 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ( if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) ) + of "table", "orderedtable": + let tableKeyType = typeSym[1] + if ($tableKeyType).cmpIgnoreStyle("string") != 0: + error("JSON doesn't support keys of type " & $tableKeyType) + let tableValueType = typeSym[2] + + let forLoopKey = genSym(nskForVar, "key") + let indexerNode = createJsonIndexer(jsonNode, forLoopKey) + let constructorNode = createConstructor(tableValueType, indexerNode) + + let tableInit = + if bracketName == "table": + bindSym("initTable") + else: + bindSym("initOrderedTable") + + # Create a statement expression containing a for loop. + result = quote do: + ( + var map = `tableInit`[`tableKeyType`, `tableValueType`](); + verifyJsonKind(`jsonNode`, {JObject}, astToStr(`jsonNode`)); + for `forLoopKey` in keys(`jsonNode`.fields): map[`forLoopKey`] = `constructorNode`; + map + ) of "ref": # Ref type. var typeName = $typeSym[1] diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index f9f94606cc..01fa43aa7e 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -2,7 +2,7 @@ discard """ file: "tjsonmacro.nim" output: "" """ -import json, strutils, options +import json, strutils, options, tables when isMainModule: # Tests inspired by own use case (with some additional tests). @@ -315,5 +315,28 @@ when isMainModule: doAssert noYearDeser.year.isNone doAssert noYearDeser.engine.name == "V8" - # TODO: Table[T, Y] support. + # Table[T, Y] support. + block: + type + Friend = object + name: string + age: int + + Dynamic = object + name: string + friends: Table[string, Friend] + + let data = """ + {"friends": { + "John": {"name": "John", "age": 35}, + "Elizabeth": {"name": "Elizabeth", "age": 23} + }, "name": "Dominik"} + """ + + let dataParsed = parseJson(data) + let dataDeser = to(dataParsed, Dynamic) + doAssert dataDeser.name == "Dominik" + doAssert dataDeser.friends["John"].age == 35 + doAssert dataDeser.friends["Elizabeth"].age == 23 + # TODO: JsonNode support \ No newline at end of file From 8d6126237226a80ca4c78206c625009ce285c348 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 29 Nov 2017 20:47:56 +0000 Subject: [PATCH 56/92] Implement support for JsonNode in json.to. --- lib/pure/json.nim | 5 +++++ tests/stdlib/tjsonmacro.nim | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 6153e2f03a..1d2f480c4b 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1744,6 +1744,11 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = let obj = getType(typeSym) result = processType(typeSym, obj, jsonNode, false) of nnkSym: + # Handle JsonNode. + if ($typeSym).cmpIgnoreStyle("jsonnode") == 0: + return jsonNode + + # Handle all other types. let obj = getType(typeSym) if obj.kind == nnkBracketExpr: # When `Sym "Foo"` turns out to be a `ref object`. diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 01fa43aa7e..2baa7bed1f 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -339,4 +339,21 @@ when isMainModule: doAssert dataDeser.friends["John"].age == 35 doAssert dataDeser.friends["Elizabeth"].age == 23 - # TODO: JsonNode support \ No newline at end of file + # JsonNode support + block: + type + Test = object + name: string + fallback: JsonNode + + let data = """ + {"name": "FooBar", "fallback": 56.42} + """ + + let dataParsed = parseJson(data) + let dataDeser = to(dataParsed, Test) + doAssert dataDeser.name == "FooBar" + doAssert dataDeser.fallback.kind == JFloat + doAssert dataDeser.fallback.getFloat() == 56.42 + + # TODO: Cycles lead to infinite loops. \ No newline at end of file From 2bb2e6975e397bef1b320cd5dbafb6b3338fdaf0 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 30 Nov 2017 18:43:34 +0000 Subject: [PATCH 57/92] Fix infinite recursion when using json.to on ref with cycle. --- lib/pure/json.nim | 10 ++++++++++ tests/stdlib/tjsonmacro.nim | 4 +--- tests/stdlib/tjsonmacro_reject2.nim | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/stdlib/tjsonmacro_reject2.nim diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 1d2f480c4b..9e7510e45e 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1651,6 +1651,13 @@ import options proc workaroundMacroNone[T](): Option[T] = none(T) +proc depth(n: NimNode, current = 0): int = + result = 1 + for child in n: + let d = 1 + child.depth(current + 1) + if d > result: + result = d + proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc. ## @@ -1660,6 +1667,9 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = # echo("--createConsuctor-- \n", treeRepr(typeSym)) # echo() + if depth(jsonNode) > 150: + error("The `to` macro does not support ref objects with cycles.", jsonNode) + case typeSym.kind of nnkBracketExpr: var bracketName = ($typeSym[0]).normalize diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 2baa7bed1f..e2d8c27cfc 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -354,6 +354,4 @@ when isMainModule: let dataDeser = to(dataParsed, Test) doAssert dataDeser.name == "FooBar" doAssert dataDeser.fallback.kind == JFloat - doAssert dataDeser.fallback.getFloat() == 56.42 - - # TODO: Cycles lead to infinite loops. \ No newline at end of file + doAssert dataDeser.fallback.getFloat() == 56.42 \ No newline at end of file diff --git a/tests/stdlib/tjsonmacro_reject2.nim b/tests/stdlib/tjsonmacro_reject2.nim new file mode 100644 index 0000000000..b01153553c --- /dev/null +++ b/tests/stdlib/tjsonmacro_reject2.nim @@ -0,0 +1,21 @@ +discard """ + file: "tjsonmacro_reject2.nim" + line: 10 + errormsg: "The `to` macro does not support ref objects with cycles." +""" +import json + +type + Misdirection = object + cycle: Cycle + + Cycle = ref object + foo: string + cycle: Misdirection + +let data = """ + {"cycle": null} +""" + +let dataParsed = parseJson(data) +let dataDeser = to(dataParsed, Cycle) \ No newline at end of file From 578ab935cbb1a9b53c0192d389c1a01c4e6e95ac Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 30 Nov 2017 18:56:34 +0000 Subject: [PATCH 58/92] Support all int, uint and float variants in json.to macro. --- lib/pure/json.nim | 30 ++++++++++++++++-------------- tests/stdlib/tjsonmacro.nim | 28 +++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 9e7510e45e..b5b84863ac 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1609,25 +1609,14 @@ proc processType(typeName: NimNode, obj: NimNode, `getEnumCall` ) of nnkSym: - case ($typeName).normalize - of "float": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JFloat, JInt}, astToStr(`jsonNode`)); - if `jsonNode`.kind == JFloat: `jsonNode`.fnum else: `jsonNode`.num.float - ) + let name = ($typeName).normalize + case name of "string": result = quote do: ( verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`)); if `jsonNode`.kind == JNull: nil else: `jsonNode`.str ) - of "int": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); - `jsonNode`.num.int - ) of "biggestint": result = quote do: ( @@ -1641,7 +1630,20 @@ proc processType(typeName: NimNode, obj: NimNode, `jsonNode`.bval ) else: - doAssert false, "Unable to process nnkSym " & $typeName + if name.startsWith("int") or name.startsWith("uint"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); + `jsonNode`.num.`obj` + ) + elif name.startsWith("float"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt, JFloat}, astToStr(`jsonNode`)); + if `jsonNode`.kind == JFloat: `jsonNode`.fnum.`obj` else: `jsonNode`.num.`obj` + ) + else: + doAssert false, "Unable to process nnkSym " & $typeName else: doAssert false, "Unable to process type: " & $obj.kind diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index e2d8c27cfc..2cdd823058 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -354,4 +354,30 @@ when isMainModule: let dataDeser = to(dataParsed, Test) doAssert dataDeser.name == "FooBar" doAssert dataDeser.fallback.kind == JFloat - doAssert dataDeser.fallback.getFloat() == 56.42 \ No newline at end of file + doAssert dataDeser.fallback.getFloat() == 56.42 + + # int64, float64 etc support. + block: + type + Test1 = object + a: int8 + b: int16 + c: int32 + d: int64 + e: uint8 + f: uint16 + g: uint32 + h: uint64 + i: float32 + j: float64 + + let data = """ + {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, + "h": 8, "i": 9.9, "j": 10.10} + """ + + let dataParsed = parseJson(data) + let dataDeser = to(dataParsed, Test1) + doAssert dataDeser.a == 1 + doAssert dataDeser.f == 6 + doAssert dataDeser.i == 9.9'f32 \ No newline at end of file From fa92c519aa6ac04f10655b1ab6992701549d4aed Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 1 Dec 2017 01:52:00 +0100 Subject: [PATCH 59/92] more progress on destructors; removed old destructor based code as it proved confusing --- compiler/destroyer.nim | 38 +++--- compiler/semasgn.nim | 14 ++- compiler/semdestruct.nim | 185 ------------------------------ compiler/semexprs.nim | 3 +- compiler/semstmts.nim | 40 ++----- tests/destructor/tatomicptrs.nim | 101 ++++++++++++++++ tests/destructor/tdestructor.nim | 6 +- tests/destructor/tdestructor2.nim | 27 ----- tests/destructor/tdestructor3.nim | 8 +- 9 files changed, 152 insertions(+), 270 deletions(-) delete mode 100644 compiler/semdestruct.nim create mode 100644 tests/destructor/tatomicptrs.nim delete mode 100644 tests/destructor/tdestructor2.nim diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 729480f81b..caa18af92f 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -167,10 +167,13 @@ template interestingSym(s: PSym): bool = proc patchHead(n: PNode) = if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1: let s = n[0].sym - if sfFromGeneric in s.flags and s.name.s[0] == '=' and - s.name.s in ["=sink", "=", "=destroy"]: - excl(s.flags, sfFromGeneric) - patchHead(s.getBody) + if s.name.s[0] == '=' and s.name.s in ["=sink", "=", "=destroy"]: + if sfFromGeneric in s.flags: + excl(s.flags, sfFromGeneric) + patchHead(s.getBody) + if n[1].typ.isNil: + # XXX toptree crashes without this workaround. Figure out why. + return let t = n[1].typ.skipTypes({tyVar, tyGenericInst, tyAlias, tyInferred}) template patch(op, field) = if s.name.s == op and field != nil and field != s: @@ -181,24 +184,30 @@ proc patchHead(n: PNode) = for x in n: patchHead(x) +proc patchHead(s: PSym) = + if sfFromGeneric in s.flags: + patchHead(s.ast[bodyPos]) + +template genOp(opr, opname) = + let op = opr + if op == nil: + globalError(dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t)) + elif op.ast[genericParamsPos].kind != nkEmpty: + globalError(dest.info, "internal error: '" & opname & "' operator is generic") + patchHead op + result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest)) + proc genSink(t: PType; dest: PNode): PNode = let t = t.skipTypes({tyGenericInst, tyAlias}) - let op = if t.sink != nil: t.sink else: t.assignment - assert op != nil - patchHead op.ast[bodyPos] - result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest)) + genOp(if t.sink != nil: t.sink else: t.assignment, "=sink") proc genCopy(t: PType; dest: PNode): PNode = let t = t.skipTypes({tyGenericInst, tyAlias}) - assert t.assignment != nil - patchHead t.assignment.ast[bodyPos] - result = newTree(nkCall, newSymNode(t.assignment), newTree(nkHiddenAddr, dest)) + genOp(t.assignment, "=") proc genDestroy(t: PType; dest: PNode): PNode = let t = t.skipTypes({tyGenericInst, tyAlias}) - assert t.destructor != nil - patchHead t.destructor.ast[bodyPos] - result = newTree(nkCall, newSymNode(t.destructor), newTree(nkHiddenAddr, dest)) + genOp(t.destructor, "=destroy") proc addTopVar(c: var Con; v: PNode) = c.topLevelVars.add newTree(nkIdentDefs, v, emptyNode, emptyNode) @@ -287,6 +296,7 @@ proc p(n: PNode; c: var Con): PNode = recurse(n, result) proc injectDestructorCalls*(owner: PSym; n: PNode): PNode = + echo "injecting into ", n var c: Con c.owner = owner c.tmp = newSym(skTemp, getIdent":d", owner, n.info) diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index cad5087082..db08605cfb 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -7,8 +7,8 @@ # distribution, for details about the copyright. # -## This module implements lifting for assignments. Later versions of this code -## will be able to also lift ``=deepCopy`` and ``=destroy``. +## This module implements lifting for type-bound operations +## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``). # included from sem.nim @@ -302,6 +302,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp; n.sons[paramsPos] = result.typ.n n.sons[bodyPos] = body result.ast = n + incl result.flags, sfFromGeneric proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym = @@ -319,8 +320,10 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) = ## to ensure we lift assignment, destructors and moves properly. ## The later 'destroyer' pass depends on it. if not newDestructors or not hasDestructor(typ): return - # do not produce wrong liftings while we're still instantiating generics: - if c.typesWithOps.len > 0: return + when false: + # do not produce wrong liftings while we're still instantiating generics: + # now disabled; breaks topttree.nim! + if c.typesWithOps.len > 0: return let typ = typ.skipTypes({tyGenericInst, tyAlias}) # we generate the destructor first so that other operators can depend on it: if typ.destructor == nil: @@ -329,3 +332,6 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) = liftBody(c, typ, attachedAsgn, info) if typ.sink == nil: liftBody(c, typ, attachedSink, info) + +#proc patchResolvedTypeBoundOp*(c: PContext; n: PNode): PNode = +# if n.kind == nkCall and diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim deleted file mode 100644 index b16bf004ff..0000000000 --- a/compiler/semdestruct.nim +++ /dev/null @@ -1,185 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements destructors. - -# included from sem.nim - -# special marker values that indicates that we are -# 1) AnalyzingDestructor: currently analyzing the type for destructor -# generation (needed for recursive types) -# 2) DestructorIsTrivial: completed the analysis before and determined -# that the type has a trivial destructor -var analyzingDestructor, destructorIsTrivial: PSym -new(analyzingDestructor) -new(destructorIsTrivial) - -var - destructorName = getIdent"destroy_" - destructorParam = getIdent"this_" - -proc instantiateDestructor(c: PContext, typ: PType): PType - -proc doDestructorStuff(c: PContext, s: PSym, n: PNode) = - var t = s.typ.sons[1].skipTypes({tyVar}) - if t.kind == tyGenericInvocation: - for i in 1 ..< t.sonsLen: - if t.sons[i].kind != tyGenericParam: - localError(n.info, errDestructorNotGenericEnough) - return - t = t.base - elif t.kind == tyCompositeTypeClass: - t = t.base - if t.kind != tyGenericBody: - localError(n.info, errDestructorNotGenericEnough) - return - - t.destructor = s - # automatically insert calls to base classes' destructors - if n.sons[bodyPos].kind != nkEmpty: - for i in countup(0, t.sonsLen - 1): - # when inheriting directly from object - # there will be a single nil son - if t.sons[i] == nil: continue - let destructableT = instantiateDestructor(c, t.sons[i]) - if destructableT != nil: - n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[ - useSym(destructableT.destructor, c.graph.usageSym), - n.sons[paramsPos][1][0]])) - -proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode - -proc destroySym(c: PContext, field: PSym, holder: PNode): PNode = - let destructableT = instantiateDestructor(c, field.typ) - if destructableT != nil: - result = newNode(nkCall, field.info, @[ - useSym(destructableT.destructor, c.graph.usageSym), - newNode(nkDotExpr, field.info, @[holder, useSym(field, c.graph.usageSym)])]) - -proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode = - var nonTrivialFields = 0 - result = newNode(nkCaseStmt, n.info, @[]) - # case x.kind - result.addSon(newNode(nkDotExpr, n.info, @[holder, n.sons[0]])) - for i in countup(1, n.len - 1): - # of A, B: - let ni = n[i] - var caseBranch = newNode(ni.kind, ni.info, ni.sons[0..ni.len-2]) - - let stmt = destroyFieldOrFields(c, ni.lastSon, holder) - if stmt == nil: - caseBranch.addSon(newNode(nkStmtList, ni.info, @[])) - else: - caseBranch.addSon(stmt) - nonTrivialFields += stmt.len - - result.addSon(caseBranch) - - # maybe no fields were destroyed? - if nonTrivialFields == 0: - result = nil - -proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode = - template maybeAddLine(e) = - let stmt = e - if stmt != nil: - if result == nil: result = newNode(nkStmtList) - result.addSon(stmt) - - case field.kind - of nkRecCase: - maybeAddLine destroyCase(c, field, holder) - of nkSym: - maybeAddLine destroySym(c, field.sym, holder) - of nkRecList: - for son in field: - maybeAddLine destroyFieldOrFields(c, son, holder) - else: - internalAssert false - -proc generateDestructor(c: PContext, t: PType): PNode = - ## generate a destructor for a user-defined object or tuple type - ## returns nil if the destructor turns out to be trivial - - # XXX: This may be true for some C-imported types such as - # Tposix_spawnattr - if t.n == nil or t.n.sons == nil: return - internalAssert t.n.kind == nkRecList - let destructedObj = newIdentNode(destructorParam, unknownLineInfo()) - # call the destructods of all fields - result = destroyFieldOrFields(c, t.n, destructedObj) - # base classes' destructors will be automatically called by - # semProcAux for both auto-generated and user-defined destructors - -proc instantiateDestructor(c: PContext, typ: PType): PType = - # returns nil if a variable of type `typ` doesn't require a - # destructor. Otherwise, returns the type, which holds the - # destructor that must be used for the varialbe. - # The destructor is either user-defined or automatically - # generated by the compiler in a member-wise fashion. - var t = typ.skipGenericAlias - let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t - - if typeHoldingUserDefinition.destructor != nil: - # XXX: This is not entirely correct for recursive types, but we need - # it temporarily to hide the "destroy is already defined" problem - if typeHoldingUserDefinition.destructor notin - [analyzingDestructor, destructorIsTrivial]: - return typeHoldingUserDefinition - else: - return nil - - t = t.skipTypes({tyGenericInst, tyAlias}) - case t.kind - of tySequence, tyArray, tyOpenArray, tyVarargs: - t.destructor = analyzingDestructor - if instantiateDestructor(c, t.sons[0]) != nil: - t.destructor = getCompilerProc"nimDestroyRange" - return t - else: - return nil - of tyTuple, tyObject: - t.destructor = analyzingDestructor - let generated = generateDestructor(c, t) - if generated != nil: - internalAssert t.sym != nil - let info = t.sym.info - let fullDef = newNode(nkProcDef, info, @[ - newIdentNode(destructorName, info), - emptyNode, - emptyNode, - newNode(nkFormalParams, info, @[ - emptyNode, - newNode(nkIdentDefs, info, @[ - newIdentNode(destructorParam, info), - symNodeFromType(c, makeVarType(c, t), t.sym.info), - emptyNode]), - ]), - emptyNode, - emptyNode, - generated - ]) - let semantizedDef = semProc(c, fullDef) - t.destructor = semantizedDef[namePos].sym - return t - else: - t.destructor = destructorIsTrivial - return nil - else: - return nil - -proc createDestructorCall(c: PContext, s: PSym): PNode = - let varTyp = s.typ - if varTyp == nil or sfGlobal in s.flags: return - let destructableT = instantiateDestructor(c, varTyp) - if destructableT != nil: - let call = semStmt(c, newNode(nkCall, s.info, @[ - useSym(destructableT.destructor, c.graph.usageSym), - useSym(s, c.graph.usageSym)])) - result = newNode(nkDefer, s.info, @[call]) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 1598d1909e..65b111d8f5 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -53,7 +53,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = else: if efNoProcvarCheck notin flags: semProcvarCheck(c, result) if result.typ.kind == tyVar: result = newDeref(result) - semDestructorCheck(c, result, flags) proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExpr(c, n, flags) @@ -66,7 +65,6 @@ proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result.typ = errorType(c) else: semProcvarCheck(c, result) - semDestructorCheck(c, result, flags) proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = result = symChoice(c, n, s, scClosed) @@ -671,6 +669,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = if callee.magic != mNone: result = magicsAfterOverloadResolution(c, result, flags) if result.typ != nil: liftTypeBoundOps(c, result.typ, n.info) + #result = patchResolvedTypeBoundOp(c, result) if c.matchedConcept == nil: result = evalAtCompileTime(c, result) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c1bf3662f6..e01f867faf 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -97,27 +97,12 @@ template semProcvarCheck(c: PContext, n: PNode) = proc semProc(c: PContext, n: PNode): PNode -include semdestruct - -proc semDestructorCheck(c: PContext, n: PNode, flags: TExprFlags) {.inline.} = - if not newDestructors: - if efAllowDestructor notin flags and - n.kind in nkCallKinds+{nkObjConstr,nkBracket}: - if instantiateDestructor(c, n.typ) != nil: - localError(n.info, warnDestructor) - # This still breaks too many things: - when false: - if efDetermineType notin flags and n.typ.kind == tyTypeDesc and - c.p.owner.kind notin {skTemplate, skMacro}: - localError(n.info, errGenerated, "value expected, but got a type") - proc semExprBranch(c: PContext, n: PNode): PNode = result = semExpr(c, n) if result.typ != nil: # XXX tyGenericInst here? semProcvarCheck(c, result) if result.typ.kind == tyVar: result = newDeref(result) - semDestructorCheck(c, result, {}) proc semExprBranchScope(c: PContext, n: PNode): PNode = openScope(c) @@ -421,15 +406,6 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) = else: result.add identDefs -proc addDefer(c: PContext; result: var PNode; s: PSym) = - let deferDestructorCall = createDestructorCall(c, s) - if deferDestructorCall != nil: - if result.kind != nkStmtList: - let oldResult = result - result = newNodeI(nkStmtList, result.info) - result.add oldResult - result.add deferDestructorCall - proc isDiscardUnderscore(v: PSym): bool = if v.name.s == "_": v.flags.incl(sfGenSym) @@ -609,7 +585,6 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if def.kind == nkPar: v.ast = def[j] setVarType(v, tup.sons[j]) b.sons[j] = newSymNode(v) - if not newDestructors: addDefer(c, result, v) checkNilable(v) if sfCompileTime in v.flags: hasCompileTime = true if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result) @@ -1041,6 +1016,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = checkConstructedType(s.info, s.typ) if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil: checkForMetaFields(s.typ.n) + instAllTypeBoundOp(c, n.info) + proc semAllTypeSections(c: PContext; n: PNode): PNode = proc gatherStmts(c: PContext; n: PNode; result: PNode) {.nimcall.} = @@ -1095,9 +1072,11 @@ proc semTypeSection(c: PContext, n: PNode): PNode = ## to allow the type definitions in the section to reference each other ## without regard for the order of their definitions. if sfNoForward notin c.module.flags or nfSem notin n.flags: + inc c.inTypeContext typeSectionLeftSidePass(c, n) typeSectionRightSidePass(c, n) typeSectionFinalPass(c, n) + dec c.inTypeContext result = n proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) = @@ -1318,7 +1297,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = var obj = t.sons[1].sons[0] while true: incl(obj.flags, tfHasAsgn) - if obj.kind == tyGenericBody: obj = obj.lastSon + if obj.kind in {tyGenericBody, tyGenericInst}: obj = obj.lastSon elif obj.kind == tyGenericInvocation: obj = obj.sons[0] else: break if obj.kind in {tyObject, tyDistinct}: @@ -1331,10 +1310,6 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = if not noError and sfSystemModule notin s.owner.flags: localError(n.info, errGenerated, "signature for '" & s.name.s & "' must be proc[T: object](x: var T)") - else: - doDestructorStuff(c, s, n) - if not experimentalMode(c): - localError n.info, "use the {.experimental.} pragma to enable destructors" incl(s.flags, sfUsed) of "deepcopy", "=deepcopy": if s.typ.len == 2 and @@ -1561,8 +1536,11 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, s.options = gOptions if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n) if s.name.s[0] in {'.', '('}: - if s.name.s in [".", ".()", ".=", "()"] and not experimentalMode(c): + if s.name.s in [".", ".()", ".="] and not experimentalMode(c) and not newDestructors: message(n.info, warnDeprecated, "overloaded '.' and '()' operators are now .experimental; " & s.name.s) + elif s.name.s == "()" and not experimentalMode(c): + message(n.info, warnDeprecated, "overloaded '()' operators are now .experimental; " & s.name.s) + if n.sons[bodyPos].kind != nkEmpty: # for DLL generation it is annoying to check for sfImportc! if sfBorrow in s.flags: diff --git a/tests/destructor/tatomicptrs.nim b/tests/destructor/tatomicptrs.nim new file mode 100644 index 0000000000..d20596415c --- /dev/null +++ b/tests/destructor/tatomicptrs.nim @@ -0,0 +1,101 @@ +discard """ + output: '''allocating +allocating +allocating +55 +60 +99 +deallocating +deallocating +deallocating +''' + cmd: '''nim c --newruntime $file''' +""" + +type + SharedPtr*[T] = object + x: ptr T + +#proc isNil[T](s: SharedPtr[T]): bool {.inline.} = s.x.isNil + +template incRef(x) = + atomicInc(x.refcount) + +template decRef(x): untyped = atomicDec(x.refcount) + +proc makeShared*[T](x: T): SharedPtr[T] = + # XXX could benefit from 'sink' parameter. + # XXX could benefit from a macro that generates it. + result = cast[SharedPtr[T]](allocShared(sizeof(x))) + result.x[] = x + echo "allocating" + +proc `=destroy`*[T](dest: var SharedPtr[T]) = + var s = dest.x + if s != nil and decRef(s) == 0: + `=destroy`(s[]) + deallocShared(s) + echo "deallocating" + dest.x = nil + +proc `=`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = + var s = src.x + if s != nil: incRef(s) + #atomicSwap(dest, s) + # XXX use an atomic store here: + swap(dest.x, s) + if s != nil and decRef(s) == 0: + `=destroy`(s[]) + deallocShared(s) + echo "deallocating" + +proc `=sink`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = + ## XXX make this an atomic store: + if dest.x != src.x: + let s = dest.x + if s != nil: + `=destroy`(s[]) + deallocShared(s) + echo "deallocating" + dest.x = src.x + +template `.`*[T](s: SharedPtr[T]; field: untyped): untyped = + s.x.field + +template `.=`*[T](s: SharedPtr[T]; field, value: untyped) = + s.x.field = value + +from macros import unpackVarargs + +template `.()`*[T](s: SharedPtr[T]; field: untyped, args: varargs[untyped]): untyped = + unpackVarargs(s.x.field, args) + + +type + Tree = SharedPtr[TreeObj] + TreeObj = object + refcount: int + le, ri: Tree + data: int + +proc takesTree(a: Tree) = + if not a.isNil: + takesTree(a.le) + echo a.data + takesTree(a.ri) + +proc createTree(data: int): Tree = + result = makeShared(TreeObj(refcount: 1, data: data)) + +proc createTree(data: int; le, ri: Tree): Tree = + result = makeShared(TreeObj(refcount: 1, le: le, ri: ri, data: data)) + + +proc main = + let le = createTree(55) + let ri = createTree(99) + let t = createTree(60, le, ri) + takesTree(t) + +main() + diff --git a/tests/destructor/tdestructor.nim b/tests/destructor/tdestructor.nim index 639dba9411..c9f1caf2da 100644 --- a/tests/destructor/tdestructor.nim +++ b/tests/destructor/tdestructor.nim @@ -20,10 +20,10 @@ myobj destroyed ---- myobj destroyed ''' + cmd: '''nim c --newruntime $file''' + disabled: "true" """ -{.experimental.} - type TMyObj = object x, y: int @@ -61,7 +61,7 @@ proc `=destroy`(o: var TMyObj) = if o.p != nil: dealloc o.p echo "myobj destroyed" -proc `=destroy`(o: var TMyGeneric1) = +proc `=destroy`(o: var TMyGeneric1[int]) = echo "mygeneric1 destroyed" proc `=destroy`[A, B](o: var TMyGeneric2[A, B]) = diff --git a/tests/destructor/tdestructor2.nim b/tests/destructor/tdestructor2.nim deleted file mode 100644 index 34fa466af1..0000000000 --- a/tests/destructor/tdestructor2.nim +++ /dev/null @@ -1,27 +0,0 @@ -discard """ - line: 23 - nimout: " usage of a type with a destructor in a non destructible context" -""" - -{.experimental.} - -type - TMyObj = object - x, y: int - p: pointer - -proc `=destroy`(o: var TMyObj) = - if o.p != nil: dealloc o.p - -proc open: TMyObj = - result = TMyObj(x: 1, y: 2, p: alloc(3)) - - -proc `$`(x: TMyObj): string = $x.y - -proc foo = - discard open() - -# XXX doesn't trigger this yet: -#echo open() - diff --git a/tests/destructor/tdestructor3.nim b/tests/destructor/tdestructor3.nim index d0c53c7bd3..3e177d3cdd 100644 --- a/tests/destructor/tdestructor3.nim +++ b/tests/destructor/tdestructor3.nim @@ -2,14 +2,14 @@ discard """ output: '''assign destroy destroy -destroy Foo: 5 5 -destroy Foo: 123 -123''' +123 +destroy Foo: 5 +destroy Foo: 123''' + cmd: '''nim c --newruntime $file''' """ # bug #2821 -{.experimental.} type T = object From 547961f71e75594ae1d9e2e26b0e54e0a63008e3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 1 Dec 2017 02:05:48 +0100 Subject: [PATCH 60/92] dotops: add a simple object delegation test --- tests/specialops/tdotops.nim | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/specialops/tdotops.nim b/tests/specialops/tdotops.nim index 20066a4969..227204f514 100644 --- a/tests/specialops/tdotops.nim +++ b/tests/specialops/tdotops.nim @@ -11,7 +11,8 @@ no params call to a no params call to b 100 one param call to c with 10 -100''' +100 +0 4''' """ type @@ -63,3 +64,24 @@ echo tt.c(10) assert(not compiles(tt.d("x"))) assert(not compiles(tt.d(1, 2))) + +# test simple usage that delegates fields: +type + Other = object + a: int + b: string + MyObject = object + nested: Other + x, y: int + +template `.`(x: MyObject; field: untyped): untyped = + x.nested.field + +template `.=`(x: MyObject; field, value: untyped) = + x.nested.field = value + +var m: MyObject + +m.a = 4 +m.b = "foo" +echo m.x, " ", m.a From 6a101c5004c64b229a99e437af1640342aa98709 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 1 Dec 2017 02:31:47 +0100 Subject: [PATCH 61/92] os.nim bugfix: system() returns bullshit on Posix-like systems in general --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 8b26552dee..a59134007a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -630,7 +630,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - when defined(linux): + when defined(posix): result = c_system(command) shr 8 else: result = c_system(command) From 3181f3b04c7fe871548d127e6598d589a4800b17 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 30 Nov 2017 19:39:16 -0600 Subject: [PATCH 62/92] favor 'select' over 'poll' on Genode (#6821) The 'poll' of the Genode C runtime is a wrapper over 'select'. --- lib/pure/ioselects/ioselectors_select.nim | 28 ++++++++++++----------- lib/pure/selectors.nim | 2 ++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 017d08117d..c787f0070f 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -279,8 +279,9 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, inc(s.count) pkey.events = events -proc unregister*[T](s: Selector[T], fd: SocketHandle) = +proc unregister*[T](s: Selector[T], fd: SocketHandle|int) = s.withSelectLock(): + let fd = fd.SocketHandle var pkey = s.getKey(fd) if Event.Read in pkey.events: IOFD_CLR(fd, addr s.rSet) @@ -438,16 +439,17 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = mixin withSelectLock s.withSelectLock(): - var value: ptr T - let fdi = int(fd) - var i = 0 - while i < FD_SETSIZE: - if s.fds[i].ident == fdi: - value = addr(s.fds[i].data) - break - inc(i) - if i != FD_SETSIZE: - body1 - else: - body2 + block: + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 3a7d1523fa..518cc4bd52 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -303,6 +303,8 @@ else: include ioselects/ioselectors_select elif defined(solaris): include ioselects/ioselectors_poll # need to replace it with event ports + elif defined(genode): + include ioselects/ioselectors_select # TODO: use the native VFS layer else: include ioselects/ioselectors_poll From 96b7c2481cf792b9003844aea435b7bb937ec44d Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 1 Dec 2017 10:18:49 +0100 Subject: [PATCH 63/92] make tests green again --- compiler/semcall.nim | 2 - tests/concepts/texplain.nim | 80 +++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a77fcad974..a51b9afe30 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -238,7 +238,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, # leave the op head symbol empty, # we are going to try multiple variants - errors = nil n.sons[0..1] = [nil, n[1], f] orig.sons[0..1] = [nil, orig[1], f] @@ -255,7 +254,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, tryOp "." elif nfDotSetter in n.flags and f.kind == nkIdent and n.len == 3: - errors = nil # we need to strip away the trailing '=' here: let calleeName = newIdentNode(getIdent(f.ident.s[0..f.ident.s.len-2]), n.info) let callOp = newIdentNode(getIdent".=", n.info) diff --git a/tests/concepts/texplain.nim b/tests/concepts/texplain.nim index 417d1e5022..de8ddf890f 100644 --- a/tests/concepts/texplain.nim +++ b/tests/concepts/texplain.nim @@ -1,62 +1,66 @@ discard """ cmd: "nim c --verbosity:0 --colors:off $file" nimout: ''' -texplain.nim(99, 10) Hint: Non-matching candidates for e(y) +texplain.nim(103, 10) Hint: Non-matching candidates for e(y) proc e(i: int): int -texplain.nim(102, 7) Hint: Non-matching candidates for e(10) +texplain.nim(106, 7) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int -texplain.nim(65, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(65, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(65, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(65, 5) ExplainedConcept: concept predicate failed -texplain.nim(66, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(66, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(66, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(65, 5) ExplainedConcept: concept predicate failed +texplain.nim(69, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(69, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(69, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(69, 5) ExplainedConcept: concept predicate failed +texplain.nim(70, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(70, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(70, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(69, 5) ExplainedConcept: concept predicate failed -texplain.nim(105, 10) Hint: Non-matching candidates for e(10) +texplain.nim(109, 10) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int -texplain.nim(65, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(65, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(65, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(65, 5) ExplainedConcept: concept predicate failed -texplain.nim(66, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(66, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(66, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(65, 5) ExplainedConcept: concept predicate failed +texplain.nim(69, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(69, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(69, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(69, 5) ExplainedConcept: concept predicate failed +texplain.nim(70, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(70, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(70, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(69, 5) ExplainedConcept: concept predicate failed -texplain.nim(109, 20) Error: type mismatch: got (NonMatchingType) -but expected one of: +texplain.nim(113, 20) Error: type mismatch: got (NonMatchingType) +but expected one of: proc e(o: ExplainedConcept): int -texplain.nim(65, 5) ExplainedConcept: concept predicate failed +texplain.nim(69, 5) ExplainedConcept: concept predicate failed proc e(i: int): int -texplain.nim(110, 20) Error: type mismatch: got (NonMatchingType) -but expected one of: +expression: e(n) +texplain.nim(114, 20) Error: type mismatch: got (NonMatchingType) +but expected one of: proc r(o: RegularConcept): int -texplain.nim(69, 5) RegularConcept: concept predicate failed +texplain.nim(73, 5) RegularConcept: concept predicate failed proc r[T](a: SomeNumber; b: T; c: auto) proc r(i: string): int -texplain.nim(111, 20) Hint: Non-matching candidates for r(y) +expression: r(n) +texplain.nim(115, 20) Hint: Non-matching candidates for r(y) proc r[T](a: SomeNumber; b: T; c: auto) proc r(i: string): int -texplain.nim(119, 2) Error: type mismatch: got (MatchingType) -but expected one of: +texplain.nim(123, 2) Error: type mismatch: got (MatchingType) +but expected one of: proc f(o: NestedConcept) -texplain.nim(69, 6) RegularConcept: undeclared field: 'foo' -texplain.nim(69, 6) RegularConcept: undeclared field: '.' -texplain.nim(69, 6) RegularConcept: expression '.' cannot be called -texplain.nim(69, 5) RegularConcept: concept predicate failed -texplain.nim(70, 6) RegularConcept: undeclared field: 'bar' -texplain.nim(70, 6) RegularConcept: undeclared field: '.' -texplain.nim(70, 6) RegularConcept: expression '.' cannot be called -texplain.nim(69, 5) RegularConcept: concept predicate failed -texplain.nim(73, 5) NestedConcept: concept predicate failed +texplain.nim(73, 6) RegularConcept: undeclared field: 'foo' +texplain.nim(73, 6) RegularConcept: undeclared field: '.' +texplain.nim(73, 6) RegularConcept: expression '.' cannot be called +texplain.nim(73, 5) RegularConcept: concept predicate failed +texplain.nim(74, 6) RegularConcept: undeclared field: 'bar' +texplain.nim(74, 6) RegularConcept: undeclared field: '.' +texplain.nim(74, 6) RegularConcept: expression '.' cannot be called +texplain.nim(73, 5) RegularConcept: concept predicate failed +texplain.nim(77, 5) NestedConcept: concept predicate failed + +expression: f(y) ''' - line: 119 + line: 123 errormsg: "type mismatch: got (MatchingType)" """ From d27c0b219249b2584fc7a3f175adfde862ca701f Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 1 Dec 2017 11:20:50 +0100 Subject: [PATCH 64/92] make asyncdispatch compile with the foreign GCs --- lib/system/mmdisp.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 9af36c7b8c..d65d8a10ed 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -571,3 +571,11 @@ when not declared(nimNewSeqOfCap): cast[PGenericSeq](result).reserved = cap {.pop.} + +when not declared(ForeignCell): + type ForeignCell* = object + data*: pointer + + proc protect*(x: pointer): ForeignCell = ForeignCell(data: x) + proc dispose*(x: ForeignCell) = discard + proc isNotForeign*(x: ForeignCell): bool = false From 1699d7c2a4c3f3f2ae604dc1ddd29aa34d69ceb1 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Fri, 1 Dec 2017 16:42:10 +0200 Subject: [PATCH 65/92] Implement codegenDecl for js (#6851) --- compiler/jsgen.nim | 44 ++++++++++++++++++++++++++--------- tests/js/tcodegendeclproc.nim | 11 +++++++++ tests/js/tcodegendeclvar.nim | 10 ++++++++ 3 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 tests/js/tcodegendeclproc.nim create mode 100644 tests/js/tcodegendeclvar.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 855a85be7f..bc0f90e179 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1563,14 +1563,22 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = internalError("createVar: " & $t.kind) result = nil +template returnType: untyped = + ~"" + proc genVarInit(p: PProc, v: PSym, n: PNode) = var a: TCompRes s: Rope + varCode: string + if v.constraint.isNil: + varCode = "var $2" + else: + varCode = v.constraint.strVal if n.kind == nkEmpty: let mname = mangleName(v, p.target) - lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n", - [mname, createVar(p, v.typ, isIndirect(v))]) + lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n", + [returnType, mname, createVar(p, v.typ, isIndirect(v))]) if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex: lineF(p, "var $1_Idx = 0;$n", [ mname ]) else: @@ -1587,25 +1595,25 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {} if a.typ == etyBaseIndex: if targetBaseIndex: - lineF(p, "var $1 = $2, $1_Idx = $3;$n", - [v.loc.r, a.address, a.res]) + lineF(p, varCode & " = $3, $2_Idx = $4;$n", + [returnType, v.loc.r, a.address, a.res]) else: - lineF(p, "var $1 = [$2, $3];$n", - [v.loc.r, a.address, a.res]) + lineF(p, varCode & " = [$3, $4];$n", + [returnType, v.loc.r, a.address, a.res]) else: if targetBaseIndex: let tmp = p.getTemp lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n", [tmp, a.res, v.loc.r]) else: - lineF(p, "var $1 = $2;$n", [v.loc.r, a.res]) + lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, a.res]) return else: s = a.res if isIndirect(v): - lineF(p, "var $1 = [$2];$n", [v.loc.r, s]) + lineF(p, varCode & " = [$3];$n", [returnType, v.loc.r, s]) else: - lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n", [v.loc.r, s]) + lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n", [returnType, v.loc.r, s]) proc genVarStmt(p: PProc, n: PNode) = for i in countup(0, sonsLen(n) - 1): @@ -2162,8 +2170,22 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = returnStmt = "return $#;$n" % [a.res] p.nested: genStmt(p, prc.getBody) - let def = "function $#($#) {$n$#$#$#$#$#" % - [name, header, + + var def: Rope + if not prc.constraint.isNil: + def = (prc.constraint.strVal & " {$n$#$#$#$#$#") % + [ returnType, + name, + header, + optionaLine(p.globals), + optionaLine(p.locals), + optionaLine(resultAsgn), + optionaLine(genProcBody(p, prc)), + optionaLine(p.indentLine(returnStmt))] + else: + def = "function $#($#) {$n$#$#$#$#$#" % + [ name, + header, optionaLine(p.globals), optionaLine(p.locals), optionaLine(resultAsgn), diff --git a/tests/js/tcodegendeclproc.nim b/tests/js/tcodegendeclproc.nim new file mode 100644 index 0000000000..3acf0bc137 --- /dev/null +++ b/tests/js/tcodegendeclproc.nim @@ -0,0 +1,11 @@ +discard """ + output: ''' +-1 +8 +''' + ccodecheck: "'console.log(-1); function fac_' \\d+ '(n_' \\d+ ')'" +""" +proc fac(n: int): int {.codegenDecl: "console.log(-1); function $2($3)".} = + return n + +echo fac(8) diff --git a/tests/js/tcodegendeclvar.nim b/tests/js/tcodegendeclvar.nim new file mode 100644 index 0000000000..645443ef7a --- /dev/null +++ b/tests/js/tcodegendeclvar.nim @@ -0,0 +1,10 @@ +discard """ + output: ''' +-1 +2 +''' + ccodecheck: "'console.log(-1); var v_' \\d+ ' = [2]'" +""" + +var v {.codegenDecl: "console.log(-1); var $2".} = 2 +echo v From 82870058d31f5291b63b690139522050a589d233 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 2 Dec 2017 00:57:13 +0100 Subject: [PATCH 66/92] finish.nim: make it work with spaces in the path to curl --- tools/finish.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/finish.nim b/tools/finish.nim index 45d7dd3a87..207f15f76c 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -32,7 +32,7 @@ proc downloadMingw(): DownloadResult = let curl = findExe"curl" var cmd: string if curl.len > 0: - cmd = curl & " --out " & "dist" / mingw & " " & url + cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url elif fileExists"bin/nimgrab.exe": cmd = "bin/nimgrab.exe " & url & " dist" / mingw if cmd.len > 0: From c039bbf6e1186ba8a6f65fa6e0179c8245f2d483 Mon Sep 17 00:00:00 2001 From: pqflx3 Date: Sat, 2 Dec 2017 14:40:00 -0500 Subject: [PATCH 67/92] Fixed printing nimsuggest commandline help message (#6863) --- nimsuggest/nimsuggest.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index 67645f0437..0328b817a0 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -526,6 +526,9 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = of cmdEnd: break of cmdLongoption, cmdShortOption: case p.key.normalize + of "help": + stdout.writeline(Usage) + quit() of "port": gPort = parseInt(p.val).Port gMode = mtcp From a0699870e37f392b8e61ea5bb5216856f720a3ee Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 3 Dec 2017 15:20:13 +0100 Subject: [PATCH 68/92] osalloc: improve error message when virtualFree fails --- lib/system/osalloc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 65a057772c..444f113069 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -166,7 +166,7 @@ elif defined(windows): # space heavily, so we now treat Windows as a strange unmap target. when reallyOsDealloc: if virtualFree(p, 0, MEM_RELEASE) == 0: - cprintf "yes, failing!" + cprintf "virtualFree failing!" quit 1 #VirtualFree(p, size, MEM_DECOMMIT) From 9820c2c4561ee56f30c1578672dd1247be25cb11 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 3 Dec 2017 15:20:50 +0100 Subject: [PATCH 69/92] bitops: add 'hamming weight' to the doc index --- lib/pure/bitops.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index d1207603d2..3f213c5ea3 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -181,7 +181,7 @@ elif useICC_builtins: proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} = - ## Counts the set bits in integer. (also called Hamming weight.) + ## Counts the set bits in integer. (also called `Hamming weight`:idx:.) # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. # like GCC and MSVC when nimvm: From 35d7a99b6a4cdd44e216811790e528ae6b8e44ff Mon Sep 17 00:00:00 2001 From: jcosborn Date: Mon, 4 Dec 2017 10:37:25 -0600 Subject: [PATCH 70/92] fix getTypeInst for tyGenericInst (#6868) --- compiler/vmdeps.nim | 9 ++++++--- tests/macros/tgettypeinst.nim | 32 ++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 3d43046e93..44550a3895 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -121,22 +121,25 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 ..< t.len: result.add mapTypeToAst(t.sons[i], info) - of tyGenericInst, tyAlias: + of tyGenericInst: if inst: if allowRecursion: result = mapTypeToAstR(t.lastSon, info) else: result = newNodeX(nkBracketExpr) - result.add mapTypeToAst(t.lastSon, info) + #result.add mapTypeToAst(t.lastSon, info) + result.add mapTypeToAst(t[0], info) for i in 1 ..< t.len-1: result.add mapTypeToAst(t.sons[i], info) else: result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion) of tyGenericBody: if inst: - result = mapTypeToAstX(t.lastSon, info, inst, true) + result = mapTypeToAstR(t.lastSon, info) else: result = mapTypeToAst(t.lastSon, info) + of tyAlias: + result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion) of tyOrdinal: result = mapTypeToAst(t.lastSon, info) of tyDistinct: diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim index 8e1d9bc13f..ea98721c48 100644 --- a/tests/macros/tgettypeinst.nim +++ b/tests/macros/tgettypeinst.nim @@ -27,9 +27,10 @@ macro testX(x,inst0: typed; recurse: static[bool]; implX: typed): typed = let inst = x.getTypeInst let instr = inst.symToIdent.treeRepr let inst0r = inst0.symToIdent.treeRepr - #echo instr - #echo inst0r - doAssert(instr == inst0r) + if instr != inst0r: + echo "instr:\n", instr + echo "inst0r:\n", inst0r + doAssert(instr == inst0r) # check that getTypeImpl(x) is correct # if implX is nil then compare to inst0 @@ -41,9 +42,10 @@ macro testX(x,inst0: typed; recurse: static[bool]; implX: typed): typed = else: implX[0][2] let implr = impl.symToIdent.treerepr let impl0r = impl0.symToIdent.treerepr - #echo implr - #echo impl0r - doAssert(implr == impl0r) + if implr != impl0r: + echo "implr:\n", implr + echo "impl0r:\n", impl0r + doAssert(implr == impl0r) result = newStmtList() #template echoString(s: string) = echo s.replace("\n","\n ") @@ -111,6 +113,14 @@ type Generic[T] = seq[int] Concrete = Generic[int] + Alias1 = float + Alias2 = Concrete + + Vec[N: static[int],T] = object + arr: array[N,T] + Vec4[T] = Vec[4,T] + + test(bool) test(char) test(int) @@ -149,6 +159,16 @@ test(Generic[int]): type _ = seq[int] test(Generic[float]): type _ = seq[int] +test(Alias1): + type _ = float +test(Alias2): + type _ = Generic[int] +test(Vec[4,float32]): + type _ = object + arr: array[0..3,float32] +test(Vec4[float32]): + type _ = object + arr: array[0..3,float32] # bug #4862 static: From 6ee08cf70c9461972cfcf179e12669b5518977ff Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Dec 2017 13:53:18 +0100 Subject: [PATCH 71/92] fix documentation comments in sequtils.nim --- lib/pure/collections/sequtils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index d0f5c78e07..06e96ca36f 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -66,7 +66,7 @@ proc cycle*[T](s: openArray[T], n: Natural): seq[T] = ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## s = @[1, 2, 3] @@ -84,7 +84,7 @@ proc repeat*[T](x: T, n: Natural): seq[T] = ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## total = repeat(5, 3) From c7ba4d91a34882e94969595ba70763f9f642423c Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Wed, 6 Dec 2017 03:56:44 -0500 Subject: [PATCH 72/92] add dynlibOverrideAll switch (#6873) --- compiler/commands.nim | 3 +++ compiler/options.nim | 3 ++- doc/advopt.txt | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 11a66cf55e..de474c6e68 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -654,6 +654,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; gListFullPaths = true of "dynliboverride": dynlibOverride(switch, arg, pass, info) + of "dynliboverrideall": + expectNoArg(switch, arg, pass, info) + gDynlibOverrideAll = true of "cs": # only supported for compatibility. Does nothing. expectArg(switch, arg, pass, info) diff --git a/compiler/options.nim b/compiler/options.nim index eec9ce4487..8c4fe485eb 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -145,6 +145,7 @@ var gNoNimblePath* = false gExperimentalMode*: bool newDestructors*: bool + gDynlibOverrideAll*: bool proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools} proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc @@ -427,7 +428,7 @@ proc inclDynlibOverride*(lib: string) = gDllOverrides[lib.canonDynlibName] = "true" proc isDynlibOverride*(lib: string): bool = - result = gDllOverrides.hasKey(lib.canonDynlibName) + result = gDynlibOverrideAll or gDllOverrides.hasKey(lib.canonDynlibName) proc binaryStrSearch*(x: openArray[string], y: string): int = var a = 0 diff --git a/doc/advopt.txt b/doc/advopt.txt index 60fd081b8a..ab10d65ba7 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -79,6 +79,7 @@ Advanced options: symbol matching is fuzzy so that --dynlibOverride:lua matches dynlib: "liblua.so.3" + --dynlibOverrideAll makes the dynlib pragma have no effect --listCmd list the commands used to execute external programs --parallelBuild:0|1|... perform a parallel build value = number of processors (0 for auto-detect) From ede38a70fc4c7bb403555fb5b95fb50d609e73dc Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 10:54:46 +0100 Subject: [PATCH 73/92] make allocator use the TLSF algorithm; work in progress --- lib/system/alloc.nim | 157 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 127 insertions(+), 30 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 19d27e7d2b..d587b16989 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -8,8 +8,6 @@ # # Low level allocator for Nim. Has been designed to support the GC. -# TODO: -# - make searching for block O(1) {.push profiler:off.} include osalloc @@ -19,14 +17,16 @@ template track(op, address, size) = memTrackerOp(op, address, size) # We manage *chunks* of memory. Each chunk is a multiple of the page size. -# Each chunk starts at an address that is divisible by the page size. Chunks -# that are bigger than ``ChunkOsReturn`` are returned back to the operating -# system immediately. +# Each chunk starts at an address that is divisible by the page size. const - ChunkOsReturn = 256 * PageSize # 1 MB - InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn! + InitialMemoryRequest = 128 * PageSize # 0.5 MB SmallChunkSize = PageSize + MaxFli = 30 + MaxLog2Sli = 5 + MaxSli = 1 shl MaxLog2Sli + FliOffset = 6 + RealFli = MaxFli - FliOffset type PTrunk = ptr Trunk @@ -99,10 +99,12 @@ type MemRegion = object minLargeObj, maxLargeObj: int freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] + flBitmap: uint32 + slBitmap: array[RealFli, uint32] + matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk currMem, maxMem, freeMem: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly - freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. @@ -110,7 +112,104 @@ type bottomData: AvlNode heapLinks: HeapLinks -{.deprecated: [TMemRegion: MemRegion].} +const + fsLookupTable: array[byte, int8] = [ + -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7 + ] + +proc msbit(x: uint32): int {.inline.} = + let a = if x <= 0xff_ff: + (if x <= 0xff: 0 else: 8) + else: + (if x <= 0xff_ff_ff: 16 else: 24) + result = int(fsLookupTable[byte(x shr a)]) + a + +proc lsbit(x: uint32): int {.inline.} = + msbit(x and ((not x) + 1)) + +proc setBit(nr: int; dest: var uint32) {.inline.} = + dest = dest or (1u32 shl (nr and 0x1f)) + +proc clearBit(nr: int; dest: var uint32) {.inline.} = + dest = dest and not (1u32 shl (nr and 0x1f)) + +proc mappingSearch(r, fl, sl: var int) {.inline.} = + let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1 + r = r + t + fl = msbit(uint32 r) + sl = (r shr (fl - MaxLog2Sli)) - MaxSli + dec fl, FliOffset + r = r and not t + +# See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of +# this algorithm. + +proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} = + result.fl = msbit(uint32 r) + result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli + dec result.fl, FliOffset + +template mat(): untyped = a.matrix[fl][sl] + +proc findSuitableBlock(a: MemRegion; fl, sl: var int): PBigChunk {.inline.} = + let tmp = a.slBitmap[fl] and (not 0u32 shl sl) + result = nil + if tmp != 0: + sl = lsbit(tmp) + result = mat() + else: + fl = lsbit(a.flBitmap and (not 0u32 shl (fl + 1))) + if fl > 0: + sl = lsbit(a.slBitmap[fl]) + result = mat() + +template clearBits(sl, fl) = + clearBit(sl, a.slBitmap[fl]) + if a.slBitmap[fl] == 0u32: + # do not forget to cascade: + clearBit(fl, a.flBitmap) + +proc removeChunkFromMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + if b.next != nil: b.next.prev = b.prev + if b.prev != nil: b.prev.next = b.next + if mat() == b: + mat() = b.next + if mat() == nil: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc removeChunkFromMatrix2(a: var MemRegion; b: PBigChunk; fl, sl: int) = + mat() = b.next + if mat() != nil: + mat().prev = nil + else: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + b.prev = nil + b.next = mat() + if mat() != nil: + mat().prev = b + mat() = b + setBit(sl, a.slBitmap[fl]) + setBit(fl, a.flBitmap) {.push stack_trace: off.} proc initAllocator() = discard "nothing to do anymore" @@ -419,7 +518,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri): - listRemove(a.freeChunksList, cast[PBigChunk](ri)) + removeChunkFromMatrix(a, cast[PBigChunk](ri)) inc(c.size, ri.size) excl(a.chunkStarts, pageIndex(ri)) when coalescLeft: @@ -430,49 +529,42 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le): - listRemove(a.freeChunksList, cast[PBigChunk](le)) + removeChunkFromMatrix(a, cast[PBigChunk](le)) inc(le.size, c.size) excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) incl(a, a.chunkStarts, pageIndex(c)) updatePrevSize(a, c, c.size) - listAdd(a.freeChunksList, c) + addChunkToMatrix(a, c) # set 'used' to false: c.prevSize = c.prevSize and not 1 proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = var rest = cast[PBigChunk](cast[ByteAddress](c) +% size) - sysAssert(rest notin a.freeChunksList, "splitChunk") rest.size = c.size - size track("rest.origSize", addr rest.origSize, sizeof(int)) + # XXX check if these two nil assignments are dead code given + # addChunkToMatrix's implementation: rest.next = nil rest.prev = nil - # size and not used + # size and not used: rest.prevSize = size sysAssert((size and 1) == 0, "splitChunk 2") updatePrevSize(a, c, rest.size) c.size = size incl(a, a.chunkStarts, pageIndex(rest)) - listAdd(a.freeChunksList, rest) + addChunkToMatrix(a, rest) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # use first fit for now: sysAssert((size and PageMask) == 0, "getBigChunk 1") sysAssert(size > 0, "getBigChunk 2") - result = a.freeChunksList - block search: - while result != nil: - sysAssert chunkUnused(result), "getBigChunk 3" - if result.size == size: - listRemove(a.freeChunksList, result) - break search - elif result.size > size: - listRemove(a.freeChunksList, result) - splitChunk(a, result, size) - break search - result = result.next - sysAssert result != a.freeChunksList, "getBigChunk 4" + var size = size # roundup(size, PageSize) + var fl, sl: int + mappingSearch(size, fl, sl) + result = findSuitableBlock(a, fl, sl) + if result == nil: if size < InitialMemoryRequest: result = requestOsChunks(a, InitialMemoryRequest) splitChunk(a, result, size) @@ -481,7 +573,10 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # if we over allocated split the chunk: if result.size > size: splitChunk(a, result, size) - + else: + removeChunkFromMatrix2(a, result, fl, sl) + if result.size >= size + PageSize: + splitChunk(a, result, size) # set 'used' to to true: result.prevSize = 1 track("setUsedToFalse", addr result.origSize, sizeof(int)) @@ -577,9 +672,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = var c = getBigChunk(a, size) sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" - sysAssert c.size == size, "rawAlloc 12" result = addr(c.data) sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13") + #if (cast[ByteAddress](result) and PageMask) != 0: + # cprintf("Address is %p for size %ld\n", result, c.size) + sysAssert((cast[ByteAddress](result) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) sysAssert(isAccessible(a, result), "rawAlloc 14") From 7e7ce19ec1ffb1e1fb09b6b3d81e1b1f157ba20e Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 13:14:28 +0100 Subject: [PATCH 74/92] GC test workaround: use a lock for 'echo' for Windows --- tests/parallel/tdisjoint_slice2.nim | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/parallel/tdisjoint_slice2.nim b/tests/parallel/tdisjoint_slice2.nim index 1e86ea644a..25cb2362f8 100644 --- a/tests/parallel/tdisjoint_slice2.nim +++ b/tests/parallel/tdisjoint_slice2.nim @@ -11,12 +11,19 @@ discard """ sortoutput: true """ -import threadpool +import threadpool, locks + +var echoLock: Lock +initLock echoLock proc f(a: openArray[int]) = - for x in a: echo x + for x in a: + withLock echoLock: + echo x -proc f(a: int) = echo a +proc f(a: int) = + withLock echoLock: + echo a proc main() = var a: array[0..9, int] = [0,1,2,3,4,5,6,7,8,9] From dc7a69cb669618eb5d4b53c18e4e3780a76b1fee Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 13:15:00 +0100 Subject: [PATCH 75/92] Threading: increase TLS size for new allocator --- lib/system/threads.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 016bf5822b..f61cc4280f 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -255,9 +255,9 @@ when emulatedThreadVars: proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} # we preallocate a fixed size for thread local storage, so that no heap -# allocations are needed. Currently less than 7K are used on a 64bit machine. +# allocations are needed. Currently less than 16K are used on a 64bit machine. # We use ``float`` for proper alignment: -const nimTlsSize {.intdefine.} = 8000 +const nimTlsSize {.intdefine.} = 16000 type ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float] From 7c9a3161daaf971303779c9d2199474b2d955082 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 13:24:18 +0100 Subject: [PATCH 76/92] make the new allocator work --- lib/system/alloc.nim | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index d587b16989..725c9ccd38 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -23,7 +23,8 @@ const InitialMemoryRequest = 128 * PageSize # 0.5 MB SmallChunkSize = PageSize MaxFli = 30 - MaxLog2Sli = 5 + MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' + # everywhere! MaxSli = 1 shl MaxLog2Sli FliOffset = 6 RealFli = MaxFli - FliOffset @@ -146,17 +147,22 @@ proc clearBit(nr: int; dest: var uint32) {.inline.} = dest = dest and not (1u32 shl (nr and 0x1f)) proc mappingSearch(r, fl, sl: var int) {.inline.} = - let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1 + #let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1 + # This diverges from the standard TLSF algorithm because we need to ensure + # PageSize alignment: + let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 r = r + t fl = msbit(uint32 r) sl = (r shr (fl - MaxLog2Sli)) - MaxSli dec fl, FliOffset r = r and not t + sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned") # See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of # this algorithm. proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} = + sysAssert((r and PageMask) == 0, "mappingInsert: still not aligned") result.fl = msbit(uint32 r) result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli dec result.fl, FliOffset @@ -468,6 +474,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = result.prevSize = 0 or (result.prevSize and 1) # unknown # but do not overwrite 'used' field a.lastSize = size # for next request + sysAssert((cast[int](result) and PageMask) == 0, "requestOschunks: unaligned chunk") proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} = result = contains(a.chunkStarts, pageIndex(p)) @@ -551,6 +558,8 @@ proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = # size and not used: rest.prevSize = size sysAssert((size and 1) == 0, "splitChunk 2") + sysAssert((size and PageMask) == 0, + "splitChunk: size is not a multiple of the PageSize") updatePrevSize(a, c, rest.size) c.size = size incl(a, a.chunkStarts, pageIndex(rest)) @@ -558,11 +567,11 @@ proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # use first fit for now: - sysAssert((size and PageMask) == 0, "getBigChunk 1") sysAssert(size > 0, "getBigChunk 2") var size = size # roundup(size, PageSize) var fl, sl: int mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") result = findSuitableBlock(a, fl, sl) if result == nil: if size < InitialMemoryRequest: @@ -667,16 +676,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") else: - size = roundup(requestedSize+bigChunkOverhead(), PageSize) + size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block var c = getBigChunk(a, size) sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13") - #if (cast[ByteAddress](result) and PageMask) != 0: - # cprintf("Address is %p for size %ld\n", result, c.size) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) sysAssert(isAccessible(a, result), "rawAlloc 14") From 226532f8f39e8d37d392282ed57aa891c0e69736 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 15:58:46 +0100 Subject: [PATCH 77/92] cleanup todo.txt --- lib/system/mmdisp.nim | 3 +- tests/fragmentation/data.nim | 7629 +++++++++++++++++++++++ tests/fragmentation/tfragment_alloc.nim | 19 + tests/fragmentation/tfragment_gc.nim | 16 + todo.txt | 2 - 5 files changed, 7666 insertions(+), 3 deletions(-) create mode 100644 tests/fragmentation/data.nim create mode 100644 tests/fragmentation/tfragment_alloc.nim create mode 100644 tests/fragmentation/tfragment_gc.nim diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index d65d8a10ed..9ac039e192 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -42,7 +42,8 @@ type # Page size of the system; in most cases 4096 bytes. For exotic OS or # CPU this needs to be changed: const - PageShift = when defined(cpu16): 8 else: 12 + PageShift = when defined(cpu16): 8 else: 12 # \ + # my tests showed no improvments for using larger page sizes. PageSize = 1 shl PageShift PageMask = PageSize-1 diff --git a/tests/fragmentation/data.nim b/tests/fragmentation/data.nim new file mode 100644 index 0000000000..d14123c0ff --- /dev/null +++ b/tests/fragmentation/data.nim @@ -0,0 +1,7629 @@ + +const sizes* = [ + 643584, + 140800, + 802, + 2464928, + 488122, + 1161601, + 861051, + 1413, + 389, + 2412032, + 328704, + 323488, + 484864, + 31232, + 38400, + 87552, + 328704, + 242176, + 55808, + 32256, + 15360, + 15080, + 567074, + 1390, + 1390, + 366080, + 186536, + 314368, + 4154, + 3017582, + 62658, + 3874662, + 227, + 126616, + 290, + 261792, + 287, + 151200, + 293, + 34976, + 293, + 28840, + 296, + 20664, + 314, + 223232, + 288, + 53248, + 326, + 12800, + 312, + 473600, + 306, + 2676224, + 308, + 2846720, + 308, + 563712, + 308, + 567296, + 308, + 576000, + 308, + 577024, + 308, + 577536, + 308, + 577536, + 308, + 578560, + 308, + 578560, + 308, + 145920, + 310, + 159232, + 312, + 364544, + 310, + 178176, + 312, + 8032448, + 323, + 29392, + 332, + 35112, + 394, + 85768, + 371, + 88864, + 385, + 23840, + 385, + 176128, + 386, + 126976, + 389, + 24800, + 401, + 118784, + 379, + 21792, + 385, + 259152, + 385, + 172032, + 389, + 40960, + 389, + 114688, + 403, + 57344, + 407, + 7680, + 407, + 126216, + 373, + 29952, + 367, + 148768, + 385, + 27384, + 362, + 24832, + 362, + 50944, + 364, + 20136, + 302, + 32416, + 293, + 69296, + 305, + 36016, + 308, + 89784, + 305, + 5120, + 305, + 49152, + 320, + 30424, + 331, + 12288, + 326, + 62976, + 69120, + 72192, + 503808, + 77824, + 382144, + 163840, + 88728, + 1581, + 1718096, + 66728, + 82172, + 116756, + 4554752, + 59342, + 45794, + 39284, + 66384, + 60294, + 83748, + 83748, + 262148, + 20320, + 28288, + 382, + 3072, + 4222976, + 161, + 1737888, + 2975744, + 487424, + 258048, + 113664, + 372736, + 261632, + 5287936, + 80896, + 89600, + 503808, + 77824, + 554176, + 163840, + 84632, + 1581, + 1714000, + 66728, + 82172, + 116756, + 4571136, + 59342, + 45794, + 39284, + 66384, + 60294, + 83748, + 83748, + 262148, + 20320, + 28288, + 4006400, + 161, + 2256032, + 3150336, + 503296, + 245760, + 133120, + 358400, + 283136, + 5296128, + 10752, + 507904, + 315392, + 11264, + 921600, + 7680, + 118784, + 166560, + 13312, + 5120, + 18768, + 52736, + 8192, + 77824, + 6656, + 106496, + 389120, + 733184, + 53248, + 36864, + 36864, + 655360, + 139264, + 802816, + 169864, + 77824, + 10752, + 94208, + 17808, + 40960, + 36864, + 749568, + 45056, + 188416, + 28672, + 16384, + 69632, + 102400, + 11776, + 10752, + 290816, + 36864, + 36864, + 667648, + 53248, + 49152, + 200704, + 45056, + 40960, + 77824, + 10240, + 9216, + 77504, + 161472, + 163528, + 26304, + 26312, + 96448, + 99016, + 19648, + 19656, + 127680, + 127680, + 364224, + 323272, + 73408, + 71368, + 24776, + 138944, + 128200, + 27328, + 48832, + 37576, + 32960, + 52928, + 40128, + 89792, + 80584, + 25792, + 499392, + 504520, + 163520, + 167624, + 36032, + 36040, + 138944, + 127688, + 4208320, + 4839104, + 77504, + 81608, + 2768576, + 2757832, + 331456, + 321728, + 65224, + 65216, + 56008, + 163520, + 154816, + 48832, + 77504, + 62656, + 48832, + 77504, + 50888, + 57024, + 47296, + 52928, + 40640, + 71360, + 71360, + 22208, + 22208, + 57024, + 45768, + 48832, + 48840, + 3720896, + 4037832, + 790208, + 868040, + 230592, + 230592, + 20160, + 20160, + 29376, + 31944, + 1355456, + 1419976, + 839360, + 851656, + 32960, + 36040, + 179904, + 265920, + 85696, + 85696, + 33472, + 33472, + 392896, + 391872, + 397008, + 48832, + 48832, + 48832, + 48832, + 52928, + 48832, + 48832, + 57024, + 36032, + 36032, + 48832, + 48840, + 48840, + 48832, + 52928, + 48832, + 48832, + 57024, + 36032, + 36032, + 48848, + 48848, + 48848, + 48848, + 52944, + 48848, + 48848, + 57040, + 36048, + 36048, + 57024, + 45760, + 20160, + 20160, + 34528, + 13024, + 397312, + 5120, + 28672, + 659456, + 372736, + 110592, + 9216, + 9728, + 61440, + 28672, + 5632, + 41984, + 29736, + 20056, + 216632, + 110184, + 27032, + 37472, + 31592, + 24376, + 102728, + 280912, + 27472, + 27936, + 23392, + 56624, + 57216, + 38744, + 51016, + 36696, + 48896, + 53024, + 32032, + 34120, + 34120, + 20256, + 107848, + 109928, + 1267040, + 46984, + 496544, + 1174384, + 762288, + 101744, + 32664, + 95712, + 48480, + 365344, + 98448, + 86440, + 35312, + 37704, + 75664, + 101672, + 47912, + 15872, + 11168, + 22856, + 22856, + 506616, + 556824, + 61784, + 30008, + 44848, + 81816, + 31528, + 32768, + 12800, + 27000, + 155000, + 14712, + 14712, + 14712, + 15224, + 14712, + 15224, + 14712, + 14712, + 14712, + 15736, + 14712, + 13688, + 14200, + 122232, + 14200, + 14200, + 14200, + 14200, + 14200, + 14200, + 14712, + 13688, + 14200, + 14200, + 15224, + 14200, + 13688, + 13176, + 978296, + 1580408, + 134520, + 15224, + 15736, + 15736, + 15736, + 15736, + 15736, + 16248, + 15736, + 15736, + 15224, + 16760, + 15224, + 14712, + 15224, + 171384, + 175480, + 159096, + 171384, + 179576, + 171384, + 191864, + 175480, + 175480, + 171384, + 216440, + 167288, + 155000, + 155000, + 40312, + 27000, + 48504, + 14200, + 14200, + 14712, + 14200, + 14200, + 14200, + 14712, + 13688, + 14200, + 14200, + 14712, + 14712, + 13688, + 14200, + 52600, + 56696, + 146704, + 21792, + 286720, + 15360, + 13824, + 7168, + 7168, + 421888, + 36864, + 36864, + 110592, + 4608, + 4096, + 29576, + 29576, + 29576, + 29576, + 40840, + 30088, + 32136, + 27368, + 27528, + 315392, + 381, + 3072, + 1318, + 3072, + 375, + 3072, + 378, + 3072, + 598016, + 53248, + 32768, + 110592, + 43696, + 5283840, + 196608, + 139264, + 397312, + 249856, + 163840, + 864256, + 372736, + 532480, + 36864, + 5632, + 110592, + 5120, + 129664, + 110592, + 10752, + 3203072, + 163840, + 45056, + 57344, + 8192, + 425984, + 81920, + 28672, + 49152, + 671744, + 61440, + 53248, + 2879488, + 229376, + 15360, + 397312, + 684032, + 57344, + 110592, + 356352, + 696320, + 462848, + 53248, + 163840, + 11776, + 98304, + 470240, + 47328, + 745472, + 36864, + 974848, + 397312, + 5062656, + 544768, + 401408, + 290816, + 36864, + 188416, + 28672, + 40960, + 630784, + 81920, + 6144, + 24576, + 32768, + 446464, + 65536, + 126976, + 53248, + 131072, + 11776, + 389120, + 3010560, + 278528, + 253952, + 143360, + 13824, + 258048, + 77824, + 237568, + 16896, + 212992, + 307200, + 32768, + 970752, + 131072, + 11776, + 98304, + 270336, + 28672, + 5992448, + 73728, + 36864, + 491520, + 32768, + 569344, + 69632, + 114688, + 40960, + 692224, + 65536, + 28672, + 77824, + 3584, + 229376, + 32768, + 4096, + 16896, + 139264, + 131072, + 11264, + 15360, + 1282048, + 335872, + 49152, + 634880, + 835584, + 81920, + 98304, + 622592, + 61440, + 7168, + 839680, + 81920, + 5025792, + 434176, + 12288, + 3584, + 1142784, + 188416, + 1630208, + 311296, + 540672, + 36864, + 507904, + 102400, + 2056192, + 139264, + 167936, + 172032, + 4096, + 380928, + 8192, + 40960, + 4096, + 98304, + 7168, + 10584, + 81080, + 23376, + 1257472, + 90112, + 94208, + 5120, + 150168, + 25600, + 2553856, + 69120, + 220672, + 51712, + 1349120, + 1349120, + 167424, + 786432, + 515584, + 291328, + 1681920, + 33792, + 102400, + 102912, + 60928, + 44032, + 631296, + 44544, + 11776, + 408576, + 138752, + 1849856, + 13824, + 33280, + 44544, + 17920, + 11776, + 9153536, + 150016, + 13824, + 24576, + 293888, + 651264, + 818176, + 17920, + 141312, + 64512, + 583168, + 58368, + 830976, + 109568, + 65536, + 24576, + 771584, + 1545728, + 1540608, + 11776, + 144896, + 11776, + 18944, + 67584, + 65536, + 17920, + 848896, + 43520, + 30720, + 27648, + 1935872, + 127488, + 11264, + 44032, + 583168, + 11264, + 67584, + 632320, + 38400, + 8470528, + 127488, + 257536, + 30720, + 111104, + 286720, + 1711616, + 64000, + 240128, + 22016, + 86528, + 174080, + 16896, + 531456, + 17920, + 23552, + 20992, + 20992, + 22528, + 20992, + 19456, + 19456, + 20992, + 20992, + 11500544, + 12260864, + 368128, + 14345216, + 29184, + 29184, + 84480, + 64512, + 925696, + 8003072, + 141312, + 978432, + 2305024, + 6658048, + 1116160, + 1593344, + 1059328, + 777216, + 214016, + 627200, + 11935232, + 1840640, + 12438528, + 5466624, + 86528, + 98816, + 3358720, + 145408, + 126976, + 73728, + 78848, + 94208, + 348672, + 76288, + 2347008, + 2347008, + 2176512, + 237056, + 1130496, + 416768, + 713216, + 1126400, + 61440, + 163328, + 158208, + 61440, + 2173952, + 681472, + 33792, + 15580160, + 16565248, + 463360, + 19202560, + 47104, + 116736, + 89088, + 10677248, + 192512, + 1320448, + 3324416, + 8741888, + 1639936, + 2320384, + 1482752, + 11899392, + 1028608, + 296448, + 921600, + 15390720, + 2292224, + 17390080, + 6968320, + 125952, + 136192, + 4978176, + 193536, + 169984, + 104960, + 43520, + 300, + 112128, + 536, + 110592, + 536, + 112128, + 536, + 2664448, + 1060, + 2664448, + 1060, + 2664448, + 1060, + 71168, + 572, + 72192, + 572, + 71168, + 572, + 131072, + 2416, + 82944, + 3084, + 12800, + 3508, + 15872, + 3576, + 1321984, + 1364, + 653312, + 2124, + 18492416, + 2048, + 6656, + 344, + 1219584, + 416, + 548864, + 2444, + 6656, + 344, + 11776, + 1008, + 54272, + 404, + 63488, + 404, + 64000, + 404, + 624640, + 5752, + 624640, + 5720, + 89600, + 1736, + 164352, + 1096, + 178688, + 1096, + 16384, + 352, + 326656, + 1368, + 315392, + 1368, + 699392, + 1004, + 54784, + 360, + 75776, + 352, + 69120, + 352, + 61440, + 352, + 62976, + 352, + 61952, + 352, + 53760, + 360, + 61440, + 352, + 62464, + 352, + 33280, + 320, + 23552, + 368, + 26112, + 360, + 26624, + 324, + 26112, + 360, + 27648, + 320, + 27136, + 320, + 27136, + 320, + 28160, + 360, + 27648, + 320, + 27648, + 320, + 27648, + 320, + 33280, + 320, + 27648, + 320, + 26112, + 360, + 29696, + 320, + 31232, + 360, + 26112, + 360, + 27648, + 320, + 25088, + 328, + 27648, + 320, + 26624, + 320, + 25088, + 328, + 25088, + 328, + 25088, + 328, + 27648, + 320, + 26112, + 360, + 27136, + 320, + 27136, + 320, + 27648, + 320, + 23552, + 368, + 26624, + 320, + 25600, + 320, + 25600, + 320, + 29696, + 320, + 27648, + 320, + 26624, + 324, + 1927168, + 928, + 2015232, + 928, + 414720, + 588, + 426496, + 588, + 327680, + 708, + 1290240, + 2608, + 83456, + 308, + 86016, + 308, + 109056, + 308, + 100352, + 348, + 86016, + 308, + 109056, + 308, + 83968, + 308, + 87040, + 308, + 76800, + 312, + 87040, + 308, + 88576, + 348, + 79360, + 348, + 79872, + 308, + 75264, + 312, + 87552, + 308, + 76800, + 312, + 86528, + 308, + 88064, + 308, + 80384, + 348, + 88576, + 308, + 83456, + 308, + 79872, + 308, + 96768, + 308, + 86528, + 308, + 78848, + 348, + 82944, + 308, + 88576, + 308, + 87552, + 308, + 79360, + 348, + 69120, + 352, + 83968, + 308, + 88064, + 308, + 82944, + 308, + 70144, + 352, + 80384, + 348, + 75264, + 312, + 96768, + 308, + 10240, + 364, + 9728, + 364, + 9728, + 364, + 9728, + 364, + 9728, + 364, + 9728, + 364, + 10240, + 364, + 9216, + 368, + 9216, + 368, + 163840, + 8280, + 242176, + 2980, + 210432, + 360, + 244224, + 360, + 209408, + 360, + 212480, + 360, + 192512, + 364, + 206336, + 360, + 224768, + 360, + 193536, + 364, + 206336, + 360, + 872960, + 1232, + 1231872, + 1056, + 809472, + 888, + 809472, + 992, + 3544576, + 2160, + 3544576, + 2056, + 3580928, + 2220, + 2396160, + 2568, + 145408, + 316, + 160256, + 316, + 145408, + 316, + 195584, + 364, + 140288, + 316, + 126464, + 324, + 144896, + 316, + 160256, + 316, + 139264, + 316, + 216064, + 356, + 212992, + 356, + 144896, + 316, + 139264, + 316, + 140288, + 316, + 196608, + 364, + 132608, + 316, + 128000, + 324, + 249344, + 356, + 181760, + 316, + 142848, + 316, + 228864, + 356, + 144896, + 316, + 147456, + 316, + 141824, + 316, + 144896, + 316, + 214528, + 356, + 210944, + 356, + 181760, + 316, + 140288, + 320, + 142848, + 316, + 141824, + 316, + 210432, + 356, + 128000, + 324, + 132608, + 316, + 140288, + 320, + 126464, + 324, + 147456, + 316, + 26624, + 364, + 24576, + 364, + 22016, + 368, + 29696, + 364, + 22528, + 368, + 24576, + 364, + 24576, + 364, + 24576, + 364, + 25088, + 364, + 2791936, + 2000, + 161280, + 1048, + 169472, + 1048, + 5100032, + 2612, + 5506560, + 1728, + 8192, + 380, + 101888, + 5804, + 104960, + 3332, + 99328, + 2872, + 8704, + 388, + 35328, + 368, + 11244032, + 5592, + 11244032, + 5592, + 10870784, + 5256, + 10744320, + 7916, + 540160, + 744, + 408064, + 720, + 58368, + 4604, + 58368, + 4604, + 52736, + 5008, + 1554432, + 4636, + 2222080, + 6712, + 7168, + 384, + 6656, + 400, + 6656, + 372, + 272384, + 3816, + 235008, + 2760, + 8192, + 368, + 625664, + 5764, + 563200, + 6720, + 9728, + 392, + 7168, + 404, + 415744, + 4688, + 1320960, + 6228, + 7168, + 376, + 1756160, + 4992, + 1961472, + 6552, + 244736, + 360, + 2317824, + 6480, + 1916928, + 4832, + 6656, + 400, + 8192, + 372, + 7168, + 372, + 405504, + 4152, + 411136, + 5580, + 83456, + 1784, + 84992, + 2280, + 6943744, + 6676, + 8638976, + 9976, + 16384, + 368, + 366080, + 5696, + 294912, + 6460, + 366080, + 5696, + 8192, + 380, + 384000, + 364, + 622592, + 8296, + 564224, + 7396, + 359936, + 4004, + 700416, + 6276, + 7168, + 372, + 1648640, + 6952, + 1498624, + 4620, + 38912, + 360, + 6656, + 396, + 8255488, + 5060, + 7553024, + 5084, + 8255488, + 5060, + 8048640, + 8684, + 4229120, + 7476, + 4243456, + 9716, + 57344, + 372, + 483840, + 4016, + 484352, + 4468, + 20480, + 364, + 6948352, + 8444, + 5271552, + 5800, + 9728, + 364, + 12800, + 376, + 273920, + 2708, + 114176, + 2116, + 256512, + 6212, + 186368, + 6428, + 1016320, + 3228, + 379904, + 7256, + 508928, + 6568, + 470528, + 5128, + 468480, + 3764, + 13023232, + 5424, + 12046336, + 7648, + 13023232, + 5424, + 23040, + 352, + 1604096, + 700, + 625664, + 708, + 608256, + 656, + 722944, + 3524, + 1263104, + 4232, + 243200, + 3632, + 60928, + 720, + 1249280, + 2216, + 233472, + 1212, + 110080, + 1104, + 1939456, + 2864, + 289792, + 1632, + 241664, + 2956, + 535040, + 3040, + 25088, + 688, + 38400, + 544, + 24576, + 1816, + 10240, + 600, + 10752, + 1516, + 68096, + 1056, + 75776, + 424, + 723968, + 2076, + 1162240, + 1976, + 117248, + 760, + 117248, + 3696, + 140800, + 892, + 998400, + 708, + 1829888, + 10068, + 3081216, + 10696, + 1305600, + 4820, + 172032, + 1524, + 7168, + 304, + 578048, + 2232, + 505856, + 2232, + 7168, + 288, + 149504, + 856, + 38400, + 464, + 196608, + 1268, + 198656, + 1268, + 752128, + 2188, + 799232, + 2188, + 164864, + 1912, + 197120, + 1912, + 261632, + 1552, + 259584, + 1552, + 601600, + 1372, + 583680, + 1372, + 19968, + 832, + 19968, + 832, + 375296, + 1556, + 373248, + 1556, + 44032, + 2192, + 44032, + 1696, + 148992, + 1272, + 148992, + 1272, + 71168, + 988, + 71168, + 988, + 108032, + 2516, + 96768, + 1772, + 1223680, + 3812, + 1895936, + 4020, + 560128, + 708, + 392192, + 656, + 1083904, + 2364, + 294400, + 1948, + 446464, + 1620, + 451072, + 1620, + 1251840, + 432, + 611840, + 5216, + 47104, + 620, + 53248, + 620, + 501760, + 3572, + 200192, + 3312, + 2404352, + 580, + 1298944, + 876, + 783360, + 772, + 111104, + 504, + 491008, + 748, + 580096, + 1784, + 6215680, + 696, + 1199616, + 1832, + 1252864, + 2000, + 314880, + 1076, + 346112, + 2008, + 11637248, + 2264, + 1553920, + 3336, + 311296, + 1268, + 259584, + 1652, + 1376256, + 2196, + 1782784, + 3148, + 516096, + 1248, + 1872896, + 3692, + 377856, + 316, + 96256, + 608, + 62464, + 804, + 338432, + 1500, + 296960, + 1512, + 396800, + 1652, + 329216, + 1424, + 859648, + 2168, + 2535424, + 3688, + 1282560, + 2192, + 4596736, + 5784, + 2210816, + 2420, + 139776, + 760, + 4002304, + 4776, + 4817920, + 3500, + 26624, + 352, + 488448, + 2188, + 1078272, + 5188, + 808448, + 5548, + 76288, + 1496, + 50688, + 688, + 603648, + 4856, + 600064, + 3928, + 49152, + 368, + 491008, + 3236, + 487936, + 3120, + 144896, + 2216, + 1013248, + 6708, + 805888, + 6448, + 114688, + 320, + 104448, + 328, + 103936, + 328, + 112128, + 320, + 113152, + 320, + 114688, + 320, + 110592, + 320, + 113664, + 320, + 136192, + 320, + 122880, + 320, + 110592, + 320, + 111616, + 324, + 113152, + 320, + 1485824, + 4928, + 690688, + 10996, + 737280, + 12228, + 108032, + 924, + 101888, + 740, + 6156288, + 4976, + 6376448, + 6200, + 548352, + 1492, + 441344, + 1168, + 1564672, + 8060, + 1538560, + 7084, + 169472, + 368, + 521216, + 4180, + 133632, + 2348, + 134144, + 2348, + 1046016, + 8576, + 1351680, + 4732, + 1823232, + 1872, + 829440, + 3820, + 781312, + 3720, + 5850112, + 10748, + 4831232, + 10408, + 119296, + 2012, + 118784, + 2012, + 19456, + 1652, + 9411584, + 320, + 9260032, + 320, + 367616, + 4564, + 360960, + 4372, + 69632, + 7804, + 68608, + 6724, + 1329152, + 7692, + 1319424, + 6936, + 2029568, + 3088, + 115200, + 4584, + 139776, + 5012, + 2570752, + 1576, + 43520, + 2792, + 42496, + 2792, + 842752, + 7988, + 2548224, + 11824, + 5488128, + 1760, + 260096, + 7020, + 706048, + 3612, + 740352, + 5040, + 16384, + 380, + 235520, + 2536, + 264704, + 2652, + 4364288, + 8256, + 4176896, + 6204, + 16559104, + 3220, + 107008, + 2052, + 1103872, + 3264, + 1098752, + 3056, + 1177600, + 3856, + 1028608, + 3404, + 663552, + 2212, + 665088, + 2396, + 140288, + 560, + 90112, + 2736, + 141824, + 1284, + 6045184, + 13232, + 4726784, + 12052, + 176640, + 4388, + 626688, + 3868, + 621568, + 3868, + 1908224, + 1820, + 704512, + 4976, + 700928, + 4584, + 2672128, + 7920, + 2396160, + 7872, + 4750848, + 4916, + 4597248, + 5928, + 345600, + 7092, + 318976, + 7204, + 31744, + 676, + 32256, + 676, + 593920, + 3824, + 615936, + 4292, + 1656320, + 9112, + 1626112, + 9324, + 177152, + 2020, + 5765632, + 5816, + 56320, + 2228, + 61952, + 2228, + 1814528, + 3612, + 6656, + 380, + 16896, + 456, + 31744, + 844, + 3265024, + 9080, + 4318720, + 11204, + 192512, + 784, + 188416, + 784, + 627200, + 11148, + 681472, + 11644, + 1456640, + 4156, + 306176, + 2064, + 305152, + 1908, + 2410496, + 9624, + 3928576, + 10028, + 3820544, + 9056, + 5166080, + 3392, + 417792, + 1840, + 17920, + 720, + 17920, + 720, + 732672, + 5100, + 832512, + 5676, + 1738752, + 4948, + 1936896, + 452, + 1940480, + 452, + 68608, + 3424, + 68608, + 3428, + 211456, + 4428, + 203776, + 4120, + 467456, + 3208, + 423936, + 4084, + 701952, + 5048, + 474624, + 5268, + 2202112, + 6132, + 136704, + 2432, + 136704, + 2432, + 10240, + 372, + 35840, + 824, + 35840, + 824, + 365568, + 5700, + 365568, + 6176, + 155648, + 2168, + 156160, + 2168, + 24064, + 1004, + 35840, + 1392, + 337408, + 4956, + 341504, + 4688, + 77312, + 5412, + 79360, + 6264, + 2067456, + 10360, + 435712, + 448, + 1226240, + 5600, + 1228800, + 5952, + 16384, + 324, + 16896, + 324, + 16896, + 324, + 16896, + 328, + 15872, + 332, + 16896, + 324, + 17920, + 324, + 16384, + 324, + 16896, + 324, + 16896, + 324, + 15872, + 332, + 16896, + 324, + 18432, + 324, + 99328, + 776, + 2295296, + 8784, + 1970176, + 7988, + 175104, + 7768, + 183808, + 8200, + 23552, + 1244, + 364032, + 4316, + 955904, + 6660, + 1570304, + 8804, + 1491968, + 8444, + 725504, + 528, + 729088, + 528, + 172544, + 928, + 411648, + 7312, + 294400, + 3532, + 313344, + 3416, + 48640, + 984, + 48128, + 984, + 48128, + 1760, + 160768, + 900, + 160768, + 900, + 3207680, + 2016, + 155136, + 4012, + 312320, + 4680, + 287232, + 2888, + 288768, + 2888, + 898560, + 4116, + 894464, + 4116, + 15360, + 364, + 2923520, + 6556, + 103936, + 2668, + 115200, + 4552, + 93696, + 920, + 87552, + 920, + 384512, + 1976, + 374784, + 1976, + 122368, + 2456, + 4463616, + 2788, + 1618432, + 1200, + 69632, + 1672, + 18432, + 328, + 23040, + 1268, + 13824, + 328, + 80384, + 2132, + 363008, + 5288, + 363520, + 5776, + 47616, + 2808, + 206848, + 1952, + 214016, + 2120, + 1050624, + 6560, + 1114112, + 7476, + 1883648, + 1708, + 710144, + 9276, + 937472, + 12088, + 3946496, + 10876, + 3973632, + 11776, + 6656, + 372, + 56832, + 2452, + 2269696, + 10188, + 300544, + 4128, + 299520, + 3920, + 1971712, + 7600, + 1007104, + 3108, + 9728, + 384, + 7168, + 364, + 32256, + 512, + 28672, + 512, + 640000, + 1636, + 24064, + 1976, + 272896, + 3096, + 167936, + 3860, + 1135616, + 4944, + 96768, + 1772, + 1086464, + 10516, + 1362944, + 7912, + 62976, + 656, + 63488, + 656, + 63488, + 656, + 115712, + 320, + 115200, + 320, + 895488, + 9612, + 11264, + 336, + 271360, + 2784, + 394240, + 9496, + 53760, + 784, + 360448, + 6288, + 84480, + 2488, + 588288, + 7708, + 724480, + 7052, + 2811392, + 12416, + 2999296, + 10636, + 34304, + 332, + 38400, + 336, + 38400, + 336, + 1101312, + 4140, + 1100288, + 4072, + 13824, + 332, + 13312, + 332, + 13824, + 332, + 12800, + 336, + 11776, + 372, + 13312, + 332, + 13312, + 332, + 13312, + 332, + 15360, + 332, + 13312, + 332, + 13824, + 332, + 12800, + 336, + 13312, + 332, + 13312, + 332, + 125952, + 3308, + 128000, + 3688, + 431616, + 2860, + 442880, + 3348, + 2013184, + 1400, + 660992, + 5236, + 1506816, + 5124, + 707584, + 3248, + 706560, + 3248, + 1534976, + 572, + 1537536, + 572, + 2013696, + 3344, + 2031616, + 3344, + 57856, + 2364, + 205824, + 1748, + 739328, + 1232, + 741376, + 1232, + 67584, + 1316, + 1430528, + 2832, + 429568, + 620, + 455680, + 620, + 1007616, + 5588, + 1282048, + 2624, + 1265152, + 2628, + 4728320, + 7148, + 4441600, + 6448, + 4735488, + 5152, + 3809280, + 5136, + 4013568, + 5852, + 97280, + 1332, + 978432, + 3564, + 72704, + 2504, + 28672, + 376, + 2288640, + 7400, + 2407936, + 7308, + 901632, + 8740, + 972288, + 9456, + 1052160, + 6812, + 1026560, + 6528, + 1321472, + 9780, + 837120, + 8460, + 288768, + 1108, + 308736, + 1108, + 1195520, + 1948, + 1198080, + 1948, + 181248, + 5368, + 144384, + 4900, + 776704, + 348, + 157696, + 5076, + 60928, + 444, + 60928, + 444, + 209920, + 5220, + 246784, + 756, + 246272, + 756, + 604672, + 8528, + 24064, + 564, + 24064, + 564, + 47104, + 556, + 44032, + 556, + 321536, + 2176, + 332288, + 2176, + 204288, + 2388, + 204800, + 2440, + 107008, + 912, + 82432, + 1448, + 87552, + 1996, + 89088, + 372, + 922112, + 4576, + 508928, + 1944, + 528896, + 1996, + 5450240, + 2776, + 3680768, + 15000, + 3721728, + 16696, + 845312, + 9152, + 3270656, + 4144, + 1602048, + 852, + 1606656, + 852, + 36864, + 564, + 39936, + 564, + 214016, + 3144, + 576000, + 4544, + 580096, + 4468, + 627200, + 6596, + 1623040, + 11196, + 1107456, + 9700, + 2272256, + 13208, + 2289152, + 13624, + 1189376, + 704, + 1203712, + 704, + 470528, + 6616, + 451584, + 5508, + 143360, + 1388, + 143360, + 1388, + 55296, + 2920, + 55808, + 2920, + 82944, + 2444, + 82944, + 2444, + 2862592, + 3168, + 7680, + 372, + 68096, + 1124, + 67584, + 1124, + 1022976, + 1240, + 193024, + 2660, + 193536, + 2660, + 2833408, + 1172, + 195072, + 1564, + 512000, + 2840, + 131072, + 3096, + 313344, + 2100, + 89088, + 2136, + 86016, + 2136, + 1243648, + 1840, + 1280512, + 1840, + 778752, + 1040, + 10240, + 388, + 420864, + 1660, + 3430400, + 5764, + 1161216, + 736, + 2591232, + 9460, + 1972224, + 8028, + 583168, + 6348, + 468992, + 6044, + 553472, + 4172, + 551424, + 3364, + 1433600, + 2712, + 677376, + 2552, + 1743872, + 5072, + 1784320, + 5540, + 1024512, + 2564, + 1110016, + 3572, + 5252096, + 11496, + 5855744, + 13532, + 18944, + 1000, + 12800, + 424, + 149504, + 512, + 9216, + 312, + 84480, + 1308, + 20992, + 992, + 17920, + 996, + 36352, + 1096, + 15872, + 996, + 1478144, + 1736, + 83968, + 432, + 1655296, + 2292, + 1654272, + 2200, + 604160, + 560, + 27136, + 2024, + 43520, + 656, + 21504, + 1316, + 26624, + 324, + 412160, + 2000, + 163840, + 724, + 155136, + 724, + 1257472, + 7584, + 1146368, + 3468, + 18944, + 324, + 2123264, + 5428, + 2144768, + 6080, + 57344, + 896, + 80384, + 1284, + 1655808, + 1172, + 260608, + 1280, + 908800, + 1356, + 532480, + 1196, + 46592, + 340, + 61440, + 300, + 50176, + 300, + 49664, + 300, + 50176, + 300, + 49152, + 300, + 48640, + 300, + 49664, + 300, + 49664, + 300, + 48640, + 300, + 49664, + 300, + 42496, + 340, + 46080, + 300, + 50176, + 300, + 44032, + 304, + 50176, + 300, + 43008, + 340, + 48128, + 300, + 38400, + 344, + 49664, + 300, + 43008, + 340, + 44032, + 340, + 54272, + 300, + 50176, + 300, + 44032, + 304, + 48128, + 300, + 43520, + 340, + 50176, + 300, + 49152, + 300, + 44032, + 304, + 44032, + 304, + 61440, + 300, + 52736, + 340, + 54272, + 300, + 46080, + 300, + 49664, + 300, + 36864, + 344, + 17920, + 308, + 16896, + 312, + 17920, + 308, + 17408, + 308, + 17920, + 308, + 17920, + 308, + 17920, + 308, + 17920, + 308, + 16896, + 312, + 17408, + 308, + 17920, + 308, + 18432, + 308, + 17920, + 308, + 17408, + 308, + 18432, + 308, + 17920, + 308, + 18432, + 308, + 19456, + 308, + 20480, + 308, + 18432, + 308, + 16384, + 312, + 20480, + 308, + 16384, + 312, + 17920, + 308, + 18432, + 308, + 18432, + 308, + 17408, + 308, + 19456, + 308, + 721408, + 1540, + 20518056, + 176, + 25554944, + 176, + 17408, + 1464, + 17408, + 1620, + 78848, + 296, + 118784, + 740, + 2226688, + 1360, + 2097664, + 1360, + 2452992, + 16100, + 2390528, + 15368, + 19825152, + 2436, + 536064, + 1252, + 12187136, + 1832, + 26112, + 2080, + 26112, + 2080, + 26112, + 2084, + 26112, + 2084, + 25088, + 1760, + 25600, + 2088, + 25088, + 2088, + 25088, + 1756, + 25088, + 2084, + 25088, + 2084, + 23552, + 2100, + 23552, + 2100, + 23552, + 2100, + 23552, + 2100, + 168960, + 1064, + 171520, + 1064, + 168960, + 1064, + 6656, + 372, + 7168, + 368, + 12288, + 704, + 14848, + 816, + 3638784, + 856, + 3638272, + 856, + 3700736, + 856, + 11264, + 484, + 68096, + 732, + 68096, + 732, + 68096, + 732, + 118272, + 924, + 933376, + 1352, + 933376, + 1352, + 942080, + 1352, + 683008, + 10132, + 768512, + 3312, + 768000, + 3312, + 230912, + 1488, + 200704, + 1876, + 148480, + 2048, + 391680, + 2204, + 391680, + 2204, + 395264, + 2204, + 13061632, + 620, + 10336768, + 620, + 5653504, + 2256, + 6656, + 348, + 7680, + 304, + 1251840, + 1980, + 796672, + 1984, + 812544, + 2324, + 8192, + 316, + 8704, + 304, + 181248, + 3020, + 252928, + 728, + 176128, + 2856, + 9728, + 356, + 174080, + 3208, + 10752, + 360, + 1075712, + 556, + 9216, + 360, + 150016, + 1260, + 993792, + 864, + 27648, + 300, + 9489920, + 900, + 7684608, + 900, + 7966720, + 1540, + 2586624, + 708, + 1241600, + 1180, + 1172992, + 1336, + 1838592, + 1196, + 10906624, + 2396, + 11145728, + 2580, + 31232, + 312, + 6656, + 360, + 10752, + 316, + 10240, + 316, + 8192, + 308, + 8192, + 308, + 9216, + 320, + 8704, + 312, + 460800, + 768, + 1172992, + 752, + 247808, + 1088, + 1645568, + 584, + 8704, + 308, + 391680, + 536, + 769536, + 1100, + 243200, + 8192, + 304, + 3034624, + 1732, + 8192, + 296, + 283648, + 424, + 8192, + 316, + 8704, + 316, + 6656, + 352, + 6656, + 360, + 19968, + 316, + 8192, + 304, + 9216, + 304, + 9216, + 300, + 8192, + 296, + 10752, + 308, + 26242048, + 2764, + 1180672, + 764, + 36864, + 592, + 522752, + 536, + 9728, + 304, + 128000, + 420, + 273920, + 300, + 26624, + 316, + 8704, + 312, + 1382912, + 3572, + 2391552, + 3368, + 1316864, + 3200, + 8192, + 312, + 8704, + 304, + 8192, + 316, + 636928, + 1956, + 251904, + 752, + 8192, + 312, + 35328, + 1368, + 811520, + 1276, + 306176, + 644, + 408576, + 1240, + 12288, + 316, + 102400, + 504, + 8192, + 308, + 2842112, + 1100, + 19456, + 300, + 32256, + 332, + 31744, + 316, + 9216, + 324, + 9728, + 324, + 14336, + 308, + 36352, + 560, + 9728, + 320, + 805888, + 912, + 221696, + 932, + 802304, + 592, + 19939840, + 3948, + 8192, + 304, + 8192, + 316, + 886784, + 3224, + 1123840, + 3024, + 8192, + 316, + 9216, + 308, + 8704, + 300, + 8704, + 308, + 719360, + 924, + 702464, + 972, + 13918208, + 4076, + 13918208, + 3972, + 257024, + 432, + 94720, + 600, + 1904640, + 1580, + 122880, + 1372, + 122368, + 1252, + 13563392, + 1720, + 4451328, + 2520, + 2031616, + 572, + 7577088, + 748, + 15872, + 636, + 10240, + 308, + 8704, + 308, + 9728, + 308, + 395776, + 872, + 8704, + 304, + 9728, + 300, + 240640, + 2288, + 1309184, + 3864, + 93696, + 292, + 93696, + 292, + 95232, + 292, + 635392, + 572, + 629248, + 572, + 629248, + 572, + 474624, + 3184, + 130560, + 2420, + 84480, + 3088, + 153088, + 3444, + 227840, + 3116, + 339456, + 1420, + 708096, + 5904, + 999936, + 7792, + 1446400, + 2340, + 1404928, + 2340, + 4345344, + 14076, + 4835328, + 15864, + 411136, + 1556, + 411648, + 1556, + 605696, + 1076, + 605184, + 1024, + 13824, + 808, + 183296, + 1680, + 185856, + 1680, + 347136, + 1428, + 141824, + 340, + 222720, + 344, + 721920, + 2112, + 4110336, + 1348, + 248832, + 3588, + 249344, + 3680, + 98816, + 2896, + 99328, + 3012, + 7753728, + 852, + 7665664, + 852, + 7665664, + 852, + 154112, + 892, + 155648, + 892, + 154112, + 892, + 457216, + 1660, + 133632, + 2716, + 137216, + 888, + 136704, + 888, + 136704, + 888, + 118272, + 1832, + 122368, + 736, + 135680, + 736, + 135680, + 736, + 63488, + 300, + 80384, + 572, + 81920, + 572, + 80384, + 572, + 104960, + 3084, + 166912, + 2416, + 14848, + 3508, + 18944, + 3576, + 267264, + 300, + 1542656, + 1364, + 765952, + 2124, + 23862784, + 2048, + 596480, + 2444, + 72704, + 404, + 72704, + 404, + 61440, + 404, + 129024, + 504, + 105984, + 540, + 108032, + 1736, + 28160, + 320, + 30720, + 320, + 26112, + 328, + 28160, + 324, + 28160, + 320, + 28160, + 324, + 29184, + 320, + 26112, + 328, + 28672, + 320, + 28672, + 320, + 26112, + 328, + 27136, + 320, + 28672, + 320, + 34304, + 320, + 28672, + 320, + 27648, + 320, + 27648, + 320, + 29184, + 320, + 26112, + 328, + 28672, + 320, + 27136, + 320, + 34304, + 320, + 29184, + 320, + 30720, + 320, + 28672, + 320, + 29184, + 320, + 28672, + 320, + 28672, + 320, + 2295296, + 928, + 2386432, + 928, + 561664, + 588, + 445952, + 708, + 577536, + 588, + 88064, + 308, + 88576, + 308, + 97792, + 308, + 81408, + 308, + 88064, + 308, + 77824, + 312, + 89088, + 308, + 97792, + 308, + 89088, + 308, + 83968, + 308, + 87040, + 308, + 84480, + 308, + 85504, + 308, + 81408, + 308, + 89600, + 308, + 110080, + 308, + 87552, + 308, + 110080, + 308, + 76288, + 312, + 84480, + 308, + 89600, + 308, + 83968, + 308, + 87552, + 308, + 88576, + 308, + 87040, + 308, + 76288, + 312, + 85504, + 308, + 77824, + 312, + 288768, + 2980, + 193536, + 8280, + 1096192, + 1232, + 1049088, + 992, + 1049088, + 888, + 4356608, + 2160, + 4356608, + 2056, + 2875904, + 2568, + 161792, + 316, + 127488, + 324, + 146432, + 316, + 148480, + 316, + 146432, + 316, + 144384, + 316, + 141312, + 316, + 148480, + 316, + 145920, + 316, + 129024, + 324, + 141312, + 316, + 182784, + 316, + 141824, + 320, + 142848, + 316, + 127488, + 324, + 146432, + 316, + 133632, + 316, + 161792, + 316, + 140288, + 316, + 144384, + 316, + 129024, + 324, + 133632, + 316, + 140288, + 316, + 141824, + 320, + 142848, + 316, + 145920, + 316, + 146432, + 316, + 182784, + 316, + 190464, + 1048, + 6645760, + 2612, + 7172608, + 1728, + 130048, + 5804, + 129536, + 3332, + 14091776, + 5592, + 13456896, + 7916, + 13585920, + 5256, + 671232, + 744, + 65536, + 5008, + 71168, + 4604, + 2784256, + 6712, + 339456, + 3816, + 709120, + 6720, + 1713152, + 6228, + 2441216, + 6552, + 2871296, + 6480, + 541184, + 5580, + 93696, + 2280, + 11535872, + 9976, + 473600, + 5696, + 386048, + 6460, + 800768, + 8296, + 953344, + 6276, + 2007040, + 6952, + 11059712, + 5060, + 10085888, + 5084, + 10799104, + 8684, + 5849600, + 9716, + 633344, + 4468, + 9155072, + 8444, + 327168, + 2708, + 235008, + 6428, + 1310720, + 3228, + 476160, + 7256, + 634368, + 5128, + 16174592, + 5424, + 15025152, + 7648, + 1963008, + 700, + 308224, + 3632, + 82944, + 720, + 1541632, + 2216, + 304640, + 1212, + 132608, + 1104, + 2478592, + 2864, + 383488, + 1632, + 302592, + 2956, + 29184, + 688, + 50176, + 544, + 30208, + 1816, + 12800, + 600, + 13312, + 1516, + 79872, + 1056, + 94720, + 424, + 148992, + 760, + 148992, + 3696, + 221696, + 892, + 225280, + 1524, + 8704, + 304, + 8192, + 288, + 188416, + 856, + 53760, + 464, + 1364480, + 2364, + 91648, + 500, + 1593856, + 432, + 2812416, + 580, + 135680, + 504, + 664064, + 328, + 626688, + 748, + 548864, + 1120, + 720896, + 1784, + 1544192, + 1832, + 1606144, + 2000, + 4137472, + 3300, + 7470592, + 2380, + 387584, + 1076, + 440320, + 2008, + 739840, + 1608, + 13512704, + 2264, + 1871360, + 3336, + 381440, + 1268, + 332288, + 1652, + 1762304, + 2196, + 214016, + 1252, + 2290176, + 3148, + 671232, + 1248, + 1467392, + 2232, + 80896, + 608, + 2438144, + 3692, + 489984, + 316, + 117760, + 608, + 437760, + 1500, + 377344, + 1512, + 175616, + 760, + 744448, + 3928, + 111616, + 320, + 112640, + 324, + 123904, + 320, + 113152, + 320, + 114176, + 320, + 114176, + 320, + 111616, + 320, + 116224, + 320, + 105472, + 328, + 114688, + 320, + 105472, + 328, + 137216, + 320, + 115712, + 320, + 882688, + 12228, + 155136, + 2348, + 144384, + 2012, + 23040, + 1652, + 89600, + 7804, + 161280, + 5012, + 754688, + 3868, + 34304, + 676, + 816128, + 11644, + 372736, + 1908, + 551424, + 5268, + 86016, + 6264, + 16896, + 332, + 17920, + 324, + 17920, + 324, + 18944, + 324, + 17920, + 324, + 18432, + 324, + 19968, + 324, + 17408, + 332, + 17920, + 324, + 17920, + 324, + 17920, + 324, + 17920, + 324, + 17920, + 328, + 241152, + 8200, + 919040, + 528, + 913408, + 528, + 371712, + 2888, + 1105920, + 4116, + 2182656, + 1708, + 1213440, + 12088, + 524288, + 9496, + 460800, + 6288, + 3119616, + 12416, + 47104, + 336, + 14848, + 332, + 16384, + 332, + 14848, + 332, + 13824, + 336, + 13824, + 336, + 14336, + 332, + 14336, + 332, + 14336, + 332, + 14848, + 332, + 14848, + 332, + 14848, + 332, + 14336, + 332, + 14336, + 332, + 913920, + 3248, + 2378240, + 3344, + 998912, + 1232, + 60416, + 556, + 233472, + 2388, + 4872192, + 16696, + 1746432, + 852, + 1432576, + 704, + 1444352, + 704, + 98304, + 2444, + 218112, + 2660, + 4998144, + 716, + 5272064, + 716, + 240128, + 1564, + 1604608, + 1840, + 1565696, + 1840, + 1565696, + 1840, + 1497088, + 736, + 22016, + 1000, + 14848, + 424, + 1531392, + 2684, + 188928, + 512, + 10240, + 312, + 97792, + 1308, + 24064, + 992, + 20480, + 996, + 44544, + 1096, + 18432, + 996, + 31744, + 2024, + 51712, + 656, + 25600, + 1316, + 35328, + 324, + 611328, + 1444, + 526848, + 2000, + 26112, + 324, + 7213568, + 1568, + 2115072, + 1172, + 316416, + 1280, + 646144, + 1196, + 1097216, + 1356, + 47104, + 300, + 50688, + 300, + 51200, + 300, + 51200, + 300, + 51200, + 300, + 45056, + 304, + 49664, + 300, + 47104, + 300, + 51200, + 300, + 49664, + 300, + 50688, + 300, + 62464, + 300, + 51200, + 300, + 50688, + 300, + 50688, + 300, + 49664, + 300, + 55296, + 300, + 45056, + 304, + 51200, + 300, + 45056, + 304, + 50688, + 300, + 55296, + 300, + 49664, + 300, + 51200, + 300, + 62464, + 300, + 51200, + 300, + 50688, + 300, + 45056, + 304, + 19456, + 308, + 19456, + 308, + 18944, + 308, + 19456, + 308, + 18944, + 308, + 18944, + 308, + 22016, + 308, + 22016, + 308, + 20992, + 308, + 18944, + 308, + 19456, + 308, + 18432, + 308, + 18944, + 308, + 19456, + 308, + 18944, + 308, + 19456, + 308, + 19456, + 308, + 17920, + 312, + 18944, + 308, + 18432, + 308, + 19456, + 308, + 17920, + 312, + 19456, + 308, + 17920, + 312, + 20992, + 308, + 19456, + 308, + 18944, + 308, + 17920, + 312, + 910848, + 1540, + 22551712, + 176, + 98816, + 296, + 139264, + 740, + 2679296, + 1360, + 2817024, + 1360, + 23356416, + 2436, + 650752, + 1252, + 14530048, + 1832, + 29696, + 2080, + 29184, + 2084, + 26624, + 2100, + 13312, + 484, + 73728, + 732, + 73728, + 732, + 74240, + 732, + 143872, + 924, + 303104, + 1488, + 12712448, + 620, + 7215616, + 2256, + 9216, + 304, + 1687552, + 1980, + 1063424, + 1984, + 1082880, + 2324, + 9728, + 316, + 10752, + 304, + 349696, + 728, + 1405440, + 556, + 169984, + 1260, + 1175552, + 864, + 31744, + 300, + 10212352, + 900, + 9414656, + 1540, + 3118080, + 708, + 1555456, + 1180, + 1391616, + 1336, + 2229760, + 1196, + 12569600, + 2396, + 35328, + 312, + 12288, + 316, + 12800, + 316, + 9216, + 308, + 9216, + 308, + 10240, + 320, + 10240, + 312, + 1125376, + 1128, + 547328, + 768, + 1424384, + 752, + 295424, + 1088, + 1996288, + 584, + 10752, + 308, + 486912, + 536, + 1000960, + 1100, + 334336, + 9216, + 304, + 9728, + 296, + 338944, + 424, + 9216, + 316, + 10240, + 316, + 22528, + 316, + 9216, + 304, + 10752, + 304, + 10752, + 300, + 9216, + 296, + 12800, + 308, + 2764, + 1433600, + 764, + 642048, + 536, + 314880, + 300, + 10240, + 312, + 1701888, + 3200, + 3041280, + 3368, + 1785344, + 3572, + 9216, + 312, + 10240, + 304, + 9728, + 316, + 867840, + 1948, + 300544, + 752, + 10240, + 312, + 41984, + 1368, + 932352, + 1276, + 345600, + 644, + 517120, + 1240, + 15360, + 316, + 111104, + 504, + 9216, + 308, + 3349504, + 1100, + 24576, + 300, + 36864, + 332, + 36352, + 316, + 10752, + 324, + 10752, + 324, + 16896, + 308, + 41472, + 560, + 11264, + 320, + 950272, + 912, + 260608, + 932, + 989184, + 592, + 24750080, + 3948, + 9216, + 304, + 9216, + 316, + 917504, + 3224, + 1547776, + 3024, + 9216, + 316, + 10752, + 308, + 10752, + 300, + 9728, + 308, + 872448, + 924, + 814592, + 972, + 16530944, + 4076, + 16530944, + 3972, + 290304, + 432, + 111616, + 600, + 2162688, + 1580, + 15892480, + 1720, + 5285888, + 2520, + 2543616, + 572, + 8998912, + 748, + 18944, + 636, + 10240, + 308, + 11776, + 308, + 11264, + 308, + 480256, + 872, + 10240, + 304, + 11264, + 300, + 267264, + 2288, + 108544, + 292, + 107008, + 292, + 107008, + 292, + 841728, + 572, + 847872, + 572, + 842240, + 572, + 219136, + 304, + 105984, + 3088, + 167424, + 2420, + 292352, + 3116, + 195584, + 3444, + 1740288, + 2340, + 1792000, + 2340, + 5146624, + 1348, + 176640, + 892, + 174592, + 892, + 174592, + 892, + 611840, + 1660, + 154112, + 888, + 155648, + 888, + 154112, + 888, + 197632, + 1832, + 301609, + 338048, + 64512, + 135, + 16384, + 3170304, + 1474560, + 1474560, + 16384, + 3170304, + 1024, + 1024, + 4096, + 77728, + 77728, + 4669, + 1224608, + 1209752, + 115616, + 76696, + 76704, + 45472, + 75680, + 75680, + 45472, + 79264, + 79264, + 45984, + 80288, + 80288, + 46488, + 74144, + 74144, + 74136, + 74144, + 44960, + 77728, + 77728, + 45984, + 77728, + 77728, + 75168, + 75168, + 76704, + 76704, + 45472, + 79264, + 79264, + 79256, + 79264, + 45984, + 76704, + 76696, + 78752, + 78752, + 45984, + 77216, + 77216, + 45472, + 67488, + 67488, + 42912, + 66976, + 66976, + 42912, + 75680, + 75680, + 75680, + 75672, + 1056664, + 75680, + 75680, + 45472, + 77728, + 77728, + 45472, + 77728, + 77720, + 45976, + 76704, + 76704, + 45472, + 76696, + 76704, + 45984, + 54176, + 76192, + 76192, + 77208, + 77216, + 44960, + 77216, + 77216, + 76704, + 76704, + 77216, + 77216, + 76192, + 76192, + 44960, + 75168, + 75168, + 45472, + 77208, + 77216, + 4662, + 6534, + 63904, + 63904, + 42400, + 63904, + 63904, + 42400, + 3695719, + 3878410, + 1985867, + 2373000, + 174959, + 177414, + 143754, + 145419, + 162331, + 164347, + 154427, + 156245, + 44859, + 85862, + 86178, + 49091, + 77728, + 395314, + 1, + 95648, + 99744, + 76704, + 45472, + 75672, + 45472, + 79264, + 45984, + 80288, + 46496, + 74144, + 74144, + 44960, + 77728, + 45984, + 77720, + 75160, + 76696, + 45472, + 79264, + 79264, + 45984, + 76696, + 78752, + 45976, + 77208, + 45472, + 67488, + 42904, + 66976, + 42912, + 75672, + 75680, + 812440, + 75672, + 45472, + 77728, + 45472, + 77728, + 45984, + 76704, + 45472, + 76696, + 45984, + 83352, + 54176, + 76184, + 77208, + 44960, + 77216, + 76704, + 77216, + 76192, + 44952, + 75168, + 45472, + 77216, + 4662, + 63904, + 42400, + 63904, + 42392, + 92576, + 12704, + 12192, + 67584, + 1477720, + 7680, + 7680, + 962048, + 958976, + 6480, + 34390, + 136606, + 136606, + 156118, + 556304, + 556304, + 556304, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 8774, + 8774, + 8774, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 136606, + 556304, + 556304, + 556304, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 8774, + 2560, + 1024, + 15360, + 10752, + 2560, + 12288, + 52736, + 6656, + 3072, + 8582, + 3, + 16777270, + 4606392, + 70144, + 808, + 1233, + 2222, + 2090, + 1120, + 1317, + 2038, + 2574, + 1544, + 1215, + 1277, + 2053, + 2088, + 1560, + 1815, + 2177, + 1483, + 846, + 1590, + 2724, + 1676, + 2430, + 1197, + 1429, + 1310, + 1873, + 1914, + 13868, + 8116, + 5632, + 11423, + 78336, + 6996, + 5120, + 1151, + 1848, + 1898, + 1137, + 1802, + 2593, + 4506, + 2839, + 410, + 1093, + 610, + 848, + 422, + 12109, + 5756, + 2092, + 5260, + 3584, + 5120, + 2620, + 80896, + 4680, + 3584, + 229, + 12800, + 592, + 1963, + 1068, + 1080, + 701, + 369, + 711, + 1485, + 540, + 613, + 3148, + 1390, + 2432, + 1488, + 1282, + 711, + 531, + 682, + 468, + 3728, + 9216, + 15256, + 152064, + 3280, + 8192, + 1219, + 2204, + 382, + 1470, + 734, + 347, + 2381, + 370, + 1069, + 832, + 1943, + 11026, + 1535, + 714, + 17920, + 11776, + 1141, + 1029, + 610, + 384, + 141662, + 8016, + 13824, + 29602, + 176128, + 7380, + 11776, + 22718, + 1636, + 2557, + 2804, + 1756, + 1223, + 7091, + 6746, + 1323, + 1139, + 5106, + 1825, + 1624, + 15593, + 2462, + 1932, + 2563, + 1575, + 3258, + 1591, + 1921, + 2920, + 2226, + 20053, + 10426, + 894, + 4608, + 7152, + 78848, + 814, + 4096, + 1050, + 2221, + 5322, + 858, + 779, + 5310, + 1861, + 4670, + 6144, + 17152, + 17920, + 4106, + 5632, + 1200, + 1200, + 1192, + 1204, + 3594, + 1206, + 1192, + 1131, + 1131, + 1123, + 1135, + 2313, + 1137, + 1123, + 8508, + 40057, + 1648, + 15872, + 20463, + 1062, + 7168, + 14398, + 2560, + 972, + 6144, + 1022, + 918, + 1398, + 1737, + 1445, + 1185, + 747, + 1218, + 1307, + 712, + 1331, + 1022, + 1277, + 43151, + 4245, + 3898, + 9216, + 25149, + 55808, + 3496, + 8192, + 2351, + 2054, + 660, + 248, + 321, + 3578, + 1242, + 1770, + 2523, + 2706, + 2691, + 2270, + 1902, + 1276, + 476, + 34272, + 408, + 4096, + 3977, + 2560, + 396, + 3584, + 2214, + 10719, + 751, + 427, + 1150, + 796, + 4968, + 37376, + 63654, + 79360, + 4368, + 28160, + 10240, + 27648, + 5063, + 2292, + 712, + 5480, + 2769, + 446, + 218, + 526, + 3584, + 2800, + 44032, + 504, + 3584, + 22016, + 137, + 252, + 181, + 17920, + 5966, + 164860, + 489984, + 14848, + 5366, + 951, + 770, + 9728, + 12213, + 23982, + 11079, + 567, + 54687, + 3011, + 7220, + 13312, + 23306, + 66560, + 6424, + 11264, + 42483, + 16946, + 453, + 49790, + 5766, + 3584, + 2653, + 85504, + 5062, + 3584, + 2755, + 311, + 879, + 1004, + 717, + 2301, + 2009, + 540, + 1537, + 836, + 1365, + 510, + 488, + 5025, + 651, + 11264, + 2194, + 1402, + 1224, + 1576, + 868, + 1592, + 384, + 1468, + 1556, + 1530, + 1736, + 16622, + 75264, + 9216, + 1732, + 1176, + 1014, + 1324, + 760, + 1240, + 358, + 1180, + 1234, + 1180, + 1198, + 27598, + 781, + 3102, + 2192, + 3130, + 2135, + 3091, + 389, + 3053, + 3160, + 3137, + 3144, + 3051, + 1626, + 1712, + 1695, + 1711, + 1564, + 1326, + 772, + 1492, + 2048, + 21039, + 6740, + 16384, + 28156, + 64512, + 6210, + 13312, + 4208, + 607, + 4161, + 2886, + 1025, + 555, + 377, + 1274, + 663, + 490, + 2004, + 1078, + 1402, + 4082, + 1459, + 2770, + 1100, + 663, + 757, + 654, + 746, + 1052, + 760, + 2089, + 636, + 1394, + 640, + 10752, + 1314, + 20594, + 5772, + 10752, + 13349, + 81408, + 5012, + 9216, + 838, + 2078, + 548, + 2218, + 913, + 945, + 962, + 1775, + 909, + 931, + 1491, + 47126, + 4108, + 6656, + 12750, + 78848, + 3694, + 6144, + 2672, + 3062, + 94720, + 949, + 62743, + 1154, + 5120, + 7501, + 2560, + 1064, + 4608, + 8826, + 25358, + 1396, + 1332, + 22025, + 370, + 7680, + 11953, + 12800, + 330, + 6656, + 1408, + 2046, + 2749, + 1158, + 1051, + 1078, + 2763, + 11286, + 1226, + 1220, + 1221, + 1604, + 6656, + 7379, + 67584, + 1464, + 5632, + 729, + 1129, + 645, + 3054, + 838, + 394, + 5120, + 4621, + 64512, + 392, + 4608, + 592, + 645, + 471, + 1248, + 7168, + 6613, + 81920, + 1188, + 6144, + 139, + 1084, + 1204, + 1082, + 645, + 1928, + 4192, + 18369, + 4979, + 7110, + 1390, + 6656, + 11429, + 78848, + 1310, + 6144, + 1120, + 805, + 5314, + 2613, + 1016, + 1097, + 591, + 3211, + 2542, + 4273, + 344, + 402, + 1584, + 408, + 1341, + 414, + 222486, + 65, + 74064, + 22528, + 22528, + 133, + 966656, + 966656, + 4176, + 44632, + 2560, + 1024, + 13824, + 9728, + 2560, + 11264, + 45568, + 6656, + 3072, + 4848952, + 4669136, + 10976, + 10976, + 11520, + 10976, + 11488, + 12288, + 13248, + 12800, + 13200, + 12720, + 9280, + 9504, + 9856, + 10064, + 9792, + 12304, + 12256, + 12336, + 12384, + 10976, + 12288, + 10224, + 10608, + 9472, + 10240, + 35808, + 36672, + 36656, + 37296, + 36672, + 37472, + 80896, + 70000, + 80896, + 70000, + 916684, + 866652, + 653588, + 649244, + 189124, + 182112, + 182096, + 189264, + 183276, + 188976, + 186728, + 182208, + 180452, + 188932, + 183412, + 184020, + 167620, + 414772, + 410464, + 415776, + 410988, + 10992, + 13552, + 17760, + 1330156, + 1297136, + 1094912, + 1133324, + 963568, + 1056136, + 1723352, + 809512, + 857200, + 822568, + 219524, + 227628, + 227684, + 229412, + 7216, + 6352, + 6672, + 6672, + 7232, + 7216, + 6336, + 5168, + 4320, + 5200, + 4640, + 5168, + 5168, + 4304, + 245572, + 229444, + 226704, + 224248, + 445180, + 383780, + 452056, + 392232, + 448368, + 451860, + 448788, + 455620, + 262288, + 276064, + 270728, + 281096, + 26432, + 26544, + 23440, + 35888, + 36032, + 31760, + 794216, + 793256, + 577340, + 23408, + 23440, + 25024, + 23440, + 25024, + 31712, + 31776, + 33344, + 31808, + 33360, + 667232, + 5600, + 12896, + 672300, + 611212, + 345208, + 611556, + 720012, + 631992, + 580168, + 576004, + 643852, + 313856, + 235848, + 241972, + 333636, + 341072, + 332036, + 338776, + 363200, + 316440, + 331128, + 338140, + 330012, + 65, + 57948, + 36336, + 36816, + 36656, + 876784, + 879640, + 9248, + 8384, + 8368, + 8704, + 9232, + 9248, + 8368, + 6192, + 5328, + 5344, + 5648, + 5280, + 6192, + 5312, + 84241, + 140748, + 154188, + 1805996, + 229912, + 228504, + 271648, + 250668, + 263504, + 269048, + 271832, + 249924, + 266156, + 272460, + 219644, + 207484, + 208428, + 138196, + 143448, + 136176, + 139324, + 143684, + 135328, + 139800, + 140268, + 141608, + 159464, + 140808, + 136228, + 211780, + 66660, + 75340, + 77384, + 76356, + 76352, + 76084, + 75832, + 76104, + 74244, + 74476, + 69916, + 68464, + 71600, + 82744, + 75980, + 70896, + 26040, + 26489, + 29779, + 43318, + 11056, + 12400, + 10032, + 573308, + 40264, + 5680, + 6512, + 214808, + 12896, + 14432, + 10656, + 304568, + 41584, + 38480, + 6528, + 7728, + 327420, + 393764, + 386264, + 117092, + 327516, + 13457192, + 12598388, + 5083240, + 27652, + 9533888, + 9749256, + 840752, + 291356, + 271860, + 8704, + 290236, + 9214528, + 21402208, + 14439388, + 12875900, + 23648676, + 16246936, + 12004804, + 301876, + 78272, + 272840, + 304076, + 309956, + 306076, + 298668, + 295152, + 1422772, + 1377980, + 1462828, + 75160, + 69316, + 206612, + 472216, + 419976, + 336672, + 413712, + 94412, + 99464, + 74520, + 76972, + 74200, + 76300, + 78136, + 76184, + 78756, + 74944, + 73820, + 76080, + 77632, + 73660, + 77284, + 73352, + 13312, + 11056, + 12384, + 17760, + 12288, + 202428, + 173700, + 173080, + 596924, + 581228, + 925736, + 914092, + 509768, + 878372, + 816540, + 523052, + 324448, + 351908, + 1690384, + 1383320, + 441372, + 941172, + 439396, + 448756, + 2454648, + 62944, + 65392, + 59024, + 89456, + 95488, + 84080, + 57936, + 59952, + 60752, + 63296, + 61024, + 81728, + 85360, + 86256, + 90736, + 84848, + 18214352, + 17064020, + 944436, + 932572, + 997912, + 987736, + 25216, + 32512, + 24672, + 20448, + 37952, + 19904, + 26112, + 24784, + 28912, + 24832, + 29200, + 21504, + 19600, + 23120, + 19760, + 23008, + 69232, + 73216, + 65456, + 72192, + 95840, + 67328, + 90336, + 102400, + 64656, + 66464, + 65328, + 68848, + 64400, + 89856, + 92032, + 90288, + 98256, + 89456, + 18087936, + 5680, + 12896, + 257052, + 71000, + 839004, + 768964, + 68260, + 60480, + 2576, + 1110940, + 1093272, + 826840, + 884748, + 254448, + 244240, + 225940, + 252300, + 243284, + 211316, + 223308, + 163324, + 166940, + 163128, + 165288, + 164360, + 157596, + 161736, + 160860, + 166668, + 155584, + 159576, + 158924, + 229848, + 5168, + 5168, + 5232, + 6160, + 5120, + 5552, + 5184, + 5184, + 5200, + 5184, + 6128, + 5184, + 7232, + 6272, + 6304, + 6272, + 5952, + 6528, + 5376, + 7168, + 5360, + 5376, + 6112, + 5600, + 6112, + 5168, + 7104, + 7648, + 6656, + 8704, + 7280, + 6608, + 7008, + 6912, + 6912, + 122784, + 84776, + 14542540, + 13827216, + 13760848, + 13697012, + 3869801, + 2964, + 1555440, + 12958, + 16, + 6686208, + 3592, + 3936, + 358328, + 3912, + 788768, + 354016, + 3944, + 25675936, + 3368788, + 1000444, + 2475780, + 2470724, + 2944004, + 93962, + 398424, + 696279, + 278216, + 278216, + 95294, + 24285, + 76827, + 75202, + 79881, + 91664, + 73901, + 63945, + 94637, + 47381, + 76837, + 75209, + 78636, + 87270, + 73848, + 63938, + 123207, + 170846, + 163655, + 167149, + 190382, + 153710, + 181388, + 192658, + 161077, + 155948, + 173135, + 167996, + 175365, + 175142, + 193292, + 176173, + 208311, + 184305, + 152003, + 166960, + 191011, + 172588, + 179346, + 181738, + 193635, + 172483, + 173778, + 196545, + 168973, + 133549, + 137150, + 133821, + 135595, + 137643, + 133803, + 135918, + 138784, + 134220, + 134548, + 133996, + 136312, + 134303, + 140091, + 139146, + 134894, + 142998, + 136681, + 134007, + 136206, + 137845, + 134156, + 135178, + 137714, + 139707, + 138470, + 136571, + 141663, + 136969, + 820959, + 837651, + 841715, + 852003, + 858806, + 815642, + 836702, + 862614, + 816732, + 829627, + 824255, + 838193, + 831598, + 855467, + 849236, + 832468, + 890837, + 856899, + 821074, + 813929, + 851738, + 822220, + 834980, + 854788, + 859328, + 857391, + 831954, + 870512, + 851084, + 60199, + 54388, + 55658, + 55914, + 55229, + 54883, + 55844, + 56461, + 55369, + 55528, + 55209, + 55065, + 55780, + 54999, + 55374, + 55329, + 58244, + 55876, + 55263, + 54598, + 55896, + 55350, + 55013, + 55676, + 55948, + 55812, + 57234, + 56767, + 55447, + 51921, + 50525, + 50600, + 51372, + 50556, + 50003, + 50750, + 51665, + 52094, + 50200, + 50176, + 50753, + 50648, + 50991, + 51055, + 50490, + 51838, + 50686, + 50748, + 49804, + 50825, + 50376, + 50348, + 50585, + 50627, + 50823, + 52845, + 50831, + 50655, + 80529, + 1180712, + 1177049, + 1166899, + 1170755, + 1174183, + 1156877, + 1163286, + 1172109, + 1175470, + 1161012, + 1156568, + 1165993, + 1158947, + 1176781, + 1181107, + 1161986, + 1193441, + 1171979, + 1157102, + 1168877, + 1180549, + 1160990, + 1163072, + 1172432, + 1180765, + 1171788, + 1165080, + 1184896, + 1166673, + 234710, + 68593, + 17013, + 29907, + 43386, + 230849, + 219577, + 60300, + 15070, + 29907, + 39939, + 1586, + 77849, + 19410, + 610, + 32573, + 1584, + 2409, + 41256, + 10954, + 610, + 19250, + 975360, + 18432, + 286208, + 9728, + 2546176, + 103936, + 2139136, + 65536, + 257024, + 7680, + 321, + 26605, + 8686, + 8762, + 30720, + 7702016, + 199168, + 210432, + 516, + 4, + 117416, + 116608, + 1462, + 20071, + 1392, + 294680, + 9766, + 162, + 22020, + 1527, + 82, + 2560, + 2560, + 147456, + 812544, + 8017408, + 7378944, + 40310, + 40310, + 33170, + 33170, + 177806, + 177806, + 2589632, + 7390720, + 46287, + 48350, + 1618106, + 912766, + 1331040, + 528404, + 1322024, + 404904, + 893472, + 377968, + 429472, + 7393280, + 188278, + 3692612, + 16629137, + 1523154, + 1562816, + 127488, + 319, + 1346, + 8685, + 5632, + 5120, + 162, + 167, + 765, + 977, + 565, + 766, + 368, + 1136, + 883, + 1064, + 417, + 354, + 354, + 354, + 529, + 612, + 791, + 2085, + 406, + 406, + 542, + 595, + 747, + 1881, + 354, + 354, + 354, + 529, + 529, + 612, + 612, + 791, + 791, + 2085, + 2085, + 642, + 1232, + 2946, + 2946, + 4048, + 4901, + 8060, + 22343, + 2851, + 2851, + 3886, + 4751, + 7782, + 21801, + 2946, + 2946, + 4048, + 4901, + 8060, + 22343, + 569, + 375, + 977, + 977, + 1252, + 1826, + 2338, + 5709, + 1010, + 1010, + 1270, + 1775, + 2227, + 5481, + 977, + 977, + 1252, + 1826, + 2338, + 5709, + 893, + 569, + 569, + 689, + 911, + 1250, + 2924, + 526, + 526, + 736, + 864, + 1199, + 2811, + 569, + 569, + 689, + 911, + 1250, + 2924, + 741, + 175, + 184, + 583, + 1185, + 508, + 496, + 434, + 857, + 654, + 835, + 353, + 547, + 1357, + 324, + 618, + 502, + 4, + 1648, + 1624, + 8000, + 861, + 1011, + 991, + 1005, + 1058, + 945, + 963, + 939, + 945, + 945, + 905, + 947, + 914, + 930, + 877, + 897, + 884, + 912, + 901, + 897, + 906, + 900, + 885, + 891, + 906, + 906, + 866, + 872, + 928, + 904, + 886, + 931, + 881, + 857, + 897, + 877, + 904, + 889, + 901, + 885, + 839, + 866, + 892, + 908, + 920, + 893, + 901, + 933, + 874, + 880, + 872, + 872, + 896, + 877, + 873, + 869, + 868, + 826, + 820, + 823, + 814, + 817, + 802, + 846, + 1009, + 1003, + 988, + 823, + 829, + 808, + 814, + 796, + 805, + 832, + 838, + 862, + 838, + 829, + 803, + 799, + 808, + 827, + 823, + 823, + 862, + 850, + 838, + 841, + 847, + 829, + 838, + 814, + 799, + 808, + 790, + 814, + 835, + 817, + 835, + 823, + 844, + 838, + 814, + 817, + 808, + 811, + 832, + 826, + 826, + 829, + 823, + 844, + 823, + 814, + 829, + 832, + 841, + 855, + 832, + 823, + 850, + 814, + 820, + 823, + 826, + 829, + 829, + 876, + 817, + 835, + 814, + 817, + 826, + 847, + 826, + 826, + 820, + 835, + 796, + 793, + 859, + 799, + 796, + 864, + 832, + 832, + 787, + 808, + 885, + 891, + 882, + 878, + 1002, + 1002, + 1017, + 1005, + 1003, + 1015, + 1037, + 1004, + 1036, + 1038, + 1011, + 1077, + 1068, + 1053, + 1005, + 1001, + 1033, + 1038, + 1050, + 1035, + 1006, + 1048, + 1057, + 1057, + 1050, + 1038, + 1017, + 1017, + 1020, + 1018, + 948, + 1035, + 1037, + 1033, + 981, + 972, + 1036, + 1042, + 1054, + 1018, + 964, + 998, + 991, + 976, + 982, + 976, + 970, + 979, + 976, + 1024, + 952, + 955, + 1018, + 973, + 958, + 1007, + 921, + 1050, + 961, + 975, + 981, + 954, + 958, + 1000, + 979, + 981, + 964, + 973, + 970, + 964, + 968, + 1010, + 1012, + 1027, + 1046, + 1012, + 1004, + 977, + 967, + 976, + 999, + 1005, + 1018, + 1012, + 982, + 974, + 972, + 1001, + 998, + 998, + 968, + 971, + 1022, + 1016, + 995, + 977, + 1002, + 1008, + 1026, + 1006, + 1011, + 1036, + 990, + 999, + 1006, + 966, + 1020, + 975, + 929, + 962, + 953, + 1016, + 1048, + 1000, + 1032, + 1000, + 1032, + 1008, + 1040, + 982, + 991, + 988, + 982, + 983, + 998, + 1016, + 979, + 976, + 987, + 1013, + 1025, + 1009, + 1003, + 974, + 951, + 978, + 969, + 954, + 975, + 975, + 969, + 969, + 956, + 972, + 974, + 1001, + 1001, + 1017, + 996, + 1000, + 1024, + 1009, + 987, + 984, + 993, + 1023, + 1053, + 986, + 1011, + 1030, + 1011, + 1030, + 991, + 991, + 975, + 1009, + 987, + 954, + 978, + 969, + 966, + 949, + 982, + 974, + 956, + 989, + 989, + 982, + 956, + 956, + 941, + 956, + 986, + 974, + 920, + 980, + 1056, + 1056, + 1052, + 1009, + 1014, + 1020, + 1008, + 999, + 998, + 1020, + 984, + 932, + 929, + 953, + 967, + 920, + 932, + 920, + 940, + 973, + 962, + 1022, + 1008, + 944, + 957, + 1045, + 953, + 952, + 943, + 952, + 973, + 952, + 962, + 938, + 981, + 984, + 984, + 998, + 975, + 818, + 470294, + 1202, + 1181, + 1045, + 1135, + 1019, + 981, + 1153, + 1019, + 962, + 851, + 961, + 1206, + 1011, + 913, + 830, + 1164, + 865, + 1232, + 898, + 991, + 970, + 1206, + 1168, + 1169, + 1214, + 982, + 937, + 924, + 973, + 937, + 956, + 946, + 1030, + 1154, + 1048, + 978, + 1011, + 1074, + 837, + 1116, + 1020, + 1000, + 969, + 1116, + 991, + 1159, + 1036, + 1014, + 912, + 1092, + 1067, + 988, + 998, + 1055, + 881, + 866, + 1019, + 1089, + 966, + 1024, + 1118, + 1114, + 923, + 925, + 959, + 961, + 1000, + 1206, + 821, + 991, + 824, + 929, + 979, + 1061, + 1156, + 1131, + 944, + 1183, + 995, + 1105, + 1022, + 983, + 1020, + 920, + 1039, + 954, + 1048, + 1042, + 1118, + 974, + 1243, + 1016, + 930, + 1030, + 1068, + 1130, + 938, + 981, + 1149, + 1074, + 959, + 956, + 919, + 941, + 950, + 1006, + 1020, + 913, + 1033, + 867, + 843, + 1149, + 957, + 1068, + 964, + 941, + 914, + 999, + 937, + 837, + 1000, + 1089, + 963, + 998, + 1022, + 1149, + 832, + 855, + 980, + 807, + 1062, + 938, + 947, + 979, + 1022, + 992, + 1030, + 998, + 1014, + 961, + 940, + 957, + 827, + 1153, + 866, + 1114, + 921, + 1012, + 944, + 1151, + 887, + 1060, + 937, + 912, + 1077, + 923, + 1037, + 1071, + 1192, + 986, + 985, + 968, + 994, + 1004, + 824, + 1176, + 990, + 1040, + 947, + 1062, + 1232, + 938, + 1148, + 1032, + 911, + 895, + 743, + 774, + 846, + 5877760, + 95104, + 504, + 1043968, + 53760, + 36, + 36, + 829, + 54230, + 45, + 45, + 811, + 52828, + 42, + 42, + 1055, + 166446, + 65, + 65, + 444, + 98700, + 53, + 53, + 444, + 98752, + 33, + 33, + 381, + 1288, + 36, + 36, + 6067, + 1517020, + 11222, + 20076, + 4034, + 10124, + 4510, + 4150, + 10500, + 3816, + 7532, + 2500, + 7060, + 3124, + 7660, + 2560, + 7612, + 23024, + 28740, + 9932, + 14572, + 6256, + 12140, + 124744, + 46912, + 64756, + 31, + 7177, + 995542, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 819516, + 6537, + 819516, + 30, + 7177, + 997498, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 819522, + 6537, + 819522, + 36, + 318, + 42996, + 357050, + 2242, + 6108, + 7442, + 4242, + 7500, + 3228, + 3686, + 52958, + 3754, + 7356, + 8924, + 2766, + 2846, + 2770, + 1479, + 2558, + 89110, + 6336, + 10140, + 8106, + 12492, + 17880, + 5428, + 5946, + 4008, + 1648, + 5276, + 3424, + 8372, + 6578, + 10812, + 8294, + 11572, + 3202, + 7772, + 37872, + 15974, + 16396, + 28690, + 53252, + 32834, + 20812, + 4288, + 10052, + 4746, + 7788, + 12934, + 19588, + 1154, + 1012, + 2764, + 7052, + 1008, + 690, + 3972, + 1060, + 1022, + 1128, + 692, + 3964, + 2100, + 1062, + 2306, + 6252, + 1416, + 962, + 972, + 1056, + 4500, + 1264, + 2870, + 7748, + 1992, + 1158, + 4676, + 1130, + 4612, + 1164, + 4684, + 1136, + 4636, + 1158, + 4668, + 1158, + 4700, + 1154, + 4660, + 1136, + 4644, + 1104, + 4564, + 1148, + 4652, + 1152, + 4692, + 1190, + 4764, + 1156, + 4692, + 1136, + 4628, + 1162, + 4692, + 1132, + 4620, + 1154, + 4684, + 1128, + 4668, + 1150, + 4660, + 3140, + 1202, + 2452, + 5604, + 5776, + 1546, + 1406, + 1120, + 700, + 3844, + 680, + 4012, + 1116, + 4532, + 7646, + 1316, + 1014, + 1128, + 2570, + 6876, + 1472, + 960, + 1032, + 1044, + 2572, + 1178, + 1080, + 4548, + 1180, + 1344, + 1010, + 1078, + 1114, + 1240, + 1806, + 5380, + 1680, + 5396, + 666, + 3940, + 1006, + 2326, + 6324, + 1144, + 4644, + 1356, + 1010, + 1028, + 6106, + 974, + 1126, + 1478, + 1058, + 4500, + 2694, + 6996, + 2652, + 6972, + 1162, + 3266, + 1176, + 1108, + 1842, + 1084, + 1208, + 1112, + 4580, + 1016, + 1182, + 7232, + 12444, + 31632, + 29344, + 29344, + 8156, + 7756, + 12784, + 17044, + 3426, + 7668, + 3952, + 584, + 39622, + 78064, + 2826, + 7496, + 6270, + 8332, + 110927, + 3108, + 8700, + 975476, + 500344, + 975476, + 35700, + 2230, + 6460, + 5792, + 10036, + 700038, + 5324, + 7326, + 4418, + 8124, + 2058, + 5276, + 3698, + 2314, + 6044, + 10572, + 105198, + 103572, + 68300, + 2554, + 7484, + 4156, + 10156, + 7744, + 4148, + 8660, + 5154, + 9460, + 3958, + 9700, + 70670, + 34596, + 4572, + 3716, + 9768, + 20532, + 1786, + 6036, + 26824, + 31580, + 3724, + 7516, + 4070, + 8156, + 11662, + 12916, + 11968, + 14052, + 13876, + 18172, + 15402, + 22036, + 9956, + 12724, + 10380, + 13964, + 8122, + 12612, + 12616, + 16692, + 4166, + 27662, + 73928, + 153148, + 34544, + 10586, + 34544, + 34544, + 10586, + 34544, + 34544, + 10586, + 34544, + 34544, + 34544, + 34544, + 34544, + 34544, + 34544, + 34544, + 34544, + 229, + 10586, + 229, + 4134, + 7628, + 4010, + 7148, + 60840, + 7160, + 11332, + 6450, + 9740, + 84218, + 118548, + 68062, + 134140, + 26432, + 9832, + 18844, + 2864, + 6574, + 12300, + 17868, + 27084, + 22396, + 27068, + 5378, + 9844, + 145968, + 156116, + 4588, + 8668, + 4754, + 93080, + 107772, + 65972, + 13482, + 80568, + 171806, + 5730, + 27262, + 19720, + 120372, + 28978, + 8198, + 15558, + 49800, + 14466, + 105144, + 20416, + 25830, + 88896, + 24836, + 55742, + 63380, + 8450, + 29398, + 25790, + 12086, + 137326, + 26092, + 56996, + 7526, + 12050, + 243776, + 56662, + 11166, + 23974, + 118782, + 111928, + 107186, + 45598, + 26536, + 19476, + 18622, + 266006, + 86868, + 35708, + 152552, + 19952, + 31048, + 82068, + 22124, + 89728, + 47188, + 98330, + 89444, + 98810, + 104296, + 1574382, + 94068, + 152526, + 205386, + 80412, + 216596, + 126054, + 38968, + 5028, + 66028, + 100404, + 144846, + 40530, + 35686, + 137064, + 88352, + 28582, + 8938, + 11452, + 20706, + 46182, + 44850, + 19038, + 10228, + 88788, + 72772, + 16954, + 143864, + 97254, + 254856, + 10430, + 16894, + 832958, + 24040, + 5732, + 106688, + 23304, + 9538, + 9602, + 8244, + 8322, + 76410, + 23248, + 12244, + 24678, + 24328, + 8714, + 16054, + 17254, + 8372, + 29516, + 142858, + 9088, + 61202, + 38408, + 111000, + 21148, + 6480, + 75332, + 15414, + 103226, + 24976, + 81708, + 85148, + 158182, + 53316, + 27596, + 13248, + 98472, + 10102, + 43146, + 58978, + 230698, + 55956, + 91872, + 32492, + 36630, + 26488, + 36130, + 19834, + 25730, + 18860, + 71780, + 44666, + 27150, + 6068, + 23790, + 61154, + 99100, + 6860, + 4858, + 31168, + 242920, + 75430, + 216056, + 103516, + 157736, + 207412, + 13974, + 15700, + 8942, + 11804, + 318550, + 364956, + 3200, + 8972, + 2388, + 4126, + 5912, + 12516, + 86462, + 76452, + 9070, + 790436, + 1176964, + 61436, + 2190, + 3682, + 3864, + 4182, + 3864, + 768, + 49, + 49, + 49, + 705, + 131728, + 47, + 47, + 705, + 135800, + 43838, + 5884, + 11804, + 60908, + 69412, + 58210, + 98652, + 31186, + 37644, + 3452, + 7972, + 26808, + 38398, + 3852, + 7828, + 1044, + 7892, + 13524, + 33988, + 3386, + 2208, + 25160, + 2982, + 2712, + 6404, + 176484, + 125868, + 51026, + 62572, + 29228, + 40564, + 12568, + 22628, + 19112, + 68176, + 60244, + 30092, + 40810, + 248286, + 551212, + 5276, + 6980, + 30894, + 31930, + 434302, + 98426, + 71644, + 2656, + 10542, + 13516, + 723666, + 83830, + 99418, + 97762, + 329418, + 273392, + 261692, + 37444, + 3174, + 4370, + 13582, + 45834, + 101928, + 19080, + 430184, + 82502, + 13446, + 2140, + 6166, + 87594, + 93928, + 2460, + 214348, + 2154, + 83726, + 35016, + 4186, + 4794, + 8618, + 307862, + 285242, + 64914, + 24424, + 23620, + 5292, + 8186, + 11886, + 33690, + 268356, + 110094, + 40642, + 230496, + 2380, + 5136, + 6788, + 95530, + 8470, + 13622, + 15068, + 33734, + 6066, + 116538, + 34764, + 6694, + 7100, + 3798, + 8272, + 3442, + 398670, + 136934, + 319876, + 28282, + 991868, + 15784, + 1014176, + 243844, + 526428, + 45578, + 3010, + 7756, + 15134, + 2678, + 18102, + 8926, + 10148, + 2818, + 8236, + 20190, + 20596, + 6955, + 1046848, + 369, + 1046848, + 369, + 12338, + 3113, + 91662, + 471312, + 76048, + 15032, + 3130, + 128374, + 530416, + 75288, + 16920, + 3096, + 139821, + 486344, + 75288, + 125728, + 89872, + 89144, + 90608, + 404752, + 1028, + 19667, + 28511, + 65288, + 11194, + 155192, + 315888, + 36472, + 316736, + 13916, + 3066, + 126310, + 401376, + 76048, + 12420, + 3066, + 115628, + 400656, + 75288, + 125728, + 89144, + 89872, + 404072, + 212, + 121440, + 2810, + 60778, + 155128, + 36472, + 54392, + 35424, + 9033, + 6668, + 117460, + 187392, + 36472, + 61632, + 404072, + 121440, + 10476, + 3114, + 87177, + 495376, + 73480, + 13106, + 3130, + 122503, + 501216, + 73512, + 19440, + 2716, + 191560, + 302464, + 11126, + 3093, + 89526, + 497424, + 73480, + 13024, + 3097, + 154115, + 498672, + 76784, + 16988, + 3097, + 190046, + 499184, + 76784, + 9802, + 3097, + 42986, + 499184, + 76784, + 11446, + 3315, + 140902, + 302296, + 36472, + 125728, + 90608, + 91936, + 101216, + 89888, + 90608, + 89888, + 89144, + 98496, + 60608, + 404752, + 1081, + 9697, + 2777, + 20667, + 138512, + 64168, + 14848, + 3049, + 132659, + 411120, + 89072, + 36472, + 92144, + 405488, + 256, + 19667, + 28511, + 65288, + 9311, + 166032, + 36472, + 316736, + 12590, + 3050, + 87645, + 271024, + 68264, + 14850, + 3050, + 153989, + 370968, + 73512, + 11362, + 3050, + 117277, + 370968, + 73512, + 36472, + 125728, + 92864, + 91936, + 92448, + 89872, + 79720, + 404072, + 256, + 9732, + 3046, + 84534, + 357856, + 76232, + 9782, + 3075, + 20816, + 131584, + 73480, + 5174, + 2939, + 12442, + 3114, + 92354, + 468752, + 73480, + 15038, + 3130, + 128291, + 529392, + 73512, + 17742, + 3097, + 176198, + 523248, + 76784, + 11729, + 3097, + 44389, + 522736, + 76784, + 36472, + 125728, + 90608, + 91936, + 89888, + 90608, + 80672, + 404752, + 1083, + 11897, + 2743, + 22579, + 131344, + 73480, + 16770, + 3049, + 144852, + 442864, + 89072, + 36472, + 92144, + 80880, + 405488, + 260, + 12717, + 3043, + 56912, + 318944, + 89072, + 19667, + 28511, + 2236928, + 2239488, + 48464, + 65288, + 11361, + 156656, + 10997, + 10224, + 8796, + 12249, + 8148, + 9369, + 8944, + 10400, + 494960, + 503664, + 473968, + 10758, + 12146, + 483184, + 163696, + 10046, + 10040, + 165232, + 8657, + 9915, + 71952, + 279024, + 60752, + 316400, + 36472, + 316736, + 1721576, + 16797, + 3050, + 162256, + 352024, + 73512, + 13277, + 3050, + 126428, + 352024, + 73512, + 36472, + 125728, + 91936, + 92448, + 89872, + 80672, + 404072, + 268, + 11656, + 3046, + 91773, + 388040, + 76232, + 11904, + 3075, + 22837, + 133392, + 73480, + 121440, + 12423, + 3114, + 92426, + 468752, + 73480, + 15026, + 3130, + 128070, + 529392, + 73512, + 19440, + 2716, + 191560, + 302464, + 13073, + 3093, + 94851, + 469264, + 73480, + 14946, + 3097, + 163489, + 470512, + 76784, + 18910, + 3097, + 199105, + 523248, + 76784, + 11725, + 3097, + 45051, + 522736, + 76784, + 11446, + 3315, + 140902, + 302296, + 36472, + 125728, + 90608, + 91936, + 101216, + 89888, + 90608] diff --git a/tests/fragmentation/tfragment_alloc.nim b/tests/fragmentation/tfragment_alloc.nim new file mode 100644 index 0000000000..6ee8322c7b --- /dev/null +++ b/tests/fragmentation/tfragment_alloc.nim @@ -0,0 +1,19 @@ + + + +include system/ansi_c + +import strutils, data + +proc main = + var m = 0 + for i in 0..1000_000: + let size = sizes[i mod sizes.len] + let p = alloc(size) + if p == nil: + quit "could not serve request!" + dealloc p + # c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size) + +main() +echo formatSize getOccupiedMem(), " / ", formatSize getTotalMem() diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim new file mode 100644 index 0000000000..41f86136cc --- /dev/null +++ b/tests/fragmentation/tfragment_gc.nim @@ -0,0 +1,16 @@ + + + +#include system/ansi_c + +import strutils, data + +proc main = + var m = 0 + for i in 0..1000_000: + let size = sizes[i mod sizes.len] + let p = newString(size) + # c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size) + +main() +echo formatSize getOccupiedMem(), " / ", formatSize getTotalMem() diff --git a/todo.txt b/todo.txt index bc68c21686..1fae1ddacb 100644 --- a/todo.txt +++ b/todo.txt @@ -31,7 +31,6 @@ Not critical for 1.0 - pragmas need 'bindSym' support - pragmas need re-work: 'push' is dangerous, 'hasPragma' does not work reliably with user-defined pragmas -- memory manager: add a measure of fragmentation - we need a magic thisModule symbol - optimize 'genericReset'; 'newException' leads to code bloat @@ -52,7 +51,6 @@ Bugs GC == -- use slightly bigger blocks in the allocator - resizing of strings/sequences could take into account the memory that is allocated From c96e1d0180daebd67a4e48d7665069d70164789d Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 7 Dec 2017 17:03:15 +0100 Subject: [PATCH 78/92] enable fragmentation tests --- tests/fragmentation/tfragment_alloc.nim | 16 ++++++++++++---- tests/fragmentation/tfragment_gc.nim | 20 ++++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/fragmentation/tfragment_alloc.nim b/tests/fragmentation/tfragment_alloc.nim index 6ee8322c7b..031588d271 100644 --- a/tests/fragmentation/tfragment_alloc.nim +++ b/tests/fragmentation/tfragment_alloc.nim @@ -1,7 +1,8 @@ - - -include system/ansi_c +discard """ + output: '''occupied ok: true +total ok: true''' +""" import strutils, data @@ -16,4 +17,11 @@ proc main = # c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size) main() -echo formatSize getOccupiedMem(), " / ", formatSize getTotalMem() + +let occ = getOccupiedMem() +let total = getTotalMem() + +# Current values on Win64: 824KiB / 106.191MiB + +echo "occupied ok: ", occ < 2 * 1024 * 1024 +echo "total ok: ", total < 120 * 1024 * 1024 diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 41f86136cc..42f2f1cf62 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -1,16 +1,24 @@ - - - -#include system/ansi_c +discard """ + output: '''occupied ok: true +total ok: true''' +""" import strutils, data proc main = var m = 0 - for i in 0..1000_000: + # Since the GC test is slower than the alloc test, we only iterate 100_000 times here: + for i in 0..100_000: let size = sizes[i mod sizes.len] let p = newString(size) # c_fprintf(stdout, "iteration: %ld size: %ld\n", i, size) main() -echo formatSize getOccupiedMem(), " / ", formatSize getTotalMem() + +let occ = getOccupiedMem() +let total = getTotalMem() + +# Concrete values on Win64: 58.152MiB / 188.285MiB + +echo "occupied ok: ", occ < 60 * 1024 * 1024 +echo "total ok: ", total < 200 * 1024 * 1024 From e016c9253e7b02ad4781452067dad980c677f61e Mon Sep 17 00:00:00 2001 From: Brent Pedersen Date: Thu, 7 Dec 2017 12:25:39 -0700 Subject: [PATCH 79/92] optimize setLen (#6816) inline the call to setLengthSeq and avoid decref for types if ntfNoRefs closes #6721 and speeds setLen when newLen < len for non reference types. --- lib/system/sysstr.nim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 3f8e0eff0c..56b8ade97a 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -259,7 +259,7 @@ proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = result.reserved = r proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. - compilerRtl.} = + compilerRtl, inl.} = result = seq if result.space < newLen: let r = max(resize(result.space), newLen) @@ -282,10 +282,11 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) gch.tempStack.len = len0 else: - for i in newLen..result.len-1: - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waZctDecRef) + if ntfNoRefs notin extGetCellType(result).base.flags: + for i in newLen..result.len-1: + forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% + GenericSeqSize +% (i*%elemSize)), + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). From c99654a78e4b00507b1416dd97b53de6f78c84c3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Dec 2017 20:30:51 +0100 Subject: [PATCH 80/92] lets see what appveyor reports as the used memory --- tests/fragmentation/tfragment_gc.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 42f2f1cf62..064e8b360f 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -21,4 +21,7 @@ let total = getTotalMem() # Concrete values on Win64: 58.152MiB / 188.285MiB echo "occupied ok: ", occ < 60 * 1024 * 1024 -echo "total ok: ", total < 200 * 1024 * 1024 +let totalOk = total < 210 * 1024 * 1024 +echo "total ok: ", totalOk +if not totalOk: + echo "total peak memory ", formatSize(total) From 3d5840d24fd58bec21b5ebd8c4109c71b04f1db0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Dec 2017 00:00:31 +0100 Subject: [PATCH 81/92] parsesql: some bugfixes --- lib/pure/parsesql.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 00d007d015..6891e2ff74 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -956,6 +956,7 @@ proc parseInsert(p: var SqlParser): SqlNode = if p.tok.kind == tkParLe: var n = newNode(nkColumnList) parseParIdentList(p, n) + result.add n else: result.add(nil) if isKeyw(p, "default"): @@ -1160,7 +1161,7 @@ proc ra(n: SqlNode, s: var string, indent: int) = else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: - s.add(escape(n.strVal, "e'", "'")) + s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") of nkHexStringLit: @@ -1240,7 +1241,7 @@ proc ra(n: SqlNode, s: var string, indent: int) = if n.sons[2].kind == nkDefault: s.add("default values") else: - s.add("\nvalues ") + s.add("\n") ra(n.sons[2], s, indent) s.add(';') of nkUpdate: From a9b8f383667bd7084f2a6848792bbab4853c39d2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Dec 2017 00:19:28 +0100 Subject: [PATCH 82/92] change tfragment_gc test for appveyor --- tests/fragmentation/tfragment_gc.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fragmentation/tfragment_gc.nim b/tests/fragmentation/tfragment_gc.nim index 064e8b360f..6e0ec37cea 100644 --- a/tests/fragmentation/tfragment_gc.nim +++ b/tests/fragmentation/tfragment_gc.nim @@ -22,6 +22,6 @@ let total = getTotalMem() echo "occupied ok: ", occ < 60 * 1024 * 1024 let totalOk = total < 210 * 1024 * 1024 -echo "total ok: ", totalOk if not totalOk: echo "total peak memory ", formatSize(total) +echo "total ok: ", totalOk From 17becb8d30a53d2528a872f1bc05cd30011f9b02 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 8 Dec 2017 08:07:57 +0100 Subject: [PATCH 83/92] added allocator improvments to the changelog; closes #6031 --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index f061805b14..94cea7dbf4 100644 --- a/changelog.md +++ b/changelog.md @@ -118,3 +118,8 @@ This now needs to be written as: See [special-operators](https://nim-lang.org/docs/manual.html#special-operators) for more information. - Added ``macros.unpackVarargs``. +- The memory manager now uses a variant of the TLSF algorithm that has much + better memory fragmentation behaviour. According + to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum + fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and + ``dealloc`` became O(1) operations. From 00a230e5d86598f37fb2708bae97c7ab1c0ebe0e Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 8 Dec 2017 09:19:55 +0100 Subject: [PATCH 84/92] Leak detector: give more info for anon ref objects --- compiler/ccgtypes.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8dfb82963d..cfa2afdd95 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -968,8 +968,11 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) discard cgsym(m, "TNimType") if isDefined("nimTypeNames"): + var typename = typeToString(origType, preferName) + if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: + typename = "anon ref object from " & $origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", - [name, makeCstring typeToString(origType, preferName)]) + [name, makeCstring typename]) discard cgsym(m, "nimTypeRoot") addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n", [name]) From eae1aaa37728578bf93263b0ca24064a31ca4b16 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 8 Dec 2017 10:06:20 +0100 Subject: [PATCH 85/92] fixes another sighashes problem --- compiler/sighashes.nim | 22 +++++++++++++++++----- tests/ccgbugs/tuple_canon.nim | 13 +++++++++++++ tests/cpp/tcasts.nim | 1 + 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 504992cf5c..e2032294af 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -87,6 +87,7 @@ type CoProc CoType CoOwnerSig + CoIgnoreRange proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) @@ -159,14 +160,15 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = return else: discard - c &= char(t.kind) case t.kind of tyBool, tyChar, tyInt..tyUInt64: # no canonicalization for integral types, so that e.g. ``pid_t`` is # produced instead of ``NI``: + c &= char(t.kind) if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}: c.hashSym(t.sym) of tyObject, tyEnum: + c &= char(t.kind) if t.typeInst != nil: assert t.typeInst.kind == tyGenericInst for i in countup(1, sonsLen(t.typeInst) - 2): @@ -199,26 +201,35 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = if t.len > 0 and t.sons[0] != nil: hashType c, t.sons[0], flags of tyRef, tyPtr, tyGenericBody, tyVar: + c &= char(t.kind) c.hashType t.lastSon, flags if tfVarIsPtr in t.flags: c &= ".varisptr" of tyFromExpr: + c &= char(t.kind) c.hashTree(t.n) of tyTuple: + c &= char(t.kind) if t.n != nil and CoType notin flags: assert(sonsLen(t.n) == sonsLen(t)) for i in countup(0, sonsLen(t.n) - 1): assert(t.n.sons[i].kind == nkSym) c &= t.n.sons[i].sym.name.s c &= ':' - c.hashType(t.sons[i], flags) + c.hashType(t.sons[i], flags+{CoIgnoreRange}) c &= ',' else: - for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags - of tyRange, tyStatic: - #if CoType notin flags: + for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags+{CoIgnoreRange} + of tyRange: + if CoIgnoreRange notin flags: + c &= char(t.kind) + c.hashTree(t.n) + c.hashType(t.sons[0], flags) + of tyStatic: + c &= char(t.kind) c.hashTree(t.n) c.hashType(t.sons[0], flags) of tyProc: + c &= char(t.kind) c &= (if tfIterator in t.flags: "iterator " else: "proc ") if CoProc in flags and t.n != nil: let params = t.n @@ -237,6 +248,7 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = if tfThread in t.flags: c &= ".thread" if tfVarargs in t.flags: c &= ".varargs" else: + c &= char(t.kind) for i in 0.. Date: Fri, 8 Dec 2017 23:31:06 +0100 Subject: [PATCH 86/92] fixes #6889 --- compiler/sighashes.nim | 3 +++ tests/ccgbugs/tuple_canon.nim | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index e2032294af..5d6b5978d8 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -247,6 +247,9 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = if tfNoSideEffect in t.flags: c &= ".noSideEffect" if tfThread in t.flags: c &= ".thread" if tfVarargs in t.flags: c &= ".varargs" + of tyArray: + c &= char(t.kind) + for i in 0.. Date: Fri, 8 Dec 2017 19:55:04 -0500 Subject: [PATCH 87/92] Use addCallback rather than callback= in asyncfutures.all() (#6850) * Use addCallback rather than callback= in asyncfutures.all() Addresses part of #6849 * Stop using do notation for #6849 * Update example style --- lib/pure/asyncdispatch.nim | 5 +++-- lib/pure/asyncfutures.nim | 4 ++-- tests/async/tasyncall.nim | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 4c96aa6148..a71d30ab9f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -59,9 +59,10 @@ export asyncfutures, asyncstreams ## ## .. code-block::nim ## var future = socket.recv(100) -## future.callback = +## future.addCallback( ## proc () = ## echo(future.read) +## ) ## ## All asynchronous functions returning a ``Future`` will not block. They ## will not however return immediately. An asynchronous function will have @@ -1611,4 +1612,4 @@ proc waitFor*[T](fut: Future[T]): T = fut.read -{.deprecated: [setEvent: trigger].} \ No newline at end of file +{.deprecated: [setEvent: trigger].} diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index bebd196114..4bd3227a11 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -333,7 +333,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = let totalFutures = len(futs) for fut in futs: - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: @@ -355,7 +355,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = for i, fut in futs: proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: diff --git a/tests/async/tasyncall.nim b/tests/async/tasyncall.nim index a3926eabdd..775dd0c6f0 100644 --- a/tests/async/tasyncall.nim +++ b/tests/async/tasyncall.nim @@ -40,6 +40,16 @@ proc testVarargs(x, y, z: int): seq[int] = result = waitFor all(a, b, c) +proc testWithDupes() = + var + tasks = newSeq[Future[void]](taskCount) + fut = futureWithoutValue() + + for i in 0.. Date: Sat, 9 Dec 2017 06:07:37 -0600 Subject: [PATCH 88/92] modify getTypeImpl to reduce result to final implementation (#6891) * added test case for getTypeImpl * modify getTypeImpl to reduce result to final implementation --- compiler/vmdeps.nim | 6 +++--- tests/macros/tgettypeinst.nim | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 44550a3895..fb277272b1 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -84,10 +84,10 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; if inst: if t.sym != nil: # if this node has a symbol - if allowRecursion: # getTypeImpl behavior: turn off recursion - allowRecursion = false - else: # getTypeInst behavior: return symbol + if not allowRecursion: # getTypeInst behavior: return symbol return atomicType(t.sym) + #else: # getTypeImpl behavior: turn off recursion + # allowRecursion = false case t.kind of tyNone: result = atomicType("none", mNone) diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim index ea98721c48..2f1abe193c 100644 --- a/tests/macros/tgettypeinst.nim +++ b/tests/macros/tgettypeinst.nim @@ -113,8 +113,12 @@ type Generic[T] = seq[int] Concrete = Generic[int] + Generic2[T1, T2] = seq[T1] + Concrete2 = Generic2[int, float] + Alias1 = float Alias2 = Concrete + Alias3 = Concrete2 Vec[N: static[int],T] = object arr: array[N,T] @@ -154,15 +158,21 @@ test(Tree): left: ref Tree right: ref Tree test(Concrete): - type _ = Generic[int] + type _ = seq[int] test(Generic[int]): type _ = seq[int] test(Generic[float]): type _ = seq[int] +test(Concrete2): + type _ = seq[int] +test(Generic2[int,float]): + type _ = seq[int] test(Alias1): type _ = float test(Alias2): - type _ = Generic[int] + type _ = seq[int] +test(Alias3): + type _ = seq[int] test(Vec[4,float32]): type _ = object arr: array[0..3,float32] From 14f2578604d0bf32a41ed7a26357ae82fb885d7a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 9 Dec 2017 13:40:08 +0100 Subject: [PATCH 89/92] fixes crash related to runnableExamples in Nim doc that yet uses the VM --- compiler/vmgen.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 8f0c72e454..252b7c788b 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1130,6 +1130,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = # produces a value else: globalError(n.info, "expandToAst requires a call expression") + of mRunnableExamples: + discard "just ignore any call to runnableExamples" else: # mGCref, mGCunref, globalError(n.info, "cannot generate code for: " & $m) From e24a3bd0ab5e34f18ebae784a94d8cb3459fcca7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 11 Dec 2017 09:18:11 +0100 Subject: [PATCH 90/92] allocator: minor fix for deallocOsPages --- lib/system/alloc.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 725c9ccd38..e274e8e0cd 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -308,6 +308,7 @@ proc llDeallocAll(a: var MemRegion) = var next = it.next osDeallocPages(it, PageSize) it = next + a.llmem = nil proc intSetGet(t: IntSet, key: int): PTrunk = var it = t.data[key and high(t.data)] From 6e08ae5c266a044269e750c77b487a4a23e57690 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 11 Dec 2017 11:57:21 +0100 Subject: [PATCH 91/92] merged patch #6876 manually, taking care of poDemon --- lib/pure/osproc.nim | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5440ceb674..2a1ce0c58b 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -41,6 +41,8 @@ type ## Windows: Named pipes are used so that you can peek ## at the process' output streams. poDemon ## Windows: The program creates no Window. + ## Unix: Start the program as a demon. This is still + ## work in progress! ProcessObj = object of RootObj when defined(windows): @@ -230,7 +232,7 @@ proc execProcesses*(cmds: openArray[string], ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes ## is returned. Runs `beforeRunEvent` before running each command. - + assert n > 0 if n > 1: var i = 0 @@ -710,9 +712,7 @@ elif not defined(useNimRtl): sysEnv: cstringArray workingDir: cstring pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - optionPoUsePath: bool - optionPoParentStreams: bool - optionPoStdErrToStdOut: bool + options: set[ProcessOption] {.deprecated: [TStartProcessData: StartProcessData].} const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and @@ -777,10 +777,8 @@ elif not defined(useNimRtl): data.pStdin = pStdin data.pStdout = pStdout data.pStderr = pStderr - data.optionPoParentStreams = poParentStreams in options - data.optionPoUsePath = poUsePath in options - data.optionPoStdErrToStdOut = poStdErrToStdOut in options data.workingDir = workingDir + data.options = options when useProcessAuxSpawn: var currentDir = getCurrentDir() @@ -829,19 +827,22 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - chck posix_spawnattr_setpgroup(attr, 0'i32) + if poDemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) - chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or - POSIX_SPAWN_SETSIGMASK or - POSIX_SPAWN_SETPGROUP) + var flags = POSIX_SPAWN_USEVFORK or + POSIX_SPAWN_SETSIGMASK + if poDemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP + chck posix_spawnattr_setflags(attr, flags) - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) @@ -851,7 +852,7 @@ elif not defined(useNimRtl): setCurrentDir($data.workingDir) var pid: Pid - if data.optionPoUsePath: + if (poUsePath in data.options): res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) else: res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) @@ -913,7 +914,7 @@ elif not defined(useNimRtl): # Warning: no GC here! # Or anything that touches global structures - all called nim procs # must be marked with stackTrace:off. Inspect C code after making changes. - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): discard close(data.pStdin[writeIdx]) if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) @@ -921,7 +922,7 @@ elif not defined(useNimRtl): if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) discard close(data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): if dup2(data.pStdout[writeIdx], 2) < 0: startProcessFail(data) else: @@ -935,7 +936,7 @@ elif not defined(useNimRtl): discard close(data.pErrorPipe[readIdx]) discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) - if data.optionPoUsePath: + if (poUsePath in data.options): when defined(uClibc) or defined(linux): # uClibc environment (OpenWrt included) doesn't have the full execvpe let exe = findExe(data.sysCommand) From 28e0bf9dcd62f387c79f767848cadd8b71d825de Mon Sep 17 00:00:00 2001 From: skilchen Date: Mon, 11 Dec 2017 14:43:59 +0100 Subject: [PATCH 92/92] fix #6264 and #6141 (#6884) --- lib/pure/logging.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index e2a5bed969..830820fd13 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -202,13 +202,17 @@ when not defined(js): proc countLogLines(logger: RollingFileLogger): int = result = 0 - for line in logger.file.lines(): + let fp = open(logger.baseName, fmRead) + for line in fp.lines(): result.inc() + fp.close() proc countFiles(filename: string): int = # Example: file.log.1 result = 0 - let (dir, name, ext) = splitFile(filename) + var (dir, name, ext) = splitFile(filename) + if dir == "": + dir = "." for kind, path in walkDir(dir): if kind == pcFile: let llfn = name & ext & ExtSep