From bebeee46debf4426e6bddc9d288f29cd5ac61569 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 24 Nov 2016 16:25:21 +0000 Subject: [PATCH 01/21] Improve SMTP testing Test both sync and async email delivery --- lib/pure/smtp.nim | 56 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 0507129021..87865c0058 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -256,24 +256,48 @@ proc close*(smtp: AsyncSmtp) {.async.} = smtp.sock.close() when not defined(testing) and isMainModule: - #var msg = createMessage("Test subject!", - # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) - #echo(msg) + # To test with a real SMTP service, create a smtp.ini file, e.g.: + # username = "" + # password = "" + # smtphost = "smtp.gmail.com" + # port = 465 + # use_tls = true + # sender = "" + # recipient = "" - #var smtpConn = connect("localhost", Port 25, false, true) - #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg) + import parsecfg - #echo(decode("a17sm3701420wbe.12")) - proc main() {.async.} = - var client = newAsyncSmtp("smtp.gmail.com", Port(465), true) + proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) + + let + conf = loadConfig("smtp.ini") + msg = createMessage("Hello from Nim's SMTP!", + "Hello!\n Is this awesome or what?", @[conf["recipient"]]) + + assert conf["smtphost"] != "" + + proc async_test() {.async.} = + let client = newAsyncSmtp( + conf["smtphost"], + conf["port"].parseInt.Port, + conf["use_tls"].parseBool + ) await client.connect() - await client.auth("johndoe", "foo") - var msg = createMessage("Hello from Nim's SMTP!", - "Hello!!!!.\n Is this awesome or what?", - @["blah@gmail.com"]) - echo(msg) - await client.sendMail("blah@gmail.com", @["blah@gmail.com"], $msg) - + await client.auth(conf["username"], conf["password"]) + await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) await client.close() + echo "async email sent" - waitFor main() + proc sync_test() = + var smtpConn = connect( + conf["smtphost"], + conf["port"].parseInt.Port, + conf["use_tls"].parseBool, + true, # debug + ) + smtpConn.auth(conf["username"], conf["password"]) + smtpConn.sendmail(conf["sender"], @[conf["recipient"]], $msg) + echo "sync email sent" + + waitFor async_test() + sync_test() From 64dd3eb9c1ac0358e0be3ce037c73e83d4b947c8 Mon Sep 17 00:00:00 2001 From: John Novak Date: Sat, 26 Nov 2016 16:36:08 +1000 Subject: [PATCH 02/21] Fix setStyle for Windows * setStyle did not take the current color flags into account on Windows, this is fixed now * refactoring to get rid of magic constants --- lib/pure/terminal.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index d4734c3e37..e4c42310a8 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -29,6 +29,8 @@ when defined(windows): BACKGROUND_GREEN = 32 BACKGROUND_RED = 64 BACKGROUND_INTENSITY = 128 + FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE + BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE type SHORT = int16 @@ -369,12 +371,13 @@ proc setStyle*(f: File, style: set[Style]) = ## Sets the terminal style. when defined(windows): let h = conHandle(f) + var old = getAttributes(h) and (FOREGROUND_RGB or BACKGROUND_RGB) var a = 0'i16 if styleBright in style: a = a or int16(FOREGROUND_INTENSITY) if styleBlink in style: a = a or int16(BACKGROUND_INTENSITY) if styleReverse in style: a = a or 0x4000'i16 # COMMON_LVB_REVERSE_VIDEO if styleUnderscore in style: a = a or 0x8000'i16 # COMMON_LVB_UNDERSCORE - discard setConsoleTextAttribute(h, a) + discard setConsoleTextAttribute(h, old or a) else: for s in items(style): f.write("\e[" & $ord(s) & 'm') @@ -423,7 +426,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = ## Sets the terminal's foreground color. when defined(windows): let h = conHandle(f) - var old = getAttributes(h) and not 0x0007 + var old = getAttributes(h) and not FOREGROUND_RGB if bright: old = old or FOREGROUND_INTENSITY const lookup: array[ForegroundColor, int] = [ @@ -445,7 +448,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = ## Sets the terminal's background color. when defined(windows): let h = conHandle(f) - var old = getAttributes(h) and not 0x0070 + var old = getAttributes(h) and not BACKGROUND_RGB if bright: old = old or BACKGROUND_INTENSITY const lookup: array[BackgroundColor, int] = [ From 88e01ffd925d7f1180d9e95cc59a5ff001299489 Mon Sep 17 00:00:00 2001 From: John Novak Date: Sat, 26 Nov 2016 21:05:50 +1000 Subject: [PATCH 03/21] Add hideCursor & showCursor --- lib/pure/terminal.nim | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index d4734c3e37..0a2c92bbf3 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -13,6 +13,8 @@ ## Windows API. ## Changing the style is permanent even after program termination! Use the ## code ``system.addQuitProc(resetAttributes)`` to restore the defaults. +## Similarly, if you hide the cursor, make sure to unhide it with +## ``showCursor`` before quitting. import macros @@ -49,6 +51,10 @@ when defined(windows): srWindow: SMALL_RECT dwMaximumWindowSize: COORD + CONSOLE_CURSOR_INFO = object + dwSize: DWORD + bVisible: WINBOOL + proc duplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, hTargetProcessHandle: HANDLE, lpTargetHandle: ptr HANDLE, dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, @@ -60,6 +66,14 @@ when defined(windows): lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".} + proc getConsoleCursorInfo(hConsoleOutput: HANDLE, + lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetConsoleCursorInfo".} + + proc setConsoleCursorInfo(hConsoleOutput: HANDLE, + lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleCursorInfo".} + proc terminalWidthIoctl*(handles: openArray[Handle]): int = var csbi: CONSOLE_SCREEN_BUFFER_INFO for h in handles: @@ -179,6 +193,30 @@ else: return w return 80 #Finally default to venerable value +when defined(windows): + proc setCursorVisibility(f: File, visible: bool) = + var ccsi: CONSOLE_CURSOR_INFO + let h = conHandle(f) + if getConsoleCursorInfo(h, addr(ccsi)) == 0: + raiseOSError(osLastError()) + ccsi.bVisible = if visible: 1 else: 0 + if setConsoleCursorInfo(h, addr(ccsi)) == 0: + raiseOSError(osLastError()) + +proc hideCursor*(f: File) = + ## Hides the cursor. + when defined(windows): + setCursorVisibility(f, false) + else: + f.write("\e[?25l") + +proc showCursor*(f: File) = + ## Shows the cursor. + when defined(windows): + setCursorVisibility(f, true) + else: + f.write("\e[?25h") + proc setCursorPos*(f: File, x, y: int) = ## Sets the terminal's cursor to the (x,y) position. ## (0,0) is the upper left of the screen. @@ -558,6 +596,8 @@ proc getch*(): char = discard fd.tcsetattr(TCSADRAIN, addr oldMode) # Wrappers assuming output to stdout: +template hideCursor*() = hideCursor(stdout) +template showCursor*() = showCursor(stdout) template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): From 8378a752678fccecfffaeb94dc6d086e540c50d8 Mon Sep 17 00:00:00 2001 From: John Novak Date: Sun, 27 Nov 2016 20:54:46 +1000 Subject: [PATCH 04/21] Update changelog --- web/news/e029_version_0_16_0.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/news/e029_version_0_16_0.rst b/web/news/e029_version_0_16_0.rst index 94c9757a73..08506b6d31 100644 --- a/web/news/e029_version_0_16_0.rst +++ b/web/news/e029_version_0_16_0.rst @@ -36,6 +36,10 @@ Library Additions - Added new parameter to ``error`` proc of ``macro`` module to provide better error message +- Added ``hideCursor`` and ``showCursor`` to the ``terminal`` + `(doc) `_ module. + + Tool Additions -------------- From 95682cdfb7596466a30e610c01bd0c8fad39e877 Mon Sep 17 00:00:00 2001 From: Samadi van Koten Date: Mon, 28 Nov 2016 23:02:30 +0000 Subject: [PATCH 05/21] Remove docgen.nim's dependency on things being in path compiler/docgen.nim relied heavily on $lib/packages/docutils being in path at compile-time. While this works fine when building Nim itself, things start breaking when using compiler as a nimble package. --- compiler/docgen.nim | 6 ++++-- compiler/nim.cfg | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 76b36d7962..2115449249 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,8 +13,10 @@ import ast, strutils, strtabs, options, msgs, os, ropes, idents, - wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, - importer, sempass2, json, xmltree, cgi, typesrenderer, astalgo + wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast, + packages/docutils/rst, packages/docutils/rstgen, times, + packages/docutils/highlite, importer, sempass2, json, xmltree, cgi, + typesrenderer, astalgo type TSections = array[TSymKind, Rope] diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 0ff128ba3c..853ae7e00d 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -4,8 +4,6 @@ hint[XDeclaredButNotUsed]:off path:"llvm" path:"$projectPath/.." -path:"$lib/packages/docutils" - define:booting #import:"$projectpath/testability" From d495ef57054f91799144022012eef646b9194163 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 29 Nov 2016 10:40:08 +0100 Subject: [PATCH 06/21] updated upcoming/asyncdispatch --- lib/upcoming/asyncdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 68ecbe81ee..798a7e890d 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -207,7 +207,7 @@ when defined(windows) or defined(nimdoc): result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[AsyncFD]() result.timers.newHeapQueue() - result.callbacks = initQueue[proc ()](64) + result.callbacks = initDeque[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = From c743b2969f9e7c922cbeadab983a84389953fb24 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 29 Nov 2016 20:21:56 +0100 Subject: [PATCH 07/21] closure types get names; refs #4332 --- compiler/lambdalifting.nim | 33 +++++++++++++++++---------------- compiler/lowerings.nim | 4 ++++ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 36ad2e0a60..692d9265b7 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -139,10 +139,10 @@ proc createStateField(iter: PSym): PSym = result = newSym(skField, getIdent(":state"), iter, iter.info) result.typ = createStateType(iter) -proc createEnvObj(owner: PSym): PType = +proc createEnvObj(owner: PSym; info: TLineInfo): PType = # YYY meh, just add the state field for every closure for now, it's too # hard to figure out if it comes from a closure iterator: - result = createObj(owner, owner.info) + result = createObj(owner, info) rawAddField(result, createStateField(owner)) proc getIterResult(iter: PSym): PSym = @@ -296,18 +296,19 @@ This is why need to store the 'ownerToType' table and use it during .closure'fication. """ -proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType = +proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym; + info: TLineInfo): PType = result = c.ownerToType.getOrDefault(owner.id) if result.isNil: result = newType(tyRef, owner) - let obj = createEnvObj(owner) + let obj = createEnvObj(owner, info) rawAddSon(result, obj) c.ownerToType[owner.id] = result -proc createUpField(c: var DetectionPass; dest, dep: PSym) = - let refObj = c.getEnvTypeForOwner(dest) # getHiddenParam(dest).typ +proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) = + let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ let obj = refObj.lastSon - let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ + let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ if refObj == fieldType: localError(dep.info, "internal error: invalid up reference computed") @@ -347,10 +348,10 @@ Consider: """ -proc addClosureParam(c: var DetectionPass; fn: PSym) = +proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) = var cp = getEnvParam(fn) let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner - let t = c.getEnvTypeForOwner(owner) + let t = c.getEnvTypeForOwner(owner, info) if cp == nil: cp = newSym(skParam, getIdent(paramName), fn, fn.info) incl(cp.flags, sfFromGeneric) @@ -367,7 +368,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure: # this handles the case that the inner proc was declared as # .closure but does not actually capture anything: - addClosureParam(c, s) + addClosureParam(c, s, n.info) c.somethingToDo = true let innerProc = isInnerProc(s) @@ -379,7 +380,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if ow == owner: if owner.isIterator: c.somethingToDo = true - addClosureParam(c, owner) + addClosureParam(c, owner, n.info) if interestingIterVar(s): if not c.capturedVars.containsOrIncl(s.id): let obj = getHiddenParam(owner).typ.lastSon @@ -403,11 +404,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # mark 'owner' as taking a closure: c.somethingToDo = true markAsClosure(owner, n) - addClosureParam(c, owner) + addClosureParam(c, owner, n.info) #echo "capturing ", n.info # variable 's' is actually captured: if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): - let obj = c.getEnvTypeForOwner(ow).lastSon + let obj = c.getEnvTypeForOwner(ow, n.info).lastSon #getHiddenParam(owner).typ.lastSon addField(obj, s) # create required upFields: @@ -428,8 +429,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = let up = w.skipGenericOwner #echo "up for ", w.name.s, " up ", up.name.s markAsClosure(w, n) - addClosureParam(c, w) # , ow - createUpField(c, w, up) + addClosureParam(c, w, n.info) # , ow + createUpField(c, w, up, n.info) w = up of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkTemplateDef, nkTypeSection: @@ -793,7 +794,7 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = var d = initDetectionPass(fn) detectCapturedVars(body, fn, d) if not d.somethingToDo and fn.isIterator: - addClosureParam(d, fn) + addClosureParam(d, fn, body.info) d.somethingToDo = true if d.somethingToDo: var c = initLiftingPass(fn) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 6a8eccb83a..af074e428c 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -114,6 +114,10 @@ proc createObj*(owner: PSym, info: TLineInfo): PType = rawAddSon(result, nil) incl result.flags, tfFinal result.n = newNodeI(nkRecList, info) + let s = newSym(skType, getIdent("Env_" & info.toFilename & "_" & $info.line), + owner, info) + s.typ = result + result.sym = s proc rawAddField*(obj: PType; field: PSym) = assert field.kind == skField From f9e54c1749672aa9fa92054eb21109561043b547 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 01:45:47 +0100 Subject: [PATCH 08/21] use preferName for type info names --- compiler/ccgtypes.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 68e98e92e2..4e17d628ff 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -817,7 +817,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) = if flags != 0: addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) if isDefined("nimTypeNames"): - addf(m.s[cfsTypeInit3], "$1.name = $2;$n", [name, makeCstring typeToString origType]) + addf(m.s[cfsTypeInit3], "$1.name = $2;$n", + [name, makeCstring typeToString(origType, preferName)]) discard cgsym(m, "TNimType") addf(m.s[cfsVars], "TNimType $1; /* $2 */$n", [name, rope(typeToString(typ))]) From 9b68b878b25ab77d51aaf067b8037a1e2b1283af Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 01:47:37 +0100 Subject: [PATCH 09/21] upcoming/asyncdispatch: addTimer and addEvent fixes --- lib/upcoming/asyncdispatch.nim | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 798a7e890d..6e14a9fe75 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -827,7 +827,7 @@ when defined(windows) or defined(nimdoc): cast[pointer](p.ovl)) {.pop.} - template registerWaitableEvent(mask) = + proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) = let p = getGlobalDispatcher() var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword var hEvent = wsaCreateEvent() @@ -919,7 +919,7 @@ when defined(windows) or defined(nimdoc): ## Be sure your callback ``cb`` returns ``true``, if you want to remove ## watch of `read` notifications, and ``false``, if you want to continue ## receiving notifies. - registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) proc addWrite*(fd: AsyncFD, cb: Callback) = ## Start watching the file descriptor for write availability and then call @@ -936,7 +936,7 @@ when defined(windows) or defined(nimdoc): ## Be sure your callback ``cb`` returns ``true``, if you want to remove ## watch of `write` notifications, and ``false``, if you want to continue ## receiving notifies. - registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) template registerWaitableHandle(p, hEvent, flags, pcd, timeout, handleCallback) = let handleFD = AsyncFD(hEvent) @@ -944,7 +944,8 @@ when defined(windows) or defined(nimdoc): pcd.handleFd = handleFD var ol = PCustomOverlapped() GC_ref(ol) - ol.data = CompletionData(fd: handleFD, cb: handleCallback) + ol.data.fd = handleFD + ol.data.cb = handleCallback # We need to protect our callback environment value, so GC will not free it # accidentally. ol.data.cell = system.protect(rawEnv(ol.data.cb)) @@ -986,6 +987,9 @@ when defined(windows) or defined(nimdoc): discard closeHandle(hEvent) deallocShared(cast[pointer](pcd)) p.handles.excl(fd) + else: + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) @@ -1066,8 +1070,13 @@ when defined(windows) or defined(nimdoc): if cb(fd): # we need this check to avoid exception, if `unregister(event)` was # called in callback. - if ev.hWaiter != 0: unregister(ev) + if ev.hWaiter != 0: + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) deallocShared(cast[pointer](pcd)) + else: + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) ev.hWaiter = pcd.waitFd From 5c46f268802217b774df23d4dcf07a6f7b0ebc20 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 02:08:34 +0100 Subject: [PATCH 10/21] fix upcoming/asyncdispatch properly --- lib/upcoming/asyncdispatch.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 6e14a9fe75..0b5ff89bab 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -1070,9 +1070,7 @@ when defined(windows) or defined(nimdoc): if cb(fd): # we need this check to avoid exception, if `unregister(event)` was # called in callback. - if ev.hWaiter != 0: - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + if ev.hWaiter != 0: unregister(ev) deallocShared(cast[pointer](pcd)) else: GC_ref(pcd.ovl) From d40b7831108cbb9ce7000ebb2462fe1587494926 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 30 Nov 2016 04:55:17 +0200 Subject: [PATCH 11/21] Add comments to latest fixes. One more fix. Some artifacts removed. --- lib/pure/asyncdispatch.nim | 19 ++++++------------- lib/upcoming/asyncdispatch.nim | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 01088c2e7c..1367bc411f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -893,9 +893,11 @@ when defined(windows) or defined(nimdoc): deallocShared(cast[pointer](pcd)) raiseOSError(osLastError()) else: - # we ref pcd.ovl one more time, because it will be unrefed in - # poll() + # we incref `pcd.ovl` and `protect` callback one more time, + # because it will be unrefed and disposed in `poll()` after + # callback finishes. GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) ) # We need to protect our callback environment value, so GC will not free it # accidentally. @@ -956,17 +958,8 @@ when defined(windows) or defined(nimdoc): initAll() else: import selectors - when defined(windows): - import winlean - const - EINTR = WSAEINPROGRESS - EINPROGRESS = WSAEINPROGRESS - EWOULDBLOCK = WSAEWOULDBLOCK - EAGAIN = EINPROGRESS - MSG_NOSIGNAL = 0 - else: - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL type AsyncFD* = distinct cint diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 0b5ff89bab..9a35cf3c89 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -878,9 +878,11 @@ when defined(windows) or defined(nimdoc): deallocShared(cast[pointer](pcd)) raiseOSError(osLastError()) else: - # we ref pcd.ovl one more time, because it will be unrefed in - # poll() + # we incref `pcd.ovl` and `protect` callback one more time, + # because it will be unrefed and disposed in `poll()` after + # callback finishes. GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) ) # We need to protect our callback environment value, so GC will not free it # accidentally. @@ -988,6 +990,9 @@ when defined(windows) or defined(nimdoc): deallocShared(cast[pointer](pcd)) p.handles.excl(fd) else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. GC_ref(pcd.ovl) pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) @@ -1073,6 +1078,9 @@ when defined(windows) or defined(nimdoc): if ev.hWaiter != 0: unregister(ev) deallocShared(cast[pointer](pcd)) else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. GC_ref(pcd.ovl) pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) @@ -1082,17 +1090,8 @@ when defined(windows) or defined(nimdoc): initAll() else: import ioselectors - when defined(windows): - import winlean - const - EINTR = WSAEINPROGRESS - EINPROGRESS = WSAEINPROGRESS - EWOULDBLOCK = WSAEWOULDBLOCK - EAGAIN = EINPROGRESS - MSG_NOSIGNAL = 0 - else: - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL const supportedPlatform = defined(linux) or defined(freebsd) or defined(netbsd) or defined(openbsd) or From b1b2dd606ba736775868853657e331f287423dee Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 11:00:51 +0100 Subject: [PATCH 12/21] fixes #5076 --- compiler/options.nim | 3 +-- tests/modules/UpperCased.nim | 6 ++++++ tests/modules/tuppercased.nim | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/modules/UpperCased.nim create mode 100644 tests/modules/tuppercased.nim diff --git a/compiler/options.nim b/compiler/options.nim index 9edafb17a2..b04f6a9637 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -227,8 +227,7 @@ proc setDefaultLibpath*() = libpath = parentNimLibPath proc canonicalizePath*(path: string): string = - when not FileSystemCaseSensitive: result = path.expandFilename.toLowerAscii - else: result = path.expandFilename + result = path.expandFilename proc shortenDir*(dir: string): string = ## returns the interesting part of a dir diff --git a/tests/modules/UpperCased.nim b/tests/modules/UpperCased.nim new file mode 100644 index 0000000000..7beffcc5f4 --- /dev/null +++ b/tests/modules/UpperCased.nim @@ -0,0 +1,6 @@ + +# bug #5076 + +var str*: string + +UpperCased.str = "hello" diff --git a/tests/modules/tuppercased.nim b/tests/modules/tuppercased.nim new file mode 100644 index 0000000000..65f41becd0 --- /dev/null +++ b/tests/modules/tuppercased.nim @@ -0,0 +1,8 @@ +discard """ + output: "hello" +""" + +import UpperCased + +# stress normalization rules: +echo Upper_Cased.str From a80a0972b7757b23b950ef032019d2b5007d9b19 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 30 Nov 2016 00:25:03 +0200 Subject: [PATCH 13/21] Fixed dynlink with OpenSSL >1.1.0. Added loadLibPattern. --- compiler/cgen.nim | 1 + compiler/options.nim | 11 ------- lib/pure/dynlib.nim | 32 ++++++++++++++++--- lib/pure/net.nim | 33 ++++++------------- lib/wrappers/openssl.nim | 68 ++++++++++++++++++++++++++++++++++------ 5 files changed, 97 insertions(+), 48 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 6e18c8389b..3217b86e4d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -17,6 +17,7 @@ import lowerings, semparallel from modulegraphs import ModuleGraph +from dynlib import libCandidates import strutils except `%` # collides with ropes.`%` diff --git a/compiler/options.nim b/compiler/options.nim index b04f6a9637..04ed2412ed 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -372,17 +372,6 @@ proc findModule*(modulename, currentModule: string): string = result = findFile(m) patchModule() -proc libCandidates*(s: string, dest: var seq[string]) = - var le = strutils.find(s, '(') - var ri = strutils.find(s, ')', le+1) - if le >= 0 and ri > le: - var prefix = substr(s, 0, le - 1) - var suffix = substr(s, ri + 1) - for middle in split(substr(s, le + 1, ri - 1), '|'): - libCandidates(prefix & middle & suffix, dest) - else: - add(dest, s) - proc canonDynlibName(s: string): string = let start = if s.startsWith("lib"): 3 else: 0 let ende = strutils.find(s, {'(', ')', '.'}) diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 906a9d23ee..fda41dadbd 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -11,20 +11,22 @@ ## libraries. On POSIX this uses the ``dlsym`` mechanism, on ## Windows ``LoadLibrary``. +import strutils + type LibHandle* = pointer ## a handle to a dynamically loaded library {.deprecated: [TLibHandle: LibHandle].} -proc loadLib*(path: string, global_symbols=false): LibHandle +proc loadLib*(path: string, global_symbols=false): LibHandle {.gcsafe.} ## loads a library from `path`. Returns nil if the library could not ## be loaded. -proc loadLib*(): LibHandle +proc loadLib*(): LibHandle {.gcsafe.} ## gets the handle from the current executable. Returns nil if the ## library could not be loaded. -proc unloadLib*(lib: LibHandle) +proc unloadLib*(lib: LibHandle) {.gcsafe.} ## unloads the library `lib` proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = @@ -34,7 +36,7 @@ proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = e.msg = "could not find symbol: " & $name raise e -proc symAddr*(lib: LibHandle, name: cstring): pointer +proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.} ## retrieves the address of a procedure/variable from `lib`. Returns nil ## if the symbol could not be found. @@ -44,6 +46,28 @@ proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer = result = symAddr(lib, name) if result == nil: raiseInvalidLibrary(name) +proc libCandidates*(s: string, dest: var seq[string]) = + ## given a library name pattern `s` write possible library names to `dest`. + var le = strutils.find(s, '(') + var ri = strutils.find(s, ')', le+1) + if le >= 0 and ri > le: + var prefix = substr(s, 0, le - 1) + var suffix = substr(s, ri + 1) + for middle in split(substr(s, le + 1, ri - 1), '|'): + libCandidates(prefix & middle & suffix, dest) + else: + add(dest, s) + +proc loadLibPattern*(pattern: string, global_symbols=false): LibHandle = + ## loads a library with name matching `pattern`, similar to what `dlimport` + ## pragma does. Returns nil if the library could not be loaded. + ## Warning: this proc uses the GC and so cannot be used to load the GC. + var candidates = newSeq[string]() + libCandidates(pattern, candidates) + for c in candidates: + result = loadLib(c, global_symbols) + if not result.isNil: break + when defined(posix): # # ========================================================================= diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 863a8a6f41..5e10f22919 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -90,8 +90,8 @@ when defineSsl: SslContext* = ref object context*: SslCtx - extraInternalIndex: int referencedData: HashSet[int] + extraInternal: SslContextExtraInternal SslAcceptResult* = enum AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess @@ -103,6 +103,10 @@ when defineSsl: SslServerGetPskFunc* = proc(identity: string): string + SslContextExtraInternal = ref object of RootRef + serverGetPskFunc: SslServerGetPskFunc + clientGetPskFunc: SslClientGetPskFunc + {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, TSSLAcceptResult: SSLAcceptResult].} @@ -240,11 +244,6 @@ when defineSsl: ErrLoadBioStrings() OpenSSL_add_all_algorithms() - type - SslContextExtraInternal = ref object of RootRef - serverGetPskFunc: SslServerGetPskFunc - clientGetPskFunc: SslClientGetPskFunc - proc raiseSSLError*(s = "") = ## Raises a new SSL error. if s != "": @@ -257,12 +256,6 @@ when defineSsl: var errStr = ErrErrorString(err, nil) raise newException(SSLError, $errStr) - proc getExtraDataIndex*(ctx: SSLContext): int = - ## Retrieves unique index for storing extra data in SSLContext. - result = SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil).int - if result < 0: - raiseSSLError() - proc getExtraData*(ctx: SSLContext, index: int): RootRef = ## Retrieves arbitrary data stored inside SSLContext. if index notin ctx.referencedData: @@ -347,15 +340,11 @@ when defineSsl: discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) - result = SSLContext(context: newCTX, extraInternalIndex: 0, - referencedData: initSet[int]()) - result.extraInternalIndex = getExtraDataIndex(result) - - let extraInternal = new(SslContextExtraInternal) - result.setExtraData(result.extraInternalIndex, extraInternal) + result = SSLContext(context: newCTX, referencedData: initSet[int](), + extraInternal: new(SslContextExtraInternal)) proc getExtraInternal(ctx: SSLContext): SslContextExtraInternal = - return SslContextExtraInternal(ctx.getExtraData(ctx.extraInternalIndex)) + return ctx.extraInternal proc destroyContext*(ctx: SSLContext) = ## Free memory referenced by SSLContext. @@ -379,7 +368,7 @@ when defineSsl: proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; max_psk_len: cuint): cuint {.cdecl.} = - let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX) let hintString = if hint == nil: nil else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) if psk.len.cuint > max_psk_len: @@ -398,8 +387,6 @@ when defineSsl: ## ## Only used in PSK ciphersuites. ctx.getExtraInternal().clientGetPskFunc = fun - assert ctx.extraInternalIndex == 0, - "The pskClientCallback assumes the extraInternalIndex is 0" ctx.context.SSL_CTX_set_psk_client_callback( if fun == nil: nil else: pskClientCallback) @@ -407,7 +394,7 @@ when defineSsl: return ctx.getExtraInternal().serverGetPskFunc proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} = - let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX) let pskString = (ctx.serverGetPskFunc)($identity) if psk.len.cint > max_psk_len: return 0 diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 241ad17aed..bf41270b05 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -37,6 +37,8 @@ else: DLLUtilName = "libcrypto.so" & versions from posix import SocketHandle +import dynlib + type SslStruct {.final, pure.} = object SslPtr* = ptr SslStruct @@ -185,16 +187,61 @@ const BIO_C_DO_STATE_MACHINE = 101 BIO_C_GET_SSL = 110 -proc SSL_library_init*(): cInt{.cdecl, dynlib: DLLSSLName, importc, discardable.} -proc SSL_load_error_strings*(){.cdecl, dynlib: DLLSSLName, importc.} +# Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some +# symbols are loaded dynamically and we don't use them if not found. +var dl {.threadvar.}: LibHandle +template loadSSLLib() = + if dl.isNil: + dl = loadLib() + if dl.isNil: + dl = loadLibPattern(DLLSSLName) + +proc SSL_library_init*(): cint {.discardable.} = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc(): cint {.cdecl.}](symAddr(dl, "SSL_library_init")) + if not theProc.isNil: result = theProc() + +proc SSL_load_error_strings*() = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc() {.cdecl.}](symAddr(dl, "SSL_load_error_strings")) + if not theProc.isNil: theProc() + proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} -proc SSLv23_client_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv23_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv2_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv3_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} +proc SSLv23_client_method*(): PSSL_METHOD {.deprecated.} = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv23_client_method")) + if not theProc.isNil: result = theProc() + if result.isNil: result = TLSv1_method() + +proc SSLv23_method*(): PSSL_METHOD {.deprecated.} = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv23_method")) + if not theProc.isNil: + result = theProc() + if result.isNil: + result = TLSv1_method() + +proc SSLv2_method*(): PSSL_METHOD {.deprecated.} = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv2_method")) + if not theProc.isNil: result = theProc() + if result.isNil: result = TLSv1_method() + +proc SSLv3_method*(): PSSL_METHOD {.deprecated.} = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv3_method")) + if not theProc.isNil: result = theProc() + if result.isNil: result = TLSv1_method() + proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} @@ -261,10 +308,11 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl, proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} -when defined(android): - template OpenSSL_add_all_algorithms*() = discard -else: - proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".} +proc OpenSSL_add_all_algorithms*() = + loadSSLLib() + if not dl.isNil: + let theProc = cast[proc() {.cdecl.}](symAddr(dl, "OPENSSL_add_all_algorithms_conf")) + if not theProc.isNil: theProc() proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} From 975477527a33144f338f01f4a2b1754eb3c1bee9 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 30 Nov 2016 13:53:20 +0200 Subject: [PATCH 14/21] More fixes --- lib/wrappers/openssl.nim | 76 +++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index bf41270b05..491e3a569b 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -189,58 +189,56 @@ const # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some # symbols are loaded dynamically and we don't use them if not found. -var dl {.threadvar.}: LibHandle -template loadSSLLib() = - if dl.isNil: - dl = loadLib() - if dl.isNil: - dl = loadLibPattern(DLLSSLName) +proc thisModule(): LibHandle {.inline.} = + var thisMod {.global.}: LibHandle + if thisMod.isNil: thisMod = loadLib() + result = thisMod + +proc sslModule(): LibHandle {.inline.} = + var sslMod {.global.}: LibHandle + if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + result = sslMod + +proc sslSym(name: string): pointer = + var dl = thisModule() + if not dl.isNil: + result = symAddr(dl, name) + if result.isNil: + dl = sslModule() + if not dl.isNil: + result = symAddr(dl, name) proc SSL_library_init*(): cint {.discardable.} = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc(): cint {.cdecl.}](symAddr(dl, "SSL_library_init")) - if not theProc.isNil: result = theProc() + let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) + if not theProc.isNil: result = theProc() proc SSL_load_error_strings*() = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc() {.cdecl.}](symAddr(dl, "SSL_load_error_strings")) - if not theProc.isNil: theProc() + let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) + if not theProc.isNil: theProc() proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} proc SSLv23_client_method*(): PSSL_METHOD {.deprecated.} = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv23_client_method")) - if not theProc.isNil: result = theProc() - if result.isNil: result = TLSv1_method() + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() proc SSLv23_method*(): PSSL_METHOD {.deprecated.} = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv23_method")) - if not theProc.isNil: - result = theProc() - if result.isNil: - result = TLSv1_method() + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() proc SSLv2_method*(): PSSL_METHOD {.deprecated.} = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv2_method")) - if not theProc.isNil: result = theProc() - if result.isNil: result = TLSv1_method() + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() proc SSLv3_method*(): PSSL_METHOD {.deprecated.} = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](symAddr(dl, "SSLv3_method")) - if not theProc.isNil: result = theProc() - if result.isNil: result = TLSv1_method() + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} @@ -309,10 +307,8 @@ proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc OpenSSL_add_all_algorithms*() = - loadSSLLib() - if not dl.isNil: - let theProc = cast[proc() {.cdecl.}](symAddr(dl, "OPENSSL_add_all_algorithms_conf")) - if not theProc.isNil: theProc() + let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) + if not theProc.isNil: theProc() proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} From 581c0c437b7cefce2ba0f33df70901e2b11181ab Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 30 Nov 2016 16:22:52 +0200 Subject: [PATCH 15/21] In case of synliboverride we support only openssl 1.0.* --- compiler/commands.nim | 2 + lib/wrappers/openssl.nim | 113 +++++++++++++++++++++------------------ 2 files changed, 64 insertions(+), 51 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 590c4871de..f85e535111 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -226,6 +226,8 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool = of "staticlib": result = contains(gGlobalOptions, optGenStaticLib) and not contains(gGlobalOptions, optGenGuiApp) else: localError(info, errGuiConsoleOrLibExpectedButXFound, arg) + of "dynliboverride": + result = isDynlibOverride(arg) else: invalidCmdLineOption(passCmd1, switch, info) proc testCompileOption*(switch: string, info: TLineInfo): bool = diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 491e3a569b..1bd02eaf02 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -187,58 +187,73 @@ const BIO_C_DO_STATE_MACHINE = 101 BIO_C_GET_SSL = 110 -# Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some -# symbols are loaded dynamically and we don't use them if not found. -proc thisModule(): LibHandle {.inline.} = - var thisMod {.global.}: LibHandle - if thisMod.isNil: thisMod = loadLib() - result = thisMod - -proc sslModule(): LibHandle {.inline.} = - var sslMod {.global.}: LibHandle - if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) - result = sslMod - -proc sslSym(name: string): pointer = - var dl = thisModule() - if not dl.isNil: - result = symAddr(dl, name) - if result.isNil: - dl = sslModule() - if not dl.isNil: - result = symAddr(dl, name) - -proc SSL_library_init*(): cint {.discardable.} = - let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) - if not theProc.isNil: result = theProc() - -proc SSL_load_error_strings*() = - let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) - if not theProc.isNil: theProc() - -proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} - proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv23_client_method*(): PSSL_METHOD {.deprecated.} = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() +when compileOption("dynlibOverride", "ssl"): + proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv23_method*(): PSSL_METHOD {.deprecated.} = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv2_method*(): PSSL_METHOD {.deprecated.} = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + template OpenSSL_add_all_algorithms*() = discard +else: + # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some + # symbols are loaded dynamically and we don't use them if not found. + proc thisModule(): LibHandle {.inline.} = + var thisMod {.global.}: LibHandle + if thisMod.isNil: thisMod = loadLib() + result = thisMod -proc SSLv3_method*(): PSSL_METHOD {.deprecated.} = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + proc sslModule(): LibHandle {.inline.} = + var sslMod {.global.}: LibHandle + if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + result = sslMod + + proc sslSym(name: string): pointer = + var dl = thisModule() + if not dl.isNil: + result = symAddr(dl, name) + if result.isNil: + dl = sslModule() + if not dl.isNil: + result = symAddr(dl, name) + + proc SSL_library_init*(): cint {.discardable.} = + let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) + if not theProc.isNil: result = theProc() + + proc SSL_load_error_strings*() = + let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) + if not theProc.isNil: theProc() + + proc SSLv23_client_method*(): PSSL_METHOD = + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() + + proc SSLv23_method*(): PSSL_METHOD = + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() + + proc SSLv2_method*(): PSSL_METHOD = + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() + + proc SSLv3_method*(): PSSL_METHOD = + let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method")) + if not theProc.isNil: result = theProc() + else: result = TLSv1_method() + + proc OpenSSL_add_all_algorithms*() = + let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) + if not theProc.isNil: theProc() + +proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} @@ -306,10 +321,6 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl, proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} -proc OpenSSL_add_all_algorithms*() = - let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) - if not theProc.isNil: theProc() - proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} when not useWinVersion and not defined(macosx) and not defined(android): From 20cf28adde8c6e3177f1cc0eca6fe249bc4de438 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 19:02:16 +0100 Subject: [PATCH 16/21] fixes the regressions introduced by fix for #5076 --- compiler/ccgexprs.nim | 8 ++++++-- compiler/extccomp.nim | 2 +- compiler/options.nim | 11 ++++++++++- compiler/pragmas.nim | 1 + 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 2761f888b9..b863568d3a 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1129,7 +1129,11 @@ proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = proc genConstExpr(p: BProc, n: PNode): Rope proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = - if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr: + proc forbiddenType(t: PType): bool {.inline.} = + result = t.kind == tyObject and not isObjLackingTypeField(t) + #echo "forbidden type ", result + if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr and + not forbiddenType(n.typ): var t = getUniqueType(n.typ) discard getTypeDesc(p.module, t) # so that any fields are initialized let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) @@ -1144,8 +1148,8 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = result = false proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = - if handleConstExpr(p, e, d): return #echo rendertree e, " ", e.isDeepConstExpr + if handleConstExpr(p, e, d): return var tmp: TLoc var t = e.typ.skipTypes(abstractInst) getTemp(p, t, tmp) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 6f8b0b197e..1f9af95a5a 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -34,7 +34,7 @@ type TInfoCCProps* = set[TInfoCCProp] TInfoCC* = tuple[ name: string, # the short name of the compiler - objExt: string, # the compiler's object file extenstion + objExt: string, # the compiler's object file extension optSpeed: string, # the options for optimization for speed optSize: string, # the options for optimization for size compilerExe: string, # the compiler's executable diff --git a/compiler/options.nim b/compiler/options.nim index 04ed2412ed..50f12d843a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -227,7 +227,16 @@ proc setDefaultLibpath*() = libpath = parentNimLibPath proc canonicalizePath*(path: string): string = - result = path.expandFilename + # on Windows, 'expandFilename' calls getFullPathName which doesn't do + # case corrections, so we have to use this convoluted way of retrieving + # the true filename (see tests/modules and Nimble uses 'import Uri' instead + # of 'import uri'): + when defined(windows): + result = path.expandFilename + for x in walkFiles(result): + return x + else: + result = path.expandFilename proc shortenDir*(dir: string): string = ## returns the interesting part of a dir diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index e11a8d08b4..ffb2aa8123 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -490,6 +490,7 @@ proc pragmaLine(c: PContext, n: PNode) = elif y.kind != nkIntLit: localError(n.info, errIntLiteralExpected) else: + # XXX this produces weird paths which are not properly resolved: n.info.fileIndex = msgs.fileInfoIdx(x.strVal) n.info.line = int16(y.intVal) else: From 7b44896e03cde0071bb3d9ab25ca846c4dfc599c Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 21:10:22 +0100 Subject: [PATCH 17/21] fixes #5079 --- compiler/semexprs.nim | 3 ++- tests/ccgbugs/tmissingderef2.nim | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/ccgbugs/tmissingderef2.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8aaf4f9d85..723045fb0f 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1274,7 +1274,8 @@ proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode = # this is ugly. XXX Semantic checking should use the ``nfSem`` flag for # nodes? let aOrig = nOrig[0] - result = newNode(nkCall, n.info, sons = @[setterId, a[0], semExpr(c, n[1])]) + result = newNode(nkCall, n.info, sons = @[setterId, a[0], + semExprWithType(c, n[1])]) result.flags.incl nfDotSetter let orig = newNode(nkCall, n.info, sons = @[setterId, aOrig[0], nOrig[1]]) result = semOverloadedCallAnalyseEffects(c, result, orig, {}) diff --git a/tests/ccgbugs/tmissingderef2.nim b/tests/ccgbugs/tmissingderef2.nim new file mode 100644 index 0000000000..59cd24dd16 --- /dev/null +++ b/tests/ccgbugs/tmissingderef2.nim @@ -0,0 +1,25 @@ +discard """ + output: "c" +""" + +# bug #5079 + +import tables, strutils + +type Test = ref object + s: string + +proc `test=`(t: Test, s: string) = + t.s = s + +var t = Test() + +#t.test = spaces(2) # -- works + +var a = newTable[string, string]() +a["b"] = "c" + +#t.s = a["b"] # -- works +#t.test a["b"] # -- works +t.test = a["b"] # -- prints "out of memory" and quits +echo t.s From 8494338bcbe1ffbd9845a6a5318d0853952b25d0 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 30 Nov 2016 22:25:56 +0100 Subject: [PATCH 18/21] fixes #5055 --- compiler/ccgexprs.nim | 9 ++++----- compiler/ccgmerge.nim | 2 +- compiler/cgendata.nim | 2 +- tests/ccgbugs/tobjconstr_regression.nim | 14 ++++++++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 tests/ccgbugs/tobjconstr_regression.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index b863568d3a..feb70071a5 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1129,11 +1129,7 @@ proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = proc genConstExpr(p: BProc, n: PNode): Rope proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = - proc forbiddenType(t: PType): bool {.inline.} = - result = t.kind == tyObject and not isObjLackingTypeField(t) - #echo "forbidden type ", result - if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr and - not forbiddenType(n.typ): + if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr: var t = getUniqueType(n.typ) discard getTypeDesc(p.module, t) # so that any fields are initialized let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) @@ -2156,6 +2152,9 @@ proc genNamedConstExpr(p: BProc, n: PNode): Rope = proc genConstSimpleList(p: BProc, n: PNode): Rope = var length = sonsLen(n) result = rope("{") + if n.kind == nkObjConstr and not isObjLackingTypeField(n.typ): + addf(result, "{$1}", [genTypeInfo(p.module, n.typ)]) + if n.len > 1: add(result, ",") for i in countup(ord(n.kind == nkObjConstr), length - 2): addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])]) if length > ord(n.kind == nkObjConstr): diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 92b6aa9dc8..67ffaf8b08 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -27,8 +27,8 @@ const cfsFieldInfo: "NIM_merge_FIELD_INFO", cfsTypeInfo: "NIM_merge_TYPE_INFO", cfsProcHeaders: "NIM_merge_PROC_HEADERS", - cfsData: "NIM_merge_DATA", cfsVars: "NIM_merge_VARS", + cfsData: "NIM_merge_DATA", cfsProcs: "NIM_merge_PROCS", cfsInitProc: "NIM_merge_INIT_PROC", cfsTypeInit1: "NIM_merge_TYPE_INIT1", diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index faeea7afbe..def0b4feee 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -27,8 +27,8 @@ type cfsFieldInfo, # section for field information cfsTypeInfo, # section for type information cfsProcHeaders, # section for C procs prototypes - cfsData, # section for C constant data cfsVars, # section for C variable declarations + cfsData, # section for C constant data cfsProcs, # section for C procs that are not inline cfsInitProc, # section for the C init proc cfsTypeInit1, # section 1 for declarations of type information diff --git a/tests/ccgbugs/tobjconstr_regression.nim b/tests/ccgbugs/tobjconstr_regression.nim new file mode 100644 index 0000000000..87d0378940 --- /dev/null +++ b/tests/ccgbugs/tobjconstr_regression.nim @@ -0,0 +1,14 @@ +discard """ + output: "@[(username: user, role: admin, description: desc, email_addr: email), (username: user, role: admin, description: desc, email_addr: email)]" +""" + +type + User = object of RootObj + username, role, description, email_addr: string + +# bug 5055 +let us4 = @[ + User(username:"user", role:"admin", description:"desc", email_addr:"email"), + User(username:"user", role:"admin", description:"desc", email_addr:"email"), +] +echo us4 From 2a7b4fda78758005a9c67a0e5f83b7594d616b2a Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 1 Dec 2016 08:46:44 +0100 Subject: [PATCH 19/21] fixes #5081 --- compiler/ccgexprs.nim | 3 ++- tests/cpp/tasync_cpp.nim | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index feb70071a5..b3ad6bbc3f 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2152,7 +2152,8 @@ proc genNamedConstExpr(p: BProc, n: PNode): Rope = proc genConstSimpleList(p: BProc, n: PNode): Rope = var length = sonsLen(n) result = rope("{") - if n.kind == nkObjConstr and not isObjLackingTypeField(n.typ): + if n.kind == nkObjConstr and not isObjLackingTypeField(n.typ) and + not p.module.compileToCpp: addf(result, "{$1}", [genTypeInfo(p.module, n.typ)]) if n.len > 1: add(result, ",") for i in countup(ord(n.kind == nkObjConstr), length - 2): diff --git a/tests/cpp/tasync_cpp.nim b/tests/cpp/tasync_cpp.nim index 792f2938b9..81215bd58a 100644 --- a/tests/cpp/tasync_cpp.nim +++ b/tests/cpp/tasync_cpp.nim @@ -8,4 +8,7 @@ discard """ import jester import asyncdispatch, asyncnet +# bug #5081 +import nre + echo "hello" From 2adb2be86a5302e532d8463243cbdd57beaa0d4f Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 1 Dec 2016 09:01:00 +0100 Subject: [PATCH 20/21] fixes tconsttable regression --- compiler/ccgexprs.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index b3ad6bbc3f..18b0e5e200 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2152,9 +2152,10 @@ proc genNamedConstExpr(p: BProc, n: PNode): Rope = proc genConstSimpleList(p: BProc, n: PNode): Rope = var length = sonsLen(n) result = rope("{") - if n.kind == nkObjConstr and not isObjLackingTypeField(n.typ) and + let t = n.typ.skipTypes(abstractInst) + if n.kind == nkObjConstr and not isObjLackingTypeField(t) and not p.module.compileToCpp: - addf(result, "{$1}", [genTypeInfo(p.module, n.typ)]) + addf(result, "{$1}", [genTypeInfo(p.module, t)]) if n.len > 1: add(result, ",") for i in countup(ord(n.kind == nkObjConstr), length - 2): addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])]) From de844c7767fa341c96c38e279f064a0e7f4db641 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 1 Dec 2016 09:54:45 +0100 Subject: [PATCH 21/21] attempt to make travis green again --- tests/cpp/tasync_cpp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cpp/tasync_cpp.nim b/tests/cpp/tasync_cpp.nim index 81215bd58a..ec78ae26c3 100644 --- a/tests/cpp/tasync_cpp.nim +++ b/tests/cpp/tasync_cpp.nim @@ -9,6 +9,6 @@ import jester import asyncdispatch, asyncnet # bug #5081 -import nre +#import nre echo "hello"