From 8b10609452f848a39d9c141dc96bf11d23f30eeb Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Wed, 1 Feb 2017 14:50:30 +0700 Subject: [PATCH 01/50] Allow .async pragma on methods (#5312) --- lib/pure/asyncmacro.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 2c3a099648..f74881c6d0 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -284,9 +284,9 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = ## This macro transforms a single procedure into a closure iterator. ## The ``async`` macro supports a stmtList holding multiple async procedures. - if prc.kind notin {nnkProcDef, nnkLambda}: + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef}: error("Cannot transform this node kind into an async proc." & - " Proc definition or lambda node expected.") + " proc/method definition or lambda node expected.") hint("Processing " & prc[0].getName & " as an async proc.") From 2aec5b6c49b32c5541e091d8023873bc4eceac28 Mon Sep 17 00:00:00 2001 From: Parashurama Date: Wed, 1 Feb 2017 08:51:24 +0100 Subject: [PATCH 02/50] fixes compiler ignoring passC/passL args when setting --cc:compiler. (#5310) This commit change the way passC/passL cmdline arg and setting in config files are parsed. They are added to a separate linkOptionsCmd/compileOptionsCmd and are inserted when compile/linking command list are requested. --- compiler/commands.nim | 4 ++-- compiler/extccomp.nim | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 61189fba1f..aac7405537 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -509,10 +509,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = else: localError(info, errGuiConsoleOrLibExpectedButXFound, arg) of "passc", "t": expectArg(switch, arg, pass, info) - if pass in {passCmd2, passPP}: extccomp.addCompileOption(arg) + if pass in {passCmd2, passPP}: extccomp.addCompileOptionCmd(arg) of "passl", "l": expectArg(switch, arg, pass, info) - if pass in {passCmd2, passPP}: extccomp.addLinkOption(arg) + if pass in {passCmd2, passPP}: extccomp.addLinkOptionCmd(arg) of "cincludes": expectArg(switch, arg, pass, info) if pass in {passCmd2, passPP}: cIncludes.add arg.processPath(info) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 8ca34223b9..0f283b208b 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -392,6 +392,8 @@ type var externalToLink: TLinkedList # files to link in addition to the file # we compiled + linkOptionsCmd: string = "" + compileOptionsCmd: seq[string] = @[] linkOptions: string = "" compileOptions: string = "" ccompilerpath: string = "" @@ -450,6 +452,12 @@ proc addCompileOption*(option: string) = if strutils.find(compileOptions, option, 0) < 0: addOpt(compileOptions, option) +proc addLinkOptionCmd*(option: string) = + addOpt(linkOptionsCmd, option) + +proc addCompileOptionCmd*(option: string) = + compileOptionsCmd.add(option) + proc initVars*() = # we need to define the symbol here, because ``CC`` may have never been set! for i in countup(low(CC), high(CC)): undefSymbol(CC[i].name) @@ -524,6 +532,10 @@ proc add(s: var string, many: openArray[string]) = proc cFileSpecificOptions(cfilename: string): string = result = compileOptions + for option in compileOptionsCmd: + if strutils.find(result, option, 0) < 0: + addOpt(result, option) + var trunk = splitFile(cfilename).name if optCDebug in gGlobalOptions: var key = trunk & ".debug" @@ -544,7 +556,7 @@ proc getCompileOptions: string = result = cFileSpecificOptions("__dummy__") proc getLinkOptions: string = - result = linkOptions + result = linkOptions & " " & linkOptionsCmd & " " for linkedLib in items(cLinkedLibs): result.add(CC[cCompiler].linkLibCmd % linkedLib.quoteShell) for libDir in items(cLibs): From 3c773c189fc4ba4a639a1ca2d910d5a5c6e13b21 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 1 Feb 2017 12:09:18 +0100 Subject: [PATCH 03/50] fixes #4973 --- compiler/semtypes.nim | 13 ++++++++++++- tests/seq/tsequtils.nim | 11 ++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 028baa555e..17c065b498 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -659,7 +659,8 @@ proc addInheritedFields(c: PContext, check: var IntSet, pos: var int, addInheritedFieldsAux(c, check, pos, obj.n) proc semObjectNode(c: PContext, n: PNode, prev: PType): PType = - if n.sonsLen == 0: return newConstraint(c, tyObject) + if n.sonsLen == 0: + return newConstraint(c, tyObject) var check = initIntSet() var pos = 0 var base, realBase: PType = nil @@ -1159,8 +1160,16 @@ proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType = result.sym = prev.sym assignType(prev, result) +proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) = + if prev != nil: + let result = newTypeS(tyAlias, c) + result.rawAddSon typExpr.typ + result.sym = prev.sym + assignType(prev, result) + proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = nil + if gCmd == cmdIdeTools: suggestExpr(c, n) case n.kind of nkEmpty: discard @@ -1168,6 +1177,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = # for ``type(countup(1,3))``, see ``tests/ttoseq``. checkSonsLen(n, 1) let typExpr = semExprWithType(c, n.sons[0], {efInTypeof}) + fixupTypeOf(c, prev, typExpr) result = typExpr.typ of nkPar: if sonsLen(n) == 1: result = semTypeNode(c, n.sons[0], prev) @@ -1234,6 +1244,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = elif op.id == ord(wType): checkSonsLen(n, 2) let typExpr = semExprWithType(c, n.sons[1], {efInTypeof}) + fixupTypeOf(c, prev, typExpr) result = typExpr.typ else: result = semTypeExpr(c, n, prev) diff --git a/tests/seq/tsequtils.nim b/tests/seq/tsequtils.nim index ea85a7f213..06a981e922 100644 --- a/tests/seq/tsequtils.nim +++ b/tests/seq/tsequtils.nim @@ -7,7 +7,8 @@ Filter Iterator: 7 Filter: [3, 5, 7] FilterIt: [1, 3, 7] Concat: [1, 3, 5, 7, 2, 4, 6] -Deduplicate: [1, 2, 3, 4, 5, 7]''' +Deduplicate: [1, 2, 3, 4, 5, 7] +@[()]''' """ @@ -52,4 +53,12 @@ echo "Concat: ", $$(concatseq) var seq3 = @[1,2,3,4,5,5,5,7] var dedupseq = deduplicate(seq3) echo "Deduplicate: ", $$(dedupseq) +# bug #4973 +type + SomeObj = object + OtherObj = object + field: SomeObj +let aSeq = @[OtherObj(field: SomeObj())] +let someObjSeq = aSeq.mapIt(it.field) +echo someObjSeq From d90f3f59aca668d3000d6fd64199bfe720240911 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 1 Feb 2017 13:12:26 +0200 Subject: [PATCH 04/50] Fixes for upcoming asyncdispatch and ioselectors. (#5309) --- lib/pure/ioselects/ioselectors_epoll.nim | 13 ++--- lib/pure/ioselects/ioselectors_kqueue.nim | 21 ++++---- lib/pure/ioselects/ioselectors_poll.nim | 17 +++--- lib/pure/ioselects/ioselectors_select.nim | 13 ++--- lib/upcoming/asyncdispatch.nim | 64 +++++++++++++---------- tests/async/tupcoming_async.nim | 55 ++++++++++++++----- 6 files changed, 113 insertions(+), 70 deletions(-) diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index f8feb73614..3a5cbc87aa 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -165,7 +165,7 @@ proc close*(ev: SelectEvent) = template checkFd(s, f) = if f >= s.maxFD: - raiseIOSelectorsError("Maximum file descriptors exceeded") + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = @@ -188,7 +188,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") doAssert(pkey.events * maskEvents == {}) if pkey.events != events: var epv = epoll_event(events: EPOLLRDHUP) @@ -215,8 +216,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) - + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") if pkey.events != {}: when not defined(android): if pkey.events * {Event.Read, Event.Write} != {}: @@ -277,7 +278,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.efd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) var epv = epoll_event() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: @@ -380,7 +381,7 @@ when not defined(android): proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = int(ev.efd) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") s.setKey(fdi, {Event.User}, 0, data) var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = ev.efd.uint diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 3d2aae1802..01b1b95866 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -119,12 +119,13 @@ proc newSelector*[T](): Selector[T] = result.maxFD = maxFD.int proc close*[T](s: Selector[T]) = - let res = posix.close(s.kqFD) + let res1 = posix.close(s.kqFD) + let res2 = posix.close(s.sock) when hasThreadSupport: deinitLock(s.changesLock) deallocSharedArray(s.fds) deallocShared(cast[pointer](s)) - if res != 0: + if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) template clearKey[T](key: ptr SelectorKey[T]) = @@ -157,7 +158,7 @@ proc close*(ev: SelectEvent) = template checkFd(s, f) = if f >= s.maxFD: - raiseIOSelectorsError("Maximum file descriptors exceeded!") + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") when hasThreadSupport: template withChangeLock[T](s: Selector[T], body: untyped) = @@ -241,7 +242,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") doAssert(pkey.events * maskEvents == {}) if pkey.events != events: @@ -329,7 +331,7 @@ proc registerProcess*[T](s: Selector[T], pid: int, proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = ev.rfd.int - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") setKey(s, fdi, {Event.User}, 0, data) modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) @@ -372,7 +374,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") if pkey.events != {}: if pkey.events * {Event.Read, Event.Write} != {}: @@ -431,9 +434,8 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.rfd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) - modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) when not declared(CACHE_EVENTS): flushKQueue(s) @@ -564,8 +566,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, pkey.events.incl(Event.Finished) rkey.events.incl(Event.Process) else: - pkey = addr(s.fds[cast[int](kevent.udata)]) - raiseIOSelectorsError("Unsupported kqueue filter in queue!") + doAssert(true, "Unsupported kqueue filter in the queue!") if (kevent.flags and EV_EOF) != 0: rkey.events.incl(Event.Error) diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 9c6f9796f0..1b90e08066 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -115,9 +115,8 @@ template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = s.pollfds[i].events = pollev break inc(i) - - if i == s.pollcnt: - raiseIOSelectorsError("Descriptor is not registered in queue") + doAssert(i < s.pollcnt, + "Descriptor [" & $sock & "] is not registered in the queue!") template pollRemove[T](s: Selector[T], sock: cint) = withPollLock(s): @@ -140,7 +139,7 @@ template pollRemove[T](s: Selector[T], sock: cint) = template checkFd(s, f) = if f >= s.maxFD: - raiseIOSelectorsError("Descriptor is not registered in queue") + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = @@ -157,7 +156,8 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") doAssert(pkey.events * maskEvents == {}) if pkey.events != events: @@ -172,7 +172,7 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = var fdi = int(ev.rfd) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") var events = {Event.User} setKey(s, fdi, events, 0, data) events.incl(Event.Read) @@ -182,7 +182,8 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, + "Descriptor [" & $fdi & "] is not registered in the queue!") pkey.ident = 0 pkey.events = {} s.pollRemove(fdi.cint) @@ -191,7 +192,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.rfd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != 0, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) pkey.ident = 0 pkey.events = {} diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 7a7d239823..dc3451d52e 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -202,8 +202,8 @@ proc setSelectKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], pkey.data = data break inc(i) - if i == FD_SETSIZE: - raiseIOSelectorsError("Maximum numbers of fds exceeded") + if i >= FD_SETSIZE: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = var i = 0 @@ -213,8 +213,8 @@ proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = result = addr(s.fds[i]) break inc(i) - if i == FD_SETSIZE: - raiseIOSelectorsError("Descriptor not registered in queue") + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") proc delKey[T](s: Selector[T], fd: SocketHandle) = var empty: T @@ -226,8 +226,8 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) = s.fds[i].data = empty break inc(i) - if i == FD_SETSIZE: - raiseIOSelectorsError("Descriptor not registered in queue") + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = @@ -294,6 +294,7 @@ proc unregister*[T](s: Selector[T], fd: SocketHandle) = proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fd = ev.rsock s.withSelectLock(): + var pkey = s.getKey(fd) IOFD_CLR(fd, addr s.rSet) dec(s.count) s.delKey(fd) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 31aa6c9cb3..1dfd0122a7 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -1056,16 +1056,14 @@ when defined(windows) or defined(nimdoc): proc unregister*(ev: AsyncEvent) = ## Unregisters event ``ev``. - if ev.hWaiter != 0: - let p = getGlobalDispatcher() - p.handles.excl(AsyncFD(ev.hEvent)) - if unregisterWait(ev.hWaiter) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - raiseOSError(err) - ev.hWaiter = 0 - else: - raise newException(ValueError, "Event is not registered!") + doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") + let p = getGlobalDispatcher() + p.handles.excl(AsyncFD(ev.hEvent)) + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(err) + ev.hWaiter = 0 proc close*(ev: AsyncEvent) = ## Closes event ``ev``. @@ -1076,8 +1074,7 @@ when defined(windows) or defined(nimdoc): proc addEvent*(ev: AsyncEvent, cb: Callback) = ## Registers callback ``cb`` to be called when ``ev`` will be signaled - if ev.hWaiter != 0: - raise newException(ValueError, "Event is already registered!") + doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") let p = getGlobalDispatcher() let hEvent = ev.hEvent @@ -1086,17 +1083,22 @@ when defined(windows) or defined(nimdoc): var flags = WT_EXECUTEINWAITTHREAD.Dword proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if cb(fd): - # we need this check to avoid exception, if `unregister(event)` was - # called in callback. - deallocShared(cast[pointer](pcd)) - if ev.hWaiter != 0: unregister(ev) + if ev.hWaiter != 0: + if cb(fd): + # we need this check to avoid exception, if `unregister(event)` was + # called in callback. + deallocShared(cast[pointer](pcd)) + if ev.hWaiter != 0: + unregister(ev) + else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + # if ev.hWaiter == 0, then event was unregistered before `poll()` call. + deallocShared(cast[pointer](pcd)) registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) ev.hWaiter = pcd.waitFd @@ -1205,7 +1207,7 @@ else: not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 template processBasicCallbacks(ident, rwlist: untyped) = - # Process pending descriptor's callbacks. + # Process pending descriptor's and AsyncEvent callbacks. # Invoke every callback stored in `rwlist`, until first one # returned `false`, which means callback wants to stay # alive. In such case all remaining callbacks will be added @@ -1232,6 +1234,8 @@ else: withData(p.selector, ident, adata) do: adata.rwlist = newList & adata.rwlist + rLength = len(adata.readList) + wLength = len(adata.writeList) template processCustomCallbacks(ident: untyped) = # Process pending custom event callbacks. Custom events are @@ -1275,6 +1279,8 @@ else: var custom = false let fd = keys[i].fd let events = keys[i].events + var rLength = 0 # len(data.readList) after callback + var wLength = 0 # len(data.writeList) after callback if Event.Read in events or events == {Event.Error}: processBasicCallbacks(fd, readList) @@ -1283,8 +1289,10 @@ else: processBasicCallbacks(fd, writeList) if Event.User in events or events == {Event.Error}: - custom = true processBasicCallbacks(fd, readList) + custom = true + if rLength == 0: + p.selector.unregister(fd) when ioselSupportedPlatform: if (customSet * events) != {}: @@ -1296,10 +1304,12 @@ else: if not custom: var update = false var newEvents: set[Event] = {} - p.selector.withData(fd, adata) do: - if len(adata.readList) > 0: incl(newEvents, Event.Read) - if len(adata.writeList) > 0: incl(newEvents, Event.Write) + if rLength > 0: update = true + incl(newEvents, Event.Read) + if wLength > 0: + update = true + incl(newEvents, Event.Write) if update: p.selector.updateHandle(SocketHandle(fd), newEvents) inc(i) diff --git a/tests/async/tupcoming_async.nim b/tests/async/tupcoming_async.nim index 7d255f2133..0fe9f08a5c 100644 --- a/tests/async/tupcoming_async.nim +++ b/tests/async/tupcoming_async.nim @@ -1,9 +1,6 @@ discard """ output: ''' OK -OK -OK -OK ''' """ @@ -31,11 +28,39 @@ when defined(upcoming): var fut = waitEvent(event) asyncCheck(delayedSet(event, 500)) waitFor(fut or sleepAsync(1000)) - if fut.finished: - echo "OK" - else: + if not fut.finished: echo "eventTest: Timeout expired before event received!" + proc eventTest5304() = + # Event should not be signaled if it was uregistered, + # even in case, when poll() was not called yet. + # Issue #5304. + var unregistered = false + let e = newAsyncEvent() + addEvent(e) do (fd: AsyncFD) -> bool: + assert(not unregistered) + e.setEvent() + e.unregister() + unregistered = true + poll() + + proc eventTest5298() = + # Event must raise `AssertionError` if event was unregistered twice. + # Issue #5298. + let e = newAsyncEvent() + var eventReceived = false + addEvent(e) do (fd: AsyncFD) -> bool: + eventReceived = true + return true + e.setEvent() + while not eventReceived: + poll() + try: + e.unregister() + except AssertionError: + discard + e.close() + when ioselSupportedPlatform or defined(windows): import osproc @@ -56,7 +81,6 @@ when defined(upcoming): proc timerTest() = waitFor(waitTimer(200)) - echo "OK" proc processTest() = when defined(windows): @@ -70,7 +94,7 @@ when defined(upcoming): var fut = waitProcess(process) waitFor(fut or waitTimer(2000)) if fut.finished and process.peekExitCode() == 0: - echo "OK" + discard else: echo "processTest: Timeout expired before process exited!" @@ -92,23 +116,28 @@ when defined(upcoming): var fut = waitSignal(posix.SIGINT) asyncCheck(delayedSignal(posix.SIGINT, 500)) waitFor(fut or waitTimer(1000)) - if fut.finished: - echo "OK" - else: + if not fut.finished: echo "signalTest: Timeout expired before signal received!" when ioselSupportedPlatform: timerTest() eventTest() + eventTest5304() + eventTest5298() processTest() signalTest() + echo "OK" elif defined(windows): timerTest() eventTest() + eventTest5304() + eventTest5298() processTest() echo "OK" else: eventTest() - echo "OK\nOK\nOK" + eventTest5304() + eventTest5298() + echo "OK" else: - echo "OK\nOK\nOK\nOK" + echo "OK" From c57fcf42df4206229de10c8ed0463fe94517bd4b Mon Sep 17 00:00:00 2001 From: Parashurama Date: Wed, 1 Feb 2017 12:13:01 +0100 Subject: [PATCH 05/50] fix string slice & splice (#5311) code fixes courtesy of @memophen --- lib/system.nim | 11 ++++---- tests/stdlib/tstring.nim | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/stdlib/tstring.nim diff --git a/lib/system.nim b/lib/system.nim index 75014ff269..09d48fd12b 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3252,19 +3252,18 @@ proc `/`*(x, y: int): float {.inline, noSideEffect.} = template spliceImpl(s, a, L, b: untyped): untyped = # make room for additional elements or cut: - var slen = s.len - var shift = b.len - L - var newLen = slen + shift + var shift = b.len - max(0,L) # ignore negative slice size + var newLen = s.len + shift if shift > 0: # enlarge: setLen(s, newLen) - for i in countdown(newLen-1, a+shift+1): shallowCopy(s[i], s[i-shift]) + for i in countdown(newLen-1, a+b.len): shallowCopy(s[i], s[i-shift]) else: - for i in countup(a+b.len, s.len-1+shift): shallowCopy(s[i], s[i-shift]) + for i in countup(a+b.len, newLen-1): shallowCopy(s[i], s[i-shift]) # cut down: setLen(s, newLen) # fill the hole: - for i in 0 .. len(numbers)": + # replace characters by slice of same length + s = characters + s[1..16] = numbers + doAssert s == "a1234567890rstuvwxyz" + + # test "slice of length == len(numbers)": + # replace characters by slice of same length + s = characters + s[1..10] = numbers + doAssert s == "a1234567890lmnopqrstuvwxyz" + + # test "slice of length < len(numbers)": + # replace slice of length. and insert remaining chars + s = characters + s[1..4] = numbers + doAssert s == "a1234567890fghijklmnopqrstuvwxyz" + + # test "slice of length == 1": + # replace first character. and insert remaining 9 chars + s = characters + s[1..1] = numbers + doAssert s == "a1234567890cdefghijklmnopqrstuvwxyz" + + # test "slice of length == 0": + # insert chars at slice start index + s = characters + s[2..1] = numbers + doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz" + + # test "slice of negative length": + # same as slice of zero length + s = characters + s[2..0] = numbers + doAssert s == "ab1234567890cdefghijklmnopqrstuvwxyz" + + echo("OK") + +test_string_slice() From fb8168d338cedd04bad6876538ffa02fd975568b Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Mon, 30 Jan 2017 13:18:32 +0700 Subject: [PATCH 06/50] Fix Windows accept() to fail future instead of raising Resolves: #5279 --- lib/pure/asyncdispatch.nim | 43 +++++++++++++++++--------------- lib/pure/nativesockets.nim | 3 ++- lib/pure/net.nim | 25 +++++++++++++++++++ lib/upcoming/asyncdispatch.nim | 43 +++++++++++++++++--------------- lib/windows/winlean.nim | 8 +++--- tests/async/tacceptcloserace.nim | 29 +++++++++++++++++++++ 6 files changed, 107 insertions(+), 44 deletions(-) create mode 100644 tests/async/tacceptcloserace.nim diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 8db7eba253..107e26c0cd 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -753,26 +753,6 @@ when defined(windows) or defined(nimdoc): let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) - template completeAccept() {.dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: raiseOSError(osLastError()) - - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - register(clientSock.AsyncFD) - # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 - retFuture.complete( - (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), - client: clientSock.AsyncFD) - ) - template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) @@ -785,6 +765,29 @@ when defined(windows) or defined(nimdoc): else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) + template completeAccept() {.dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: + let errcode = osLastError() + checkCloseError clientSock.closeSocket() + failAccept(errcode) + else: + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + var ol = PCustomOverlapped() GC_ref(ol) ol.data = CompletionData(fd: socket, cb: diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 5f10a7b4cb..d51dbd4750 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -22,11 +22,12 @@ const useWinVersion = defined(Windows) or defined(nimdoc) when useWinVersion: import winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, + WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: import posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, - EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET + EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7f67833582..3699835c21 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -191,6 +191,31 @@ proc isDisconnectionError*(flags: set[SocketFlag], SocketFlag.SafeDisconn in flags and lastError.int32 in {ECONNRESET, EPIPE, ENETRESET} +proc checkCloseError*(ret: cint) = + ## Asserts that the return value of close() or closeSocket() syscall + ## does not indicate a programming error (such as invalid descriptor). + ## This must only be used when an error has already occurred and + ## you are performing a cleanup. + ## Otherwise, error handling must be performed as usual. + ## + ## This procedure must be called right after perfoming the syscall. Example: + ## + ## .. code-block:: nim + ## + ## let ret = someSysCall() + ## if ret != 0: + ## let errcode = osLastError() + ## checkCloseError sock.closeSocket() + ## raise newException(OSError, osErrorMsg(errcode)) + + if ret != 0: + let errcode = osLastError() + when useWinVersion: + doAssert(errcode.int32 notin {WSANOTINITIALISED, WSAENOTSOCK, + WSAEINPROGRESS, WSAEINTR, WSAEWOULDBLOCK}) + else: + doAssert(errcode.int32 notin {EBADF}) + proc toOSFlags*(socketFlags: set[SocketFlag]): cint = ## Converts the flags into the underlying OS representation. for f in socketFlags: diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 1dfd0122a7..1fef7182d3 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -738,26 +738,6 @@ when defined(windows) or defined(nimdoc): let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) - template completeAccept() {.dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: raiseOSError(osLastError()) - - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - register(clientSock.AsyncFD) - # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 - retFuture.complete( - (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), - client: clientSock.AsyncFD) - ) - template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) @@ -770,6 +750,29 @@ when defined(windows) or defined(nimdoc): else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) + template completeAccept() {.dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: + let errcode = osLastError() + checkCloseError clientSock.closeSocket() + failAccept(errcode) + else: + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + var ol = PCustomOverlapped() GC_ref(ol) ol.data = CompletionData(fd: socket, cb: diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 367fa8b815..02821b7921 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -419,9 +419,6 @@ const ws2dll = "Ws2_32.dll" - WSAEWOULDBLOCK* = 10035 - WSAEINPROGRESS* = 10036 - proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll.} type @@ -760,6 +757,11 @@ const WSAEDISCON* = 10101 WSAENETRESET* = 10052 WSAETIMEDOUT* = 10060 + WSANOTINITIALISED* = 10093 + WSAENOTSOCK* = 10038 + WSAEINPROGRESS* = 10036 + WSAEINTR* = 10004 + WSAEWOULDBLOCK* = 10035 ERROR_NETNAME_DELETED* = 64 STATUS_PENDING* = 0x103 diff --git a/tests/async/tacceptcloserace.nim b/tests/async/tacceptcloserace.nim new file mode 100644 index 0000000000..899136b426 --- /dev/null +++ b/tests/async/tacceptcloserace.nim @@ -0,0 +1,29 @@ +import asyncdispatch, net, os, nativesockets + +# bug: https://github.com/nim-lang/Nim/issues/5279 + +proc setupServerSocket(hostname: string, port: Port): AsyncFD = + let fd = newNativeSocket() + setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) + var aiList = getAddrInfo(hostname, port) + if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: + freeAddrInfo(aiList) + raiseOSError(osLastError()) + freeAddrInfo(aiList) + if listen(fd) != 0: + raiseOSError(osLastError()) + setBlocking(fd, false) + result = fd.AsyncFD + register(result) + +const port = Port(5614) +for i in 0..100: + let serverFd = setupServerSocket("localhost", port) + serverFd.accept().callback = proc(fut: Future[AsyncFD]) = + if not fut.failed: + fut.read().closeSocket() + + var fd = newAsyncNativeSocket() + waitFor fd.connect("localhost", port) + serverFd.closeSocket() + fd.closeSocket() From 5565b9ef10f5022ea1bdf84fd40c1bca4a6e02e8 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 1 Feb 2017 21:15:47 +0100 Subject: [PATCH 07/50] Fixes #5318. --- doc/lib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lib.rst b/doc/lib.rst index 6b498e6969..b43f295ef0 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -586,4 +586,4 @@ Nim programming language. nimblepkglist.js or have javascript disabled in your browser. - + From f04d21f2793933f9b9a54ef2d67dc277e53a0c67 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 1 Feb 2017 15:39:56 +0100 Subject: [PATCH 08/50] refactoring: explict config state instead of globals --- compiler/cgen.nim | 5 +- compiler/cgendata.nim | 1 + compiler/commands.nim | 12 +++- compiler/condsyms.nim | 1 + compiler/main.nim | 4 +- compiler/modulegraphs.nim | 6 +- compiler/nim.nim | 12 ++-- compiler/nimconf.nim | 106 ++++++++++++++++---------------- compiler/options.nim | 13 +++- compiler/scriptconfig.nim | 12 ++-- lib/system/nimscript.nim | 5 ++ tools/nimsuggest/nimsuggest.nim | 12 ++-- 12 files changed, 110 insertions(+), 79 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 476b1362f5..94a8567739 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1221,7 +1221,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = injectG() result = newModule(g, module) if optGenIndex in gGlobalOptions and g.generatedHeader == nil: - let f = if headerFile.len > 0: headerFile else: gProjectFull + let f = if graph.config.headerFile.len > 0: graph.config.headerFile else: gProjectFull g.generatedHeader = rawNewModule(g, module, changeFileExt(completeCFilePath(f), hExt)) incl g.generatedHeader.flags, isHeaderFile @@ -1373,11 +1373,12 @@ proc myClose(b: PPassContext, n: PNode): PNode = for i in 0..sonsLen(disp)-1: genProcAux(m, disp.sons[i].sym) genMainProc(m) -proc cgenWriteModules*(backend: RootRef) = +proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = let g = BModuleList(backend) # we need to process the transitive closure because recursive module # deps are allowed (and the system module is processed in the wrong # order anyway) + g.config = config if g.generatedHeader != nil: finishModule(g.generatedHeader) while g.forwardedProcsCounter > 0: for m in cgenModules(g): diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 94d34c5cf1..8446b9db2e 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -115,6 +115,7 @@ type breakPointId*: int breakpoints*: Rope # later the breakpoints are inserted into the main proc typeInfoMarker*: TypeCache + config*: ConfigRef TCGen = object of TPassContext # represents a C source file s*: TCFileSections # sections of the C file diff --git a/compiler/commands.nim b/compiler/commands.nim index aac7405537..74503a414e 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -47,7 +47,8 @@ type passPP # preprocessor called processCommand() proc processCommand*(switch: string, pass: TCmdLinePass) -proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) +proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; + config: ConfigRef = nil) # implementation @@ -312,7 +313,8 @@ proc dynlibOverride(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = expectArg(switch, arg, pass, info) options.inclDynlibOverride(arg) -proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = +proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; + config: ConfigRef = nil) = var theOS: TSystemOS cpu: TSystemCPU @@ -523,7 +525,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = expectArg(switch, arg, pass, info) if pass in {passCmd2, passPP}: cLinkedLibs.add arg.processPath(info) of "header": - headerFile = arg + if config != nil: config.headerFile = arg incl(gGlobalOptions, optGenIndex) of "index": processOnOffSwitchG({optGenIndex}, arg, pass, info) @@ -646,6 +648,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optNoCppExceptions) defineSymbol("noCppExceptions") + of "cppdefine": + expectArg(switch, arg, pass, info) + if config != nil: + config.cppDefine(arg) else: if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg) else: invalidCmdLineOption(pass, switch, info) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 98c72f8627..a738ddb488 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -101,3 +101,4 @@ proc initDefines*() = defineSymbol("nimImmediateDeprecated") defineSymbol("nimNewShiftOps") defineSymbol("nimDistros") + defineSymbol("nimHasCppDefine") diff --git a/compiler/main.nim b/compiler/main.nim index 888f89ad53..2acb7620c6 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -72,7 +72,7 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = #registerPass(cleanupPass()) compileProject(graph, cache) - cgenWriteModules(graph.backend) + cgenWriteModules(graph.backend, graph.config) if gCmd != cmdRun: let proj = changeFileExt(gProjectFull, "") extccomp.callCCompiler(proj) @@ -294,4 +294,4 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) = resetAttributes() -proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache()) +proc mainCommand*() = mainCommand(newModuleGraph(newConfigRef()), newIdentCache()) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 466e12e645..0ae7076d96 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -25,7 +25,7 @@ ## - Its dependent module stays the same. ## -import ast, intsets, tables +import ast, intsets, tables, options type ModuleGraph* = ref object @@ -39,16 +39,18 @@ type importStack*: seq[int32] # The current import stack. Used for detecting recursive # module dependencies. backend*: RootRef # minor hack so that a backend can extend this easily + config*: ConfigRef {.this: g.} -proc newModuleGraph*(): ModuleGraph = +proc newModuleGraph*(config: ConfigRef = nil): ModuleGraph = result = ModuleGraph() initStrTable(result.packageSyms) result.deps = initIntSet() result.modules = @[] result.importStack = @[] result.inclToMod = initTable[int32, int32]() + result.config = config proc resetAllModules*(g: ModuleGraph) = initStrTable(packageSyms) diff --git a/compiler/nim.nim b/compiler/nim.nim index c458f76f9a..56885e9f1b 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -37,7 +37,7 @@ proc prependCurDir(f: string): string = else: result = f -proc handleCmdLine(cache: IdentCache) = +proc handleCmdLine(cache: IdentCache; config: ConfigRef) = if paramCount() == 0: writeCommandLineUsage() else: @@ -59,22 +59,22 @@ proc handleCmdLine(cache: IdentCache) = gProjectName = p.name else: gProjectPath = canonicalizePath getCurrentDir() - loadConfigs(DefaultConfig) # load all config files + loadConfigs(DefaultConfig, config) # load all config files let scriptFile = gProjectFull.changeFileExt("nims") if fileExists(scriptFile): - runNimScript(cache, scriptFile, freshDefines=false) + runNimScript(cache, scriptFile, freshDefines=false, config) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if scriptFile == gProjectFull: return elif fileExists(gProjectPath / "config.nims"): # directory wide NimScript file - runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false, config) # now process command line arguments again, because some options in the # command line can overwite the config file's settings extccomp.initVars() processCmdLine(passCmd2, "") if options.command == "": rawMessage(errNoCommand, command) - mainCommand(newModuleGraph(), cache) + mainCommand(newModuleGraph(config), cache) if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics()) #echo(GC_getStatistics()) if msgs.gErrorCounter == 0: @@ -118,5 +118,5 @@ when compileOption("gc", "v2") or compileOption("gc", "refc"): condsyms.initDefines() when not defined(selftest): - handleCmdLine(newIdentCache()) + handleCmdLine(newIdentCache(), newConfigRef()) msgQuit(int8(msgs.gErrorCounter > 0)) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 4bf2fbc9a6..808159b8f9 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -21,37 +21,37 @@ proc ppGetTok(L: var TLexer, tok: var TToken) = rawGetTok(L, tok) while tok.tokType in {tkComment}: rawGetTok(L, tok) -proc parseExpr(L: var TLexer, tok: var TToken): bool -proc parseAtom(L: var TLexer, tok: var TToken): bool = +proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool +proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool = if tok.tokType == tkParLe: ppGetTok(L, tok) - result = parseExpr(L, tok) + result = parseExpr(L, tok, config) if tok.tokType == tkParRi: ppGetTok(L, tok) else: lexMessage(L, errTokenExpected, "\')\'") elif tok.ident.id == ord(wNot): ppGetTok(L, tok) - result = not parseAtom(L, tok) + result = not parseAtom(L, tok, config) else: result = isDefined(tok.ident) ppGetTok(L, tok) -proc parseAndExpr(L: var TLexer, tok: var TToken): bool = - result = parseAtom(L, tok) +proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool = + result = parseAtom(L, tok, config) while tok.ident.id == ord(wAnd): ppGetTok(L, tok) # skip "and" - var b = parseAtom(L, tok) + var b = parseAtom(L, tok, config) result = result and b -proc parseExpr(L: var TLexer, tok: var TToken): bool = - result = parseAndExpr(L, tok) +proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool = + result = parseAndExpr(L, tok, config) while tok.ident.id == ord(wOr): ppGetTok(L, tok) # skip "or" - var b = parseAndExpr(L, tok) + var b = parseAndExpr(L, tok, config) result = result or b -proc evalppIf(L: var TLexer, tok: var TToken): bool = +proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool = ppGetTok(L, tok) # skip 'if' or 'elif' - result = parseExpr(L, tok) + result = parseExpr(L, tok, config) if tok.tokType == tkColon: ppGetTok(L, tok) else: lexMessage(L, errTokenExpected, "\':\'") @@ -66,20 +66,20 @@ type TJumpDest = enum jdEndif, jdElseEndif -proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) -proc doElse(L: var TLexer, tok: var TToken) = +proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef) +proc doElse(L: var TLexer, tok: var TToken; config: ConfigRef) = if high(condStack) < 0: lexMessage(L, errTokenExpected, "@if") ppGetTok(L, tok) if tok.tokType == tkColon: ppGetTok(L, tok) - if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif) + if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif, config) -proc doElif(L: var TLexer, tok: var TToken) = +proc doElif(L: var TLexer, tok: var TToken; config: ConfigRef) = if high(condStack) < 0: lexMessage(L, errTokenExpected, "@if") - var res = evalppIf(L, tok) - if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif) + var res = evalppIf(L, tok, config) + if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif, config) else: condStack[high(condStack)] = true -proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) = +proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef) = var nestedIfs = 0 while true: if tok.ident != nil and tok.ident.s == "@": @@ -89,11 +89,11 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) = inc(nestedIfs) of wElse: if dest == jdElseEndif and nestedIfs == 0: - doElse(L, tok) + doElse(L, tok, config) break of wElif: if dest == jdElseEndif and nestedIfs == 0: - doElif(L, tok) + doElif(L, tok, config) break of wEnd: if nestedIfs == 0: @@ -108,16 +108,16 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest) = else: ppGetTok(L, tok) -proc parseDirective(L: var TLexer, tok: var TToken) = +proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef) = ppGetTok(L, tok) # skip @ case whichKeyword(tok.ident) of wIf: setLen(condStack, len(condStack) + 1) - let res = evalppIf(L, tok) + let res = evalppIf(L, tok, config) condStack[high(condStack)] = res - if not res: jumpToDirective(L, tok, jdElseEndif) - of wElif: doElif(L, tok) - of wElse: doElse(L, tok) + if not res: jumpToDirective(L, tok, jdElseEndif, config) + of wElif: doElif(L, tok, config) + of wElse: doElse(L, tok, config) of wEnd: doEnd(L, tok) of wWrite: ppGetTok(L, tok) @@ -146,58 +146,58 @@ proc parseDirective(L: var TLexer, tok: var TToken) = ppGetTok(L, tok) else: lexMessage(L, errInvalidDirectiveX, tokToStr(tok)) -proc confTok(L: var TLexer, tok: var TToken) = +proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef) = ppGetTok(L, tok) while tok.ident != nil and tok.ident.s == "@": - parseDirective(L, tok) # else: give the token to the parser + parseDirective(L, tok, config) # else: give the token to the parser proc checkSymbol(L: TLexer, tok: TToken) = if tok.tokType notin {tkSymbol..pred(tkIntLit), tkStrLit..tkTripleStrLit}: lexMessage(L, errIdentifierExpected, tokToStr(tok)) -proc parseAssignment(L: var TLexer, tok: var TToken) = +proc parseAssignment(L: var TLexer, tok: var TToken; config: ConfigRef) = if tok.ident.s == "-" or tok.ident.s == "--": - confTok(L, tok) # skip unnecessary prefix + confTok(L, tok, config) # skip unnecessary prefix var info = getLineInfo(L, tok) # save for later in case of an error checkSymbol(L, tok) var s = tokToStr(tok) - confTok(L, tok) # skip symbol + confTok(L, tok, config) # skip symbol var val = "" while tok.tokType == tkDot: add(s, '.') - confTok(L, tok) + confTok(L, tok, config) checkSymbol(L, tok) add(s, tokToStr(tok)) - confTok(L, tok) + confTok(L, tok, config) if tok.tokType == tkBracketLe: # BUGFIX: val, not s! # BUGFIX: do not copy '['! - confTok(L, tok) + confTok(L, tok, config) checkSymbol(L, tok) add(val, tokToStr(tok)) - confTok(L, tok) - if tok.tokType == tkBracketRi: confTok(L, tok) + confTok(L, tok, config) + if tok.tokType == tkBracketRi: confTok(L, tok, config) else: lexMessage(L, errTokenExpected, "']'") add(val, ']') let percent = tok.ident != nil and tok.ident.s == "%=" if tok.tokType in {tkColon, tkEquals} or percent: if len(val) > 0: add(val, ':') - confTok(L, tok) # skip ':' or '=' or '%' + confTok(L, tok, config) # skip ':' or '=' or '%' checkSymbol(L, tok) add(val, tokToStr(tok)) - confTok(L, tok) # skip symbol + confTok(L, tok, config) # skip symbol while tok.ident != nil and tok.ident.s == "&": - confTok(L, tok) + confTok(L, tok, config) checkSymbol(L, tok) add(val, tokToStr(tok)) - confTok(L, tok) + confTok(L, tok, config) if percent: processSwitch(s, strtabs.`%`(val, options.gConfigVars, - {useEnvironment, useEmpty}), passPP, info) + {useEnvironment, useEmpty}), passPP, info, config) else: - processSwitch(s, val, passPP, info) + processSwitch(s, val, passPP, info, config) -proc readConfigFile(filename: string; cache: IdentCache) = +proc readConfigFile(filename: string; cache: IdentCache; config: ConfigRef) = var L: TLexer tok: TToken @@ -207,8 +207,8 @@ proc readConfigFile(filename: string; cache: IdentCache) = initToken(tok) openLexer(L, filename, stream, cache) tok.tokType = tkEof # to avoid a pointless warning - confTok(L, tok) # read in the first token - while tok.tokType != tkEof: parseAssignment(L, tok) + confTok(L, tok, config) # read in the first token + while tok.tokType != tkEof: parseAssignment(L, tok, config) if len(condStack) > 0: lexMessage(L, errTokenExpected, "@end") closeLexer(L) rawMessage(hintConf, filename) @@ -225,22 +225,22 @@ proc getSystemConfigPath(filename: string): string = if not existsFile(result): result = joinPath([p, "etc", filename]) if not existsFile(result): result = "/etc/" & filename -proc loadConfigs*(cfg: string; cache: IdentCache) = +proc loadConfigs*(cfg: string; cache: IdentCache; config: ConfigRef = nil) = setDefaultLibpath() if optSkipConfigFile notin gGlobalOptions: - readConfigFile(getSystemConfigPath(cfg), cache) + readConfigFile(getSystemConfigPath(cfg), cache, config) if optSkipUserConfigFile notin gGlobalOptions: - readConfigFile(getUserConfigPath(cfg), cache) + readConfigFile(getUserConfigPath(cfg), cache, config) var pd = if gProjectPath.len > 0: gProjectPath else: getCurrentDir() if optSkipParentConfigFiles notin gGlobalOptions: for dir in parentDirs(pd, fromRoot=true, inclusive=false): - readConfigFile(dir / cfg, cache) + readConfigFile(dir / cfg, cache, config) if optSkipProjConfigFile notin gGlobalOptions: - readConfigFile(pd / cfg, cache) + readConfigFile(pd / cfg, cache, config) if gProjectName.len != 0: # new project wide config file: @@ -251,8 +251,8 @@ proc loadConfigs*(cfg: string; cache: IdentCache) = projectConfig = changeFileExt(gProjectFull, "nimrod.cfg") if fileExists(projectConfig): rawMessage(warnDeprecated, projectConfig) - readConfigFile(projectConfig, cache) + readConfigFile(projectConfig, cache, config) -proc loadConfigs*(cfg: string) = +proc loadConfigs*(cfg: string; config: ConfigRef = nil) = # for backwards compatibility only. - loadConfigs(cfg, newIdentCache()) + loadConfigs(cfg, newIdentCache(), config) diff --git a/compiler/options.nim b/compiler/options.nim index 746ee9044b..2295bbf931 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -102,6 +102,17 @@ type ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod, ideHighlight, ideOutline + ConfigRef* = ref object ## eventually all global configuration should be moved here + cppDefines*: HashSet[string] + headerFile*: string + +proc newConfigRef*(): ConfigRef = + result = ConfigRef(cppDefines: initSet[string](), + headerFile: "") + +proc cppDefine*(c: ConfigRef; define: string) = + c.cppDefines.incl define + var gIdeCmd*: IdeCmd @@ -122,7 +133,7 @@ var outFile*: string = "" docSeeSrcUrl*: string = "" # if empty, no seeSrc will be generated. \ # The string uses the formatting variables `path` and `line`. - headerFile*: string = "" + #headerFile*: string = "" gVerbosity* = 1 # how verbose the compiler is gNumberOfProcessors*: int # number of processors gWholeProject*: bool # for 'doc2': output any dependency diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 75ecf4b024..245680eecd 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -25,7 +25,8 @@ proc listDirs(a: VmArgs, filter: set[PathComponent]) = if kind in filter: result.add path setResult(a, result) -proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext = +proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; + config: ConfigRef = nil): PEvalContext = # For Nimble we need to export 'setupVM'. result = newCtx(module, cache) result.mode = emRepl @@ -133,12 +134,15 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext gModuleOverrides[key] = val cbconf selfExe: setResult(a, os.getAppFilename()) + cbconf cppDefine: + if config != nil: + options.cppDefine(config, a.getString(0)) proc runNimScript*(cache: IdentCache; scriptName: string; - freshDefines=true) = + freshDefines=true; config: ConfigRef=nil) = passes.gIncludeFile = includeModule passes.gImportModule = importModule - let graph = newModuleGraph() + let graph = newModuleGraph(config) if freshDefines: initDefines() defineSymbol("nimscript") @@ -150,7 +154,7 @@ proc runNimScript*(cache: IdentCache; scriptName: string; var m = graph.makeModule(scriptName) incl(m.flags, sfMainModule) - vm.globalCtx = setupVM(m, cache, scriptName) + vm.globalCtx = setupVM(m, cache, scriptName, config) graph.compileSystemModule(cache) discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index f675a9472d..73bb91fef5 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -293,6 +293,11 @@ template task*(name: untyped; description: string; body: untyped): untyped = setCommand "nop" `name Task`() +proc cppDefine*(define: string) = + ## tell Nim that ``define`` is a C preprocessor ``#define`` and so always + ## needs to be mangled. + builtin + when not defined(nimble): # nimble has its own implementation for these things. var diff --git a/tools/nimsuggest/nimsuggest.nim b/tools/nimsuggest/nimsuggest.nim index b5e7b282f5..78fc3fd528 100644 --- a/tools/nimsuggest/nimsuggest.nim +++ b/tools/nimsuggest/nimsuggest.nim @@ -418,7 +418,7 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = options.gProjectName = unixToNativePath(p.key) # if processArgument(pass, p, argsCount): break -proc handleCmdLine(cache: IdentCache) = +proc handleCmdLine(cache: IdentCache; config: ConfigRef) = if paramCount() == 0: stdout.writeline(Usage) else: @@ -444,23 +444,23 @@ proc handleCmdLine(cache: IdentCache) = gPrefixDir = binaryPath.splitPath().head.parentDir() #msgs.writelnHook = proc (line: string) = logStr(line) - loadConfigs(DefaultConfig, cache) # load all config files + loadConfigs(DefaultConfig, cache, config) # load all config files # now process command line arguments again, because some options in the # command line can overwite the config file's settings options.command = "nimsuggest" let scriptFile = gProjectFull.changeFileExt("nims") if fileExists(scriptFile): - runNimScript(cache, scriptFile, freshDefines=false) + runNimScript(cache, scriptFile, freshDefines=false, config) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if scriptFile == gProjectFull: return elif fileExists(gProjectPath / "config.nims"): # directory wide NimScript file - runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false, config) extccomp.initVars() processCmdLine(passCmd2, "") - let graph = newModuleGraph() + let graph = newModuleGraph(config) graph.suggestMode = true mainCommand(graph, cache) @@ -472,4 +472,4 @@ when false: condsyms.initDefines() defineSymbol "nimsuggest" -handleCmdline(newIdentCache()) +handleCmdline(newIdentCache(), newConfigRef()) From 05fe52383235bc0f0435260ea2296182effdfd1b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 1 Feb 2017 23:39:15 +0100 Subject: [PATCH 09/50] updated appveyor.yml --- appveyor.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2ee3ea1166..c750401695 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,21 +34,20 @@ environment: # platform: x86 install: - - MKDIR %CD%\PCRE - - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\pcre + - MKDIR %CD%\DIST + - MKDIR %CD%\DIST\PCRE + - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%" - - 7z x -y "%SQLITE_ARCHIVE%" > nul + - 7z x -y "%SQLITE_ARCHIVE%" -o"%CD%\DIST"> nul - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%" > nul + - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - IF not exist "%FASM_ARCHIVE%" appveyor DownloadFile "%FASM_URL%" -FileName "%FASM_ARCHIVE%" - - 7z x -y "%FASM_ARCHIVE%" -o"%CD%\%FASM_DIR%" > nul - - SET PATH=%CD%\%MINGW_DIR%\bin;%CD%\Nim\bin;%CD%\%FASM_DIR%;%PATH% - - git clone https://github.com/nim-lang/Nim.git %CD%\Nim - - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\Nim\bin\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\Nim\bin\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\Nim\bin\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\Nim\bin\libssl32.dll ) - ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\Nim\bin\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\Nim\bin\libssl32.dll ) - - IF "%PLATFORM%" == "x64" ( copy %CD%\sqlite3.dll %CD%\Nim\bin\sqlite3_64.dll ) ELSE ( copy %CD%\sqlite3.dll %CD%\Nim\bin\sqlite3_32.dll ) - - IF "%PLATFORM%" == "x64" ( copy %CD%\pcre\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\Nim\bin\pcre64.dll ) ELSE ( copy %CD%\pcre\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\Nim\bin\pcre32.dll ) - - cd %CD%\Nim + - 7z x -y "%FASM_ARCHIVE%" -o"%CD%\DIST\%FASM_DIR%" > nul + - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%CD%\DIST\%FASM_DIR%;%PATH% + - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll ) + ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll ) + - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_64.dll ) ELSE ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_32.dll ) + - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll ) - git clone --depth 1 https://github.com/nim-lang/csources - cd csources - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) From d9cf9b079ec7f60dc246c93a73a3aa500ba03d42 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 1 Feb 2017 23:39:33 +0100 Subject: [PATCH 10/50] tiny progress on the name mangling --- compiler/ccgtypes.nim | 9 ++++++++- compiler/cgendata.nim | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8a11f954fd..ccf4545ac5 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -116,6 +116,13 @@ proc mangleName(m: BModule; s: PSym): Rope = add(result, m.idOrSig(s)) s.loc.r = result +template mangleParamName(m: BModule; s: PSym): Rope = mangleName(m, s) + +when false: + proc mangleName(p: BProc; s: PSym): Rope = + assert s.kind in skLocalVars + if sfGlobal in s.flags: return mangleName(p.module, s) + if isKeyword(s.name): discard const irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation, @@ -393,7 +400,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, var param = t.n.sons[i].sym if isCompileTimeOnly(param.typ): continue if params != nil: add(params, ~", ") - fillLoc(param.loc, locParam, param.typ, mangleName(m, param), + fillLoc(param.loc, locParam, param.typ, mangleParamName(m, param), param.paramStorageLoc) if ccgIntroducedPtr(param): add(params, getTypeDescWeak(m, param.typ, check)) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 8446b9db2e..77d031d71e 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -56,7 +56,7 @@ type BProc* = ref TCProc TBlock*{.final.} = object id*: int # the ID of the label; positive means that it - label*: Rope # generated text for the label + label*: Rope # generated text for the label # nil if label is not used sections*: TCProcSections # the code beloging isLoop*: bool # whether block is a loop From 56aa1ac5bc77b53b79a5b9247e88ffbd68b5cf1c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Feb 2017 10:30:01 +0100 Subject: [PATCH 11/50] new name mangling scheme implemented --- compiler/ccgtypes.nim | 41 +++++++++++++++++++++++++----- compiler/cgen.nim | 13 +++++----- compiler/cgendata.nim | 8 +++--- compiler/pragmas.nim | 9 +++++-- tests/ccgbugs/tmissingvolatile.nim | 2 +- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index ccf4545ac5..8f4db126ef 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -116,13 +116,42 @@ proc mangleName(m: BModule; s: PSym): Rope = add(result, m.idOrSig(s)) s.loc.r = result -template mangleParamName(m: BModule; s: PSym): Rope = mangleName(m, s) +proc mangleParamName(m: BModule; s: PSym): Rope = + ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc. + ## Fortunately C's scoping rules are sane enough so that that doesn't + ## cause any trouble. + result = s.loc.r + if result == nil: + result = s.name.s.mangle.rope + if isKeyword(s.name) or m.g.config.cppDefines.contains(s.name.s): + result.add "0" + s.loc.r = result -when false: - proc mangleName(p: BProc; s: PSym): Rope = - assert s.kind in skLocalVars - if sfGlobal in s.flags: return mangleName(p.module, s) - if isKeyword(s.name): discard +proc mangleLocalName(p: BProc; s: PSym): Rope = + assert s.kind in skLocalVars+{skTemp} + assert sfGlobal notin s.flags + result = s.loc.r + if result == nil: + var key = s.name.s.mangle + shallow(key) + let counter = p.sigConflicts.getOrDefault(key) + result = key.rope + if s.kind == skTemp: + # speed up conflict search for temps (these are quite common): + if counter != 0: result.add "_" & rope(counter+1) + elif counter != 0 or isKeyword(s.name) or p.module.g.config.cppDefines.contains(s.name.s): + result.add "_" & rope(counter+1) + p.sigConflicts.inc(key) + s.loc.r = result + +proc scopeMangledParam(p: BProc; param: PSym) = + ## parameter generation only takes BModule, not a BProc, so we have to + ## remember these parameter names are already in scope to be able to + ## generate unique identifiers reliably (consider that ``var a = a`` is + ## even an idiom in Nim). + var key = param.name.s.mangle + shallow(key) + p.sigConflicts.inc(key) const irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation, diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 94a8567739..538111663d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -14,7 +14,7 @@ import nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, - lowerings, semparallel, tables + lowerings, semparallel, tables, sets import strutils except `%` # collides with ropes.`%` @@ -369,7 +369,7 @@ proc localDebugInfo(p: BProc, s: PSym) = proc localVarDecl(p: BProc; s: PSym): Rope = if s.loc.k == locNone: - fillLoc(s.loc, locLocalVar, s.typ, mangleName(p.module, s), OnStack) + fillLoc(s.loc, locLocalVar, s.typ, mangleLocalName(p, s), OnStack) if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy) result = getTypeDesc(p.module, s.typ) if s.constraint.isNil: @@ -434,6 +434,7 @@ proc assignGlobalVar(p: BProc, s: PSym) = proc assignParam(p: BProc, s: PSym) = assert(s.loc.r != nil) + scopeMangledParam(p, s) localDebugInfo(p, s) proc fillProcLoc(m: BModule; sym: PSym) = @@ -1212,13 +1213,13 @@ proc newModule(g: BModuleList; module: PSym): BModule = if (sfDeadCodeElim in module.flags): internalError("added pending module twice: " & module.filename) -template injectG() {.dirty.} = +template injectG(config) {.dirty.} = if graph.backend == nil: - graph.backend = newModuleList() + graph.backend = newModuleList(config) let g = BModuleList(graph.backend) proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = - injectG() + injectG(graph.config) result = newModule(g, module) if optGenIndex in gGlobalOptions and g.generatedHeader == nil: let f = if graph.config.headerFile.len > 0: graph.config.headerFile else: gProjectFull @@ -1258,7 +1259,7 @@ proc getCFile(m: BModule): string = result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext) proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = - injectG() + injectG(graph.config) assert optSymbolFiles in gGlobalOptions var m = newModule(g, module) readMergeInfo(getCFile(m), m) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 77d031d71e..a91858b86e 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -76,7 +76,7 @@ type # leaving such scopes by raise or by return must # execute any applicable finally blocks finallySafePoints*: seq[Rope] # For correctly cleaning up exceptions when - # using return in finally statements + # using return in finally statements labels*: Natural # for generating unique labels in the C proc blocks*: seq[TBlock] # nested blocks breakIdx*: int # the block that will be exited @@ -92,6 +92,7 @@ type # (yes, C++ is weird like that) gcFrameId*: Natural # for the GC stack marking gcFrameType*: Rope # the struct {} we put the GC markers into + sigConflicts*: CountTable[string] TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] @@ -163,9 +164,10 @@ proc newProc*(prc: PSym, module: BModule): BProc = newSeq(result.blocks, 1) result.nestedTryStmts = @[] result.finallySafePoints = @[] + result.sigConflicts = initCountTable[string]() -proc newModuleList*(): BModuleList = - BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope]()) +proc newModuleList*(config: ConfigRef): BModuleList = + BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope](), config: config) iterator cgenModules*(g: BModuleList): BModule = for i in 0..high(g.modules): diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index e750cc3907..c88451de6c 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -665,9 +665,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wExportc: makeExternExport(sym, getOptionalStr(c, it, "$1"), it.info) incl(sym.flags, sfUsed) # avoid wrong hints - of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1"), it.info) + of wImportc: + let name = getOptionalStr(c, it, "$1") + cppDefine(c.graph.config, name) + makeExternImport(sym, name, it.info) of wImportCompilerProc: - processImportCompilerProc(sym, getOptionalStr(c, it, "$1"), it.info) + let name = getOptionalStr(c, it, "$1") + cppDefine(c.graph.config, name) + processImportCompilerProc(sym, name, it.info) of wExtern: setExternName(sym, expectStrLit(c, it), it.info) of wImmediate: if sym.kind in {skTemplate, skMacro}: diff --git a/tests/ccgbugs/tmissingvolatile.nim b/tests/ccgbugs/tmissingvolatile.nim index d61778ed40..4d25e5c222 100644 --- a/tests/ccgbugs/tmissingvolatile.nim +++ b/tests/ccgbugs/tmissingvolatile.nim @@ -1,7 +1,7 @@ discard """ output: "1" cmd: r"nim c --hints:on $options -d:release $file" - ccodecheck: "'NI volatile state0;'" + ccodecheck: "'NI volatile state;'" """ # bug #1539 From a86966eefd42b8d09af5a8f014d81ef844b55e4c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Feb 2017 11:01:26 +0100 Subject: [PATCH 12/50] attempt to make nimble work again --- compiler/modulegraphs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 0ae7076d96..87a35b2900 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -50,7 +50,10 @@ proc newModuleGraph*(config: ConfigRef = nil): ModuleGraph = result.modules = @[] result.importStack = @[] result.inclToMod = initTable[int32, int32]() - result.config = config + if config.isNil: + result.config = newConfigRef() + else: + result.config = config proc resetAllModules*(g: ModuleGraph) = initStrTable(packageSyms) From 137b5f43022c3c5e22a7f28c8012f1d0ea6007c6 Mon Sep 17 00:00:00 2001 From: Parashurama Date: Thu, 2 Feb 2017 11:33:54 +0100 Subject: [PATCH 13/50] fixes #4992 --- compiler/vm.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 7ce96f7dfc..dd91b80e84 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -932,7 +932,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = c.module var macroCall = newNodeI(nkCall, c.debug[pc]) macroCall.add(newSymNode(prc)) - for i in 1 .. rc-1: macroCall.add(regs[rb+i].regToNode) + for i in 1 .. rc-1: + let node = regs[rb+i].regToNode + node.info = c.debug[pc] + macroCall.add(node) let a = evalTemplate(macroCall, prc, genSymOwner) a.recSetFlagIsRef ensureKind(rkNode) From 899d84e06dc83d494ed29af039f8e66e544677f7 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 2 Feb 2017 14:10:32 +0100 Subject: [PATCH 14/50] minor website update --- web/download.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/download.rst b/web/download.rst index 34f5725e49..d2c6a0fc23 100644 --- a/web/download.rst +++ b/web/download.rst @@ -16,8 +16,14 @@ We now encourage you to install via the provided zipfiles: * | 64 bit: `nim-0.16.0_x64.zip `_ | SHA-256 e667cdad1ae8e9429147aea5031fa8a80c4ccef6d274cec0e9480252d9c3168c -Unzip these where you want and optionally run ``finish.exe`` to -detect your MingW environment. +Unzip these where you want and **optionally** run ``finish.exe`` to +detect your MingW environment. (Though that's not reliable yet.) + +You can find the required DLLs here, if you lack them for some reason: + +* | 32 and 64 bit: `DLLs.zip `_ + | SHA-256 198112d3d6dc74d7964ba452158d44bfa57adef4dc47be8c39903f2a24e4a555 + Exes %%%% From fab69661ad7132fd138c26da9f8cb0b041125965 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Feb 2017 16:27:48 +0100 Subject: [PATCH 15/50] new name mangling rules for easier debugging --- compiler/ccgexprs.nim | 2 +- compiler/ccgstmts.nim | 34 ++++++++++++++-------------- compiler/ccgthreadvars.nim | 4 ++-- compiler/ccgtrav.nim | 2 +- compiler/ccgtypes.nim | 4 +--- compiler/ccgutils.nim | 46 ++++++++++++++++++++++++++------------ compiler/cgen.nim | 27 +++++++++++----------- compiler/pragmas.nim | 1 + lib/nimbase.h | 29 +++++++++++++++++------- 9 files changed, 90 insertions(+), 59 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index eabcdd66ad..a3144bb3c4 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2001,7 +2001,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfThread in sym.flags: accessThreadLocalVar(p, sym) if emulatedThreadVars(): - putIntoDest(p, d, sym.loc.t, "NimTV->" & sym.loc.r) + putIntoDest(p, d, sym.loc.t, "NimTV_->" & sym.loc.r) else: putLocIntoDest(p, d, sym.loc) else: diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index b3d21c35ed..09b455ec0c 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -102,7 +102,7 @@ proc assignLabel(b: var TBlock): Rope {.inline.} = proc blockBody(b: var TBlock): Rope = result = b.sections[cpsLocals] if b.frameLen > 0: - result.addf("FR.len+=$1;$n", [b.frameLen.rope]) + result.addf("FR_.len+=$1;$n", [b.frameLen.rope]) result.add(b.sections[cpsInit]) result.add(b.sections[cpsStmts]) @@ -123,7 +123,7 @@ proc endBlock(p: BProc) = ~"}$n" let frameLen = p.blocks[topBlock].frameLen if frameLen > 0: - blockEnd.addf("FR.len-=$1;$n", [frameLen.rope]) + blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope]) endBlock(p, blockEnd) proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} = @@ -156,7 +156,7 @@ proc genGotoState(p: BProc, n: PNode) = initLocExpr(p, n.sons[0], a) lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)]) p.beforeRetNeeded = true - lineF(p, cpsStmts, "case -1: goto BeforeRet;$n", []) + lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", []) for i in 0 .. lastOrd(n.sons[0].typ): lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [rope(i)]) lineF(p, cpsStmts, "}$n", []) @@ -373,7 +373,7 @@ proc genReturnStmt(p: BProc, t: PNode) = # consume it before we return. var safePoint = p.finallySafePoints[p.finallySafePoints.len-1] linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint) - lineF(p, cpsStmts, "goto BeforeRet;$n", []) + lineF(p, cpsStmts, "goto BeforeRet_;$n", []) proc genGotoForCase(p: BProc; caseStmt: PNode) = for i in 1 .. ': special "gt" + of '~': special "tilde" + of ':': special "colon" + of '.': special "dot" + of '@': special "at" + of '|': special "bar" else: - add(result, "HEX" & toHex(ord(c), 2)) + add(result, "X" & toHex(ord(c), 2)) + requiresUnderscore = true + if requiresUnderscore: + result.add "_" proc makeLLVMString*(s: string): Rope = const MaxLineLength = 64 diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 538111663d..ae735d30b1 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -216,7 +216,7 @@ proc genLineDir(p: BProc, t: PNode) = {optLineTrace, optStackTrace}) and (p.prc == nil or sfPure notin p.prc.flags) and tt.info.fileIndex >= 0: if freshLineInfo(p, tt.info): - linefmt(p, cpsStmts, "nimln($1, $2);$n", + linefmt(p, cpsStmts, "nimln_($1, $2);$n", line.rope, tt.info.quotedFilename) proc postStmtActions(p: BProc) {.inline.} = @@ -338,7 +338,7 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = inc(p.labels) - result.r = "LOC" & rope(p.labels) + result.r = "T" & rope(p.labels) & "_" linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r) result.k = locTemp result.t = t @@ -347,12 +347,12 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = constructLoc(p, result, not needsInit) proc initGCFrame(p: BProc): Rope = - if p.gcFrameId > 0: result = "struct {$1} GCFRAME;$n" % [p.gcFrameType] + if p.gcFrameId > 0: result = "struct {$1} GCFRAME_;$n" % [p.gcFrameType] proc deinitGCFrame(p: BProc): Rope = if p.gcFrameId > 0: result = ropecg(p.module, - "if (((NU)&GCFRAME) < 4096) #nimGCFrame(&GCFRAME);$n") + "if (((NU)&GCFRAME_) < 4096) #nimGCFrame(&GCFRAME_);$n") proc localDebugInfo(p: BProc, s: PSym) = if {optStackTrace, optEndb} * p.options != {optStackTrace, optEndb}: return @@ -361,7 +361,7 @@ proc localDebugInfo(p: BProc, s: PSym) = var a = "&" & s.loc.r if s.kind == skParam and ccgIntroducedPtr(s): a = s.loc.r lineF(p, cpsInit, - "FR.s[$1].address = (void*)$3; FR.s[$1].typ = $4; FR.s[$1].name = $2;$n", + "FR_.s[$1].address = (void*)$3; FR_.s[$1].typ = $4; FR_.s[$1].name = $2;$n", [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a, genTypeInfo(p.module, s.loc.t)]) inc(p.maxFrameLen) @@ -443,7 +443,7 @@ proc fillProcLoc(m: BModule; sym: PSym) = proc getLabel(p: BProc): TLabel = inc(p.labels) - result = "LA" & rope(p.labels) + result = "LA" & rope(p.labels) & "_" proc fixLabel(p: BProc, labl: TLabel) = lineF(p, cpsStmts, "$1: ;$n", [labl]) @@ -521,7 +521,7 @@ proc mangleDynLibProc(sym: PSym): Rope = # NOTE: sym.loc.r is the external name! result = rope(sym.name.s) else: - result = "Dl_$1" % [rope(sym.id)] + result = "Dl_$1_" % [rope(sym.id)] proc symInDynamicLib(m: BModule, sym: PSym) = var lib = sym.annex @@ -609,11 +609,11 @@ proc initFrame(p: BProc, procname, filename: Rope): Rope = discard cgsym(p.module, "nimFrame") if p.maxFrameLen > 0: discard cgsym(p.module, "VarSlot") - result = rfmt(nil, "\tnimfrs($1, $2, $3, $4)$N", + result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4)$N", procname, filename, p.maxFrameLen.rope, p.blocks[0].frameLen.rope) else: - result = rfmt(nil, "\tnimfr($1, $2)$N", procname, filename) + result = rfmt(nil, "\tnimfr_($1, $2)$N", procname, filename) proc deinitFrame(p: BProc): Rope = result = rfmt(p.module, "\t#popFrame();$n") @@ -708,7 +708,7 @@ proc genProcAux(m: BModule, prc: PSym) = if p.beforeRetNeeded: add(generatedProc, "{") add(generatedProc, p.s(cpsInit)) add(generatedProc, p.s(cpsStmts)) - if p.beforeRetNeeded: add(generatedProc, ~"\t}BeforeRet: ;$n") + if p.beforeRetNeeded: add(generatedProc, ~"\t}BeforeRet_: ;$n") add(generatedProc, deinitGCFrame(p)) if optStackTrace in prc.options: add(generatedProc, deinitFrame(p)) add(generatedProc, returnStmt) @@ -847,7 +847,8 @@ proc genVarPrototype(m: BModule, sym: PSym) = genVarPrototypeAux(m, sym) proc addIntTypes(result: var Rope) {.inline.} = - addf(result, "#define NIM_INTBITS $1" & tnl, [ + addf(result, "#define NIM_NEW_MANGLING_RULES" & tnl & + "#define NIM_INTBITS $1" & tnl, [ platform.CPU[targetCPU].intSize.rope]) proc getCopyright(cfile: Cfile): Rope = @@ -1059,7 +1060,7 @@ proc genInitCode(m: BModule) = var procname = makeCString(m.module.name.s) add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename)) else: - add(prc, ~"\tTFrame FR; FR.len = 0;$N") + add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") add(prc, genSectionStart(cpsInit)) add(prc, m.preInitProc.s(cpsInit)) @@ -1124,7 +1125,7 @@ proc initProcOptions(m: BModule): TOptions = proc rawNewModule(g: BModuleList; module: PSym, filename: string): BModule = new(result) - result.tmpBase = rope("T" & $hashOwner(module) & "_") + result.tmpBase = rope("TM" & $hashOwner(module) & "_") initLinkedList(result.headerFiles) result.declaredThings = initIntSet() result.declaredProtos = initIntSet() diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index c88451de6c..04dbd36128 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -763,6 +763,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, processDynLib(c, it, sym) of wCompilerproc: noVal(it) # compilerproc may not get a string! + cppDefine(c.graph.config, sym.name.s) if sfFromGeneric notin sym.flags: markCompilerProc(sym) of wProcVar: noVal(it) diff --git a/lib/nimbase.h b/lib/nimbase.h index 818bff462b..a5d2616e71 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -402,16 +402,29 @@ struct TFrame { NI16 calldepth; }; -#define nimfr(proc, file) \ - TFrame FR; \ - FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR); +#ifdef NIM_NEW_MANGLING_RULES + #define nimfr_(proc, file) \ + TFrame FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; nimFrame(&FR_); -#define nimfrs(proc, file, slots, length) \ - struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \ - FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR); + #define nimfrs_(proc, file, slots, length) \ + struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; nimFrame((TFrame*)&FR_); -#define nimln(n, file) \ - FR.line = n; FR.filename = file; + #define nimln_(n, file) \ + FR_.line = n; FR_.filename = file; +#else + #define nimfr(proc, file) \ + TFrame FR; \ + FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR); + + #define nimfrs(proc, file, slots, length) \ + struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \ + FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR); + + #define nimln(n, file) \ + FR.line = n; FR.filename = file; +#endif #define NIM_POSIX_INIT __attribute__((constructor)) From e236039d109a49531098679156c5ed93a8c533b0 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 2 Feb 2017 21:12:36 +0100 Subject: [PATCH 16/50] make tests on Windows green under --pedantic --- lib/pure/terminal.nim | 2 +- tests/misc/tfsmonitor.nim | 7 +++---- tests/testament/categories.nim | 4 ++-- tests/testament/tester.nim | 19 +++++++++---------- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 7a8113b2af..31278eabf4 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -630,7 +630,7 @@ proc getch*(): char = when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) var keyEvent = KEY_EVENT_RECORD() - var numRead: cint + var numRead: cint while true: # Block until character is entered doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0) diff --git a/tests/misc/tfsmonitor.nim b/tests/misc/tfsmonitor.nim index 27e1a2e323..35f93fb47c 100644 --- a/tests/misc/tfsmonitor.nim +++ b/tests/misc/tfsmonitor.nim @@ -1,6 +1,6 @@ -# -# fsmonitor test -# +discard """ + disabled: windows +""" import unittest import fsmonitor @@ -9,4 +9,3 @@ suite "fsmonitor": test "should not raise OSError, bug# 3611": let m = newMonitor() m.add("foo", {MonitorCloseWrite, MonitorCloseNoWrite}) - diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index c788395f84..2dc8e33183 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -284,9 +284,9 @@ proc compileExample(r: var TResults, pattern, options: string, cat: Category) = testNoSpec r, makeTest(test, options, cat) proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = - var disabledSet = disabledFiles.toSet() for test in os.walkFiles(pattern): - if test notin disabledSet: + let name = extractFilename(test) + if name notin disabledFiles: let contents = readFile(test).string if contents.contains("when isMainModule"): testSpec r, makeTest(test, options, cat, actionRunNoSpec) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 2734742f49..7efd405bb8 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -108,12 +108,6 @@ proc callCompiler(cmdTemplate, filename, options: string, elif suc =~ pegSuccess: result.err = reSuccess - if result.err == reNimcCrash and - ("Your platform is not supported" in result.msg or - "cannot open 'sdl'" in result.msg or - "cannot open 'opengl'" in result.msg): - result.err = reIgnored - proc callCCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], @@ -393,9 +387,14 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile, result = TTest(cat: cat, name: test, options: options, target: target, action: action, startTime: epochTime()) -const - # array of modules disabled from compilation test of stdlib. - disabledFiles = ["-"] +when defined(windows): + const + # array of modules disabled from compilation test of stdlib. + disabledFiles = ["coro.nim", "fsmonitor.nim"] +else: + const + # array of modules disabled from compilation test of stdlib. + disabledFiles = ["-"] include categories @@ -460,7 +459,7 @@ proc main() = backend.close() if optPedantic: var failed = r.total - r.passed - r.skipped - if failed > 0 : quit(QuitFailure) + if failed > 0: quit(QuitFailure) if paramCount() == 0: quit Usage From ffd929af41d6312c47e03eaba5842d37f7046ca4 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 2 Feb 2017 22:22:14 +0100 Subject: [PATCH 17/50] minor appveyor update --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index c750401695..87640b1923 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,6 +69,6 @@ build_script: test_script: - tests\testament\tester --pedantic all - koch csource - - koch xz + - koch zip deploy: off From 2c6c865b353d0e15783a95886d672288ca7da46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Thu, 2 Feb 2017 23:06:13 +0100 Subject: [PATCH 18/50] reenabled clear test, made clear working (#5323) --- lib/pure/collections/tables.nim | 7 ++++-- tests/collections/ttables.nim | 38 ++++++++++++++++----------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 57e98bf5cd..00a81b8d5e 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -814,11 +814,14 @@ proc len*[A](t: CountTable[A]): int = ## returns the number of keys in `t`. result = t.counter -proc clear*[A](t: var CountTable[A] | CountTableRef[A]) = +proc clear*[A](t: CountTableRef[A]) = ## Resets the table so that it is empty. clearImpl() - t.counter = 0 +proc clear*[A](t: var CountTable[A]) = + ## Resets the table so that it is empty. + clearImpl() + iterator pairs*[A](t: CountTable[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index ef5ed92f57..0e06bc26f8 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -190,28 +190,26 @@ block zeroHashKeysTest: doZeroHashValueTest(toOrderedTable[string,string]({"egg": "sausage"}), "", "spam") -# Until #4448 is fixed, these tests will fail -when false: - block clearTableTest: - var t = data.toTable - assert t.len() != 0 - t.clear() - assert t.len() == 0 +block clearTableTest: + var t = data.toTable + assert t.len() != 0 + t.clear() + assert t.len() == 0 - block clearOrderedTableTest: - var t = data.toOrderedTable - assert t.len() != 0 - t.clear() - assert t.len() == 0 +block clearOrderedTableTest: + var t = data.toOrderedTable + assert t.len() != 0 + t.clear() + assert t.len() == 0 - block clearCountTableTest: - var t = initCountTable[string]() - t.inc("90", 3) - t.inc("12", 2) - t.inc("34", 1) - assert t.len() != 0 - t.clear() - assert t.len() == 0 +block clearCountTableTest: + var t = initCountTable[string]() + t.inc("90", 3) + t.inc("12", 2) + t.inc("34", 1) + assert t.len() != 0 + t.clear() + assert t.len() == 0 proc orderedTableSortTest() = var t = initOrderedTable[string, int](2) From 04c4d3d77f86086efa2447c0c9b47ba84342d245 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Feb 2017 09:48:52 +0100 Subject: [PATCH 19/50] critical realloc bugfix; refs #4818 --- lib/system/alloc.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 6e7801be5f..065b134609 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -295,10 +295,11 @@ proc writeFreeList(a: MemRegion) = proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: - if a.currMem < 64 * 1024: + let usedMem = a.currMem # - a.freeMem + if usedMem < 64 * 1024: a.nextChunkSize = PageSize*4 else: - a.nextChunkSize = min(roundup(a.currMem shr 2, PageSize), a.nextChunkSize * 2) + a.nextChunkSize = min(roundup(usedMem shr 2, PageSize), a.nextChunkSize * 2) var size = size if size > a.nextChunkSize: @@ -708,7 +709,7 @@ proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = if newsize > 0: result = alloc0(allocator, newsize) if p != nil: - copyMem(result, p, ptrSize(p)) + copyMem(result, p, min(ptrSize(p), newsize)) dealloc(allocator, p) elif p != nil: dealloc(allocator, p) From 848676cec6ba75e9bd0f8f590c6e47f6be7e696e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Feb 2017 09:49:36 +0100 Subject: [PATCH 20/50] name mangling bugfixes; ndi file generation for debugger support --- compiler/ccgstmts.nim | 2 +- compiler/ccgtypes.nim | 73 +++++++++---------------------------------- compiler/ccgutils.nim | 12 ++++--- compiler/cgen.nim | 6 +++- compiler/cgendata.nim | 3 +- compiler/ndi.nim | 40 ++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 65 deletions(-) create mode 100644 compiler/ndi.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 09b455ec0c..45d675f64a 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -64,7 +64,7 @@ proc genVarTuple(p: BProc, n: PNode) = field.r = "$1.Field$2" % [rdLoc(tup), rope(i)] else: if t.n.sons[i].kind != nkSym: internalError(n.info, "genVarTuple") - field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(t.n.sons[i].sym, t)] + field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym, t)] putLocIntoDest(p, v.loc, field) proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index ba505facb5..e30fe5598b 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -22,12 +22,10 @@ proc isKeyword(w: PIdent): bool = ord(wInline): return true else: return false -proc mangleField(name: PIdent): string = +proc mangleField(m: BModule; name: PIdent): string = result = mangle(name.s) - if isKeyword(name): - result[0] = result[0].toUpperAscii - # Mangling makes everything lowercase, - # but some identifiers are C keywords + if isKeyword(name) or m.g.config.cppDefines.contains(result): + result.add "_0" when false: proc hashOwner(s: PSym): SigHash = @@ -67,54 +65,10 @@ proc idOrSig(m: BModule; s: PSym): Rope = proc mangleName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: - let keepOrigName = s.kind in skLocalVars - {skForVar} and - {sfFromGeneric, sfGlobal, sfShadowed, sfGenSym} * s.flags == {} and - not isKeyword(s.name) - # Even with all these inefficient checks, the bootstrap - # time is actually improved. This is probably because so many - # rope concatenations are now eliminated. - # - # sfFromGeneric is needed in order to avoid multiple - # definitions of certain variables generated in transf with - # names such as: - # `r`, `res` - # I need to study where these come from. - # - # about sfShadowed: - # consider the following Nim code: - # var x = 10 - # block: - # var x = something(x) - # The generated C code will be: - # NI x; - # x = 10; - # { - # NI x; - # x = something(x); // Oops, x is already shadowed here - # } - # Right now, we work-around by not keeping the original name - # of the shadowed variable, but we can do better - we can - # create an alternative reference to it in the outer scope and - # use that in the inner scope. - # - # about isCKeyword: - # Nim variable names can be C keywords. - # We need to avoid such names in the generated code. - # - # about sfGlobal: - # This seems to be harder - a top level extern variable from - # another modules can have the same name as a local one. - # Maybe we should just implement sfShadowed for them too. - # - # about skForVar: - # These are not properly scoped now - we need to add blocks - # around for loops in transf result = s.name.s.mangle.rope - if keepOrigName: - result.add "0" - else: - add(result, m.idOrSig(s)) + add(result, m.idOrSig(s)) s.loc.r = result + writeMangledName(m.ndi, s) proc mangleParamName(m: BModule; s: PSym): Rope = ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc. @@ -122,10 +76,12 @@ proc mangleParamName(m: BModule; s: PSym): Rope = ## cause any trouble. result = s.loc.r if result == nil: - result = s.name.s.mangle.rope - if isKeyword(s.name) or m.g.config.cppDefines.contains(s.name.s): - result.add "0" + var res = s.name.s.mangle + if isKeyword(s.name) or m.g.config.cppDefines.contains(res): + res.add "_0" + result = res.rope s.loc.r = result + writeMangledName(m.ndi, s) proc mangleLocalName(p: BProc; s: PSym): Rope = assert s.kind in skLocalVars+{skTemp} @@ -139,10 +95,11 @@ proc mangleLocalName(p: BProc; s: PSym): Rope = if s.kind == skTemp: # speed up conflict search for temps (these are quite common): if counter != 0: result.add "_" & rope(counter+1) - elif counter != 0 or isKeyword(s.name) or p.module.g.config.cppDefines.contains(s.name.s): + elif counter != 0 or isKeyword(s.name) or p.module.g.config.cppDefines.contains(key): result.add "_" & rope(counter+1) p.sigConflicts.inc(key) s.loc.r = result + if s.kind != skTemp: writeMangledName(p.module.ndi, s) proc scopeMangledParam(p: BProc; param: PSym) = ## parameter generation only takes BModule, not a BProc, so we have to @@ -472,12 +429,12 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, else: add(params, ")") params = "(" & params -proc mangleRecFieldName(field: PSym, rectype: PType): Rope = +proc mangleRecFieldName(m: BModule; field: PSym, rectype: PType): Rope = if (rectype.sym != nil) and ({sfImportc, sfExportc} * rectype.sym.flags != {}): result = field.loc.r else: - result = rope(mangleField(field.name)) + result = rope(mangleField(m, field.name)) if result == nil: internalError(field.info, "mangleRecFieldName") proc genRecordFieldsAux(m: BModule, n: PNode, @@ -516,7 +473,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode, let field = n.sym if field.typ.kind == tyVoid: return #assert(field.ast == nil) - let sname = mangleRecFieldName(field, rectype) + let sname = mangleRecFieldName(m, field, rectype) let ae = if accessExpr != nil: "$1.$2" % [accessExpr, sname] else: sname fillLoc(field.loc, locField, field.typ, ae, OnUnknown) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 0985759b94..d42f0438ff 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -164,9 +164,6 @@ proc makeSingleLineCString*(s: string): string = result.add('\"') proc mangle*(name: string): string = - ## Lowercases the given name and manges any non-alphanumeric characters - ## so they are represented as `HEX____`. If the name starts with a number, - ## `N` is prepended result = newStringOfCap(name.len) var start = 0 if name[0] in Digits: @@ -179,8 +176,15 @@ proc mangle*(name: string): string = for i in start..(name.len-1): let c = name[i] case c - of 'a'..'z', '0'..'9', 'A'..'Z', '_': + of 'a'..'z', '0'..'9', 'A'..'Z': add(result, c) + of '_': + # we generate names like 'foo_9' for scope disambiguations and so + # disallow this here: + if i < name.len-1 and name[i] in Digits: + discard + else: + add(result, c) of '$': special "dollar" of '%': special "percent" of '&': special "amp" diff --git a/compiler/cgen.nim b/compiler/cgen.nim index ae735d30b1..62ed9ad6e6 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -14,7 +14,7 @@ import nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, - lowerings, semparallel, tables, sets + lowerings, semparallel, tables, sets, ndi import strutils except `%` # collides with ropes.`%` @@ -1152,6 +1152,9 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: string): BModule = incl result.flags, preventStackTrace excl(result.preInitProc.options, optStackTrace) excl(result.postInitProc.options, optStackTrace) + let ndiName = if optCDebug in gGlobalOptions: changeFileExt(completeCFilePath(filename), "ndi") + else: "" + open(result.ndi, ndiName) proc nullify[T](arr: var T) = for i in low(arr)..high(arr): @@ -1343,6 +1346,7 @@ proc writeModule(m: BModule, pending: bool) = var cf = Cfile(cname: cfile, obj: completeCFilePath(toObjFile(cfile)), flags: {}) if not existsFile(cf.obj): cf.flags = {CfileFlag.Cached} addFileToCompile(cf) + close(m.ndi) proc updateCachedModule(m: BModule) = let cfile = getCFile(m) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index a91858b86e..565399ead9 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -11,7 +11,7 @@ import ast, astalgo, ropes, passes, options, intsets, lists, platform, sighashes, - tables + tables, ndi from msgs import TLineInfo @@ -146,6 +146,7 @@ type injectStmt*: Rope sigConflicts*: CountTable[SigHash] g*: BModuleList + ndi*: NdiFile proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} = # section in the current block diff --git a/compiler/ndi.nim b/compiler/ndi.nim new file mode 100644 index 0000000000..a7ca02193c --- /dev/null +++ b/compiler/ndi.nim @@ -0,0 +1,40 @@ +# +# +# The Nim Compiler +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the generation of ``.ndi`` files for better debugging +## support of Nim code. "ndi" stands for "Nim debug info". + +import ast, msgs, ropes + +type + NdiFile* = object + enabled: bool + f: File + buf: string + +proc doWrite(f: var NdiFile; s: PSym) = + f.buf.setLen 0 + f.buf.add s.info.line.int + f.buf.add "\t" + f.buf.add s.info.col.int + f.f.write(s.name.s, "\t") + f.f.writeRope(s.loc.r) + f.f.writeLine("\t", s.info.toFullPath, "\t", f.buf) + +template writeMangledName*(f: NdiFile; s: PSym) = + if f.enabled: doWrite(f, s) + +proc open*(f: var NdiFile; filename: string) = + f.enabled = filename.len > 0 + if f.enabled: + f.f = open(filename, fmWrite, 8000) + f.buf = newStringOfCap(20) + +proc close*(f: var NdiFile) = + if f.enabled: close(f.f) From 4ac6a2603167a3199dfce60f07dba3a5188ea7a8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Feb 2017 10:29:08 +0100 Subject: [PATCH 21/50] testament: be verbose when --pedantic fails --- tests/testament/tester.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 7efd405bb8..d4a161dabd 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -459,7 +459,9 @@ proc main() = backend.close() if optPedantic: var failed = r.total - r.passed - r.skipped - if failed > 0: quit(QuitFailure) + if failed > 0: + echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ", r.skipped + quit(QuitFailure) if paramCount() == 0: quit Usage From 26fb6cb07357b0565f2d53c387c50f9727b9f579 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Feb 2017 17:35:58 +0100 Subject: [PATCH 22/50] fixes #5327 --- compiler/vm.nim | 3 ++- tests/vm/tableinstatic.nim | 38 ++++++++++++++++++++++++++++++++++ tests/vm/tcompiletimetable.nim | 10 ++++----- 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/vm/tableinstatic.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index dd91b80e84..ea82a3155d 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -559,7 +559,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind == nkRefTy: regs[ra].node = regs[rb].node.sons[0] else: - stackTrace(c, tos, pc, errGenerated, "limited VM support for pointers") + ensureKind(rkNode) + regs[ra].node = regs[rb].node else: stackTrace(c, tos, pc, errNilAccess) of opcWrDeref: diff --git a/tests/vm/tableinstatic.nim b/tests/vm/tableinstatic.nim new file mode 100644 index 0000000000..54e7c11f0e --- /dev/null +++ b/tests/vm/tableinstatic.nim @@ -0,0 +1,38 @@ +discard """ + nimout: '''0 +0 +0 +{hallo: 123, welt: 456}''' +""" + +import tables + +# bug #5327 + +type + MyType* = object + counter: int + +proc foo(t: var MyType) = + echo t.counter + +proc bar(t: MyType) = + echo t.counter + +static: + var myValue: MyType + myValue.foo # works nicely + + var refValue: ref MyType + refValue.new + + refValue[].foo # fails to compile + refValue[].bar # works again nicely + +static: + var otherTable = newTable[string, string]() + + otherTable["hallo"] = "123" + otherTable["welt"] = "456" + + echo otherTable diff --git a/tests/vm/tcompiletimetable.nim b/tests/vm/tcompiletimetable.nim index df6ead56fc..e78c06536c 100644 --- a/tests/vm/tcompiletimetable.nim +++ b/tests/vm/tcompiletimetable.nim @@ -1,5 +1,5 @@ discard """ - msg: '''2 + nimout: '''2 3 4:2 Got Hi @@ -13,7 +13,7 @@ import macros, tables, strtabs var ZOOT{.compileTime.} = initTable[int, int](2) var iii {.compiletime.} = 1 -macro zoo:stmt= +macro zoo: untyped = ZOOT[iii] = iii*2 inc iii echo iii @@ -22,7 +22,7 @@ zoo zoo -macro tupleUnpack: stmt = +macro tupleUnpack: untyped = var (y,z) = (4, 2) echo y, ":", z @@ -32,14 +32,14 @@ tupleUnpack var x {.compileTime.}: StringTableRef -macro addStuff(stuff, body: expr): stmt {.immediate.} = +macro addStuff(stuff, body: untyped): untyped = result = newNimNode(nnkStmtList) if x.isNil: x = newStringTable(modeStyleInsensitive) x[$stuff] = "" -macro dump(): stmt = +macro dump(): untyped = result = newNimNode(nnkStmtList) for y in x.keys: echo "Got ", y From 76a28d8b83190ee43826a82b0314800715e1bdbf Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Feb 2017 17:36:39 +0100 Subject: [PATCH 23/50] nimsuggest: bugfix: also output documentation comments --- compiler/docgen.nim | 38 ++++++++++++++-------------- compiler/suggest.nim | 14 ++++++++++ tools/nimsuggest/tests/tdef1.nim | 6 ++--- tools/nimsuggest/tests/tstrutils.nim | 2 +- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2115449249..26dd889ce9 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -209,26 +209,26 @@ proc getPlainDocstring(n: PNode): string = result = getPlainDocstring(n.sons[i]) if result.len > 0: return +when false: + proc findDocComment(n: PNode): PNode = + if n == nil: return nil + if not isNil(n.comment) and startsWith(n.comment, "##"): return n + for i in countup(0, safeLen(n)-1): + result = findDocComment(n.sons[i]) + if result != nil: return -proc findDocComment(n: PNode): PNode = - if n == nil: return nil - if not isNil(n.comment) and startsWith(n.comment, "##"): return n - for i in countup(0, safeLen(n)-1): - result = findDocComment(n.sons[i]) - if result != nil: return - -proc extractDocComment*(s: PSym, d: PDoc = nil): string = - let n = findDocComment(s.ast) - result = "" - if not n.isNil: - if not d.isNil: - var dummyHasToc: bool - renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), - toLinenumber(n.info), toColumn(n.info), - dummyHasToc, d.options + {roSkipPounds}), - result) - else: - result = n.comment.substr(2).replace("\n##", "\n").strip + proc extractDocComment*(s: PSym, d: PDoc = nil): string = + let n = findDocComment(s.ast) + result = "" + if not n.isNil: + if not d.isNil: + var dummyHasToc: bool + renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), + toLinenumber(n.info), toColumn(n.info), + dummyHasToc, d.options + {roSkipPounds}), + result) + else: + result = n.comment.substr(2).replace("\n##", "\n").strip proc isVisible(n: PNode): bool = result = false diff --git a/compiler/suggest.nim b/compiler/suggest.nim index f3c03d6800..66876b9b5a 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -41,6 +41,20 @@ var template origModuleName(m: PSym): string = m.name.s +proc findDocComment(n: PNode): PNode = + if n == nil: return nil + if not isNil(n.comment): return n + for i in countup(0, safeLen(n)-1): + result = findDocComment(n.sons[i]) + if result != nil: return + +proc extractDocComment(s: PSym): string = + let n = findDocComment(s.ast) + if not n.isNil: + result = n.comment.replace("\n##", "\n").strip + else: + result = "" + proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo; quality: range[0..100]): Suggest = result.section = parseIdeCmd(section) diff --git a/tools/nimsuggest/tests/tdef1.nim b/tools/nimsuggest/tests/tdef1.nim index 960ffad1c6..2cd040ea14 100644 --- a/tools/nimsuggest/tests/tdef1.nim +++ b/tools/nimsuggest/tests/tdef1.nim @@ -1,12 +1,12 @@ discard """ $nimsuggest --tester $file >def $1 -def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100 >def $1 -def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100 """ -proc hello() string = +proc hello(): string = ## Return hello "Hello" diff --git a/tools/nimsuggest/tests/tstrutils.nim b/tools/nimsuggest/tests/tstrutils.nim index f5cda95053..34da8cb53d 100644 --- a/tools/nimsuggest/tests/tstrutils.nim +++ b/tools/nimsuggest/tests/tstrutils.nim @@ -1,7 +1,7 @@ discard """ $nimsuggest --tester lib/pure/strutils.nim >def lib/pure/strutils.nim:2300:6 -def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"";;100 +def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"same as `assert` but is always turned on and not affected by the\x0A``--assertions`` command line switch.";;100 """ # Line 2300 in strutils.nim is doAssert and this is unlikely to change From bf165cb43a194a301abbc04a6ff1f2e44d9f07e7 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 4 Feb 2017 11:01:23 +0000 Subject: [PATCH 24/50] Use the correct parallelBuild value. fixes #5328 (#5329) Also check for incorrect parallelBuild values --- tools/nimweb.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 2c905057e9..29464f8e33 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -272,10 +272,13 @@ proc sexec(cmds: openarray[string]) = proc mexec(cmds: openarray[string], processors: int) = ## Multiprocessor version of exec - if processors < 2: + doAssert processors > 0, "nimweb needs at least one processor" + if processors == 1: sexec(cmds) return - if execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}) != 0: + let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}, + n = processors) + if r != 0: echo "external program failed, retrying serial work queue for logs!" sexec(cmds) From 95d8558f0cdbb3324985210e51fb0c8bb66b99a9 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 3 Feb 2017 17:24:25 +0200 Subject: [PATCH 25/50] Fix #5331 and #5332. --- lib/upcoming/asyncdispatch.nim | 25 +++++++++++++++---------- tests/async/tupcoming_async.nim | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 1dfd0122a7..c4b3e11e9e 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -1233,9 +1233,14 @@ else: newList.add(cb) withData(p.selector, ident, adata) do: + # descriptor still present in queue. adata.rwlist = newList & adata.rwlist rLength = len(adata.readList) wLength = len(adata.writeList) + do: + # descriptor was unregistered in callback via `unregister()`. + rLength = -1 + wLength = -1 template processCustomCallbacks(ident: untyped) = # Process pending custom event callbacks. Custom events are @@ -1254,11 +1259,16 @@ else: var cb = curList[0] if not cb(fd.AsyncFD): newList.add(cb) - else: - p.selector.unregister(fd) withData(p.selector, ident, adata) do: + # descriptor still present in queue. adata.readList = newList & adata.readList + if len(adata.readList) == 0: + # if no callbacks registered with descriptor, unregister it. + p.selector.unregister(fd) + do: + # descriptor was unregistered in callback via `unregister()`. + discard proc poll*(timeout = 500) = var keys: array[64, ReadyKey] @@ -1302,15 +1312,10 @@ else: # because state `data` can be modified in callback we need to update # descriptor events with currently registered callbacks. if not custom: - var update = false var newEvents: set[Event] = {} - if rLength > 0: - update = true - incl(newEvents, Event.Read) - if wLength > 0: - update = true - incl(newEvents, Event.Write) - if update: + if rLength != -1 and wLength != -1: + if rLength > 0: incl(newEvents, Event.Read) + if wLength > 0: incl(newEvents, Event.Write) p.selector.updateHandle(SocketHandle(fd), newEvents) inc(i) diff --git a/tests/async/tupcoming_async.nim b/tests/async/tupcoming_async.nim index 0fe9f08a5c..e3170620ed 100644 --- a/tests/async/tupcoming_async.nim +++ b/tests/async/tupcoming_async.nim @@ -61,6 +61,17 @@ when defined(upcoming): discard e.close() + proc eventTest5331() = + # Event must not raise any exceptions while was unregistered inside of + # own callback. + # Issue #5331. + let e = newAsyncEvent() + addEvent(e) do (fd: AsyncFD) -> bool: + e.unregister() + e.close() + e.setEvent() + poll() + when ioselSupportedPlatform or defined(windows): import osproc @@ -124,6 +135,7 @@ when defined(upcoming): eventTest() eventTest5304() eventTest5298() + eventTest5331() processTest() signalTest() echo "OK" @@ -132,12 +144,14 @@ when defined(upcoming): eventTest() eventTest5304() eventTest5298() + eventTest5331() processTest() echo "OK" else: eventTest() eventTest5304() eventTest5298() + eventTest5331() echo "OK" else: echo "OK" From abaf5d0bdba3a4908eec65199867831141ed8a55 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Feb 2017 21:00:07 +0100 Subject: [PATCH 26/50] fixes #5285 --- compiler/ast.nim | 3 ++- compiler/seminst.nim | 11 +++++++++-- compiler/semtempl.nim | 11 +++++++++-- tests/template/tgensymregression.nim | 21 +++++++++++++++++++++ tests/template/typedescids.nim | 2 +- 5 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 tests/template/tgensymregression.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index d9c886d7f4..be846e8dfc 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1541,7 +1541,8 @@ proc skipGenericOwner*(s: PSym): PSym = ## Generic instantiations are owned by their originating generic ## symbol. This proc skips such owners and goes straight to the owner ## of the generic itself (the module or the enclosing proc). - result = if s.kind in skProcKinds and sfFromGeneric in s.flags: + result = if s.kind in skProcKinds and {sfGenSym, sfFromGeneric} * s.flags == + {sfFromGeneric}: s.owner.owner else: s.owner diff --git a/compiler/seminst.nim b/compiler/seminst.nim index e1a65da742..9c57be0231 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -97,10 +97,17 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation; if inst.compilesId == id and sameInstantiation(entry, inst[]): return inst.sym +when false: + proc `$`(x: PSym): string = + result = x.name.s & " " & " id " & $x.id + proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: - if n.kind == nkSym and sfGenSym in n.sym.flags and - (n.sym.owner == orig or n.sym.owner.kind == skPackage): + #if n.kind == nkSym and sfGenSym in n.sym.flags: + # if n.sym.owner != orig: + # echo "symbol ", n.sym.name.s, " orig ", orig, " owner ", n.sym.owner + if n.kind == nkSym and {sfGenSym, sfFromGeneric} * n.sym.flags == {sfGenSym}: # and + # (n.sym.owner == orig or n.sym.owner.kind in {skPackage}): let s = n.sym var x = PSym(idTableGet(symMap, s)) if x == nil: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index b639288079..5dba125c2f 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -112,6 +112,7 @@ type toBind, toMixin, toInject: IntSet owner: PSym cursorInBody: bool # only for nimsuggest + scopeN: int bracketExpr: PNode template withBracketExpr(ctx, x, body: untyped) = @@ -141,8 +142,13 @@ proc isTemplParam(c: TemplCtx, n: PNode): bool {.inline.} = proc semTemplBody(c: var TemplCtx, n: PNode): PNode -proc openScope(c: var TemplCtx) = openScope(c.c) -proc closeScope(c: var TemplCtx) = closeScope(c.c) +proc openScope(c: var TemplCtx) = + openScope(c.c) + inc c.scopeN + +proc closeScope(c: var TemplCtx) = + dec c.scopeN + closeScope(c.c) proc semTemplBodyScope(c: var TemplCtx, n: PNode): PNode = openScope(c) @@ -166,6 +172,7 @@ proc newGenSym(kind: TSymKind, n: PNode, c: var TemplCtx): PSym = result = newSym(kind, considerQuotedIdent(n), c.owner, n.info) incl(result.flags, sfGenSym) incl(result.flags, sfShadowed) + if c.scopeN == 0: incl(result.flags, sfFromGeneric) proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) = # locals default to 'gensym': diff --git a/tests/template/tgensymregression.nim b/tests/template/tgensymregression.nim new file mode 100644 index 0000000000..e73ff258dc --- /dev/null +++ b/tests/template/tgensymregression.nim @@ -0,0 +1,21 @@ + +template mathPerComponent(op: untyped): untyped = + proc op*[N,T](v,u: array[N,T]): array[N,T] {.inline.} = + for i in 0 ..< len(result): + result[i] = `*`(v[i], u[i]) + +mathPerComponent(`***`) +# bug #5285 +when true: + if isMainModule: + var v1: array[3, float64] + var v2: array[3, float64] + echo repr(v1 *** v2) + + +proc foo(): void = + var v1: array[4, float64] + var v2: array[4, float64] + echo repr(v1 *** v2) + +foo() diff --git a/tests/template/typedescids.nim b/tests/template/typedescids.nim index ebed49b173..1df2f69fb1 100644 --- a/tests/template/typedescids.nim +++ b/tests/template/typedescids.nim @@ -6,7 +6,7 @@ discard """ var i {.compileTime.} = 2 -template defineId*(t: typedesc): stmt = +template defineId*(t: typedesc) = const id {.genSym.} = i static: inc(i) proc idFor*(T: typedesc[t]): int {.inline, raises: [].} = id From 3978845266d154c9dba42a8d17266f057a52d90f Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Sat, 4 Feb 2017 22:18:54 +0200 Subject: [PATCH 27/50] Use __NR_gettid instead of SYS_gettid (#5338) --- lib/system/threads.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 3dadfc6830..e8b34bf2e4 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -195,15 +195,15 @@ else: importc: "pthread_setaffinity_np", header: pthreadh.} when defined(linux): - proc syscall(arg: int): int {.varargs, importc: "syscall", header: "".} - var SYS_gettid {.importc, header: "".}: int + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "".} + var NR_gettid {.importc: "__NR_gettid", header: "".}: int #type Pid {.importc: "pid_t", header: "".} = distinct int #proc gettid(): Pid {.importc, header: "".} proc getThreadId*(): int = ## get the ID of the currently running thread. - result = int(syscall(SYS_gettid)) + result = int(syscall(NR_gettid)) elif defined(macosx) or defined(bsd): proc pthread_threadid_np(y: pointer; x: var uint64): cint {.importc, header: "pthread.h".} From c4dd9dc77e36eed0a71bf7b5d983d9de8e2e19e4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 08:51:35 +0100 Subject: [PATCH 28/50] fixes #5269 --- compiler/vmgen.nim | 16 ++++++++++------ tests/vm/tcopy_global_var.nim | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 tests/vm/tcopy_global_var.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index e0f737f087..9460f1809d 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1259,6 +1259,13 @@ proc isTemp(c: PCtx; dest: TDest): bool = template needsAdditionalCopy(n): untyped = not c.isTemp(dest) and not fitsRegister(n.typ) +proc genAdditionalCopy(c: PCtx; n: PNode; opc: TOpcode; + dest, idx, value: TRegister) = + var cc = c.getTemp(n.typ) + c.gABC(n, whichAsgnOpc(n), cc, value, 0) + c.gABC(n, opc, dest, idx, cc) + c.freeTemp(cc) + proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode; dest, idx, value: TRegister) = # opcLdObj et al really means "load address". We sometimes have to create a @@ -1266,10 +1273,7 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode; # mylocal = a.b # needs a copy of the data! assert n.typ != nil if needsAdditionalCopy(n): - var cc = c.getTemp(n.typ) - c.gABC(n, whichAsgnOpc(n), cc, value, 0) - c.gABC(n, opc, dest, idx, cc) - c.freeTemp(cc) + genAdditionalCopy(c, n, opc, dest, idx, value) else: c.gABC(n, opc, dest, idx, value) @@ -1352,7 +1356,7 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) = c.gABx(n, opcLdGlobal, dest, s.position) if s.ast != nil: let tmp = c.genx(s.ast) - c.preventFalseAlias(n, opcWrDeref, dest, 0, tmp) + c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp) c.freeTemp(dest) c.freeTemp(tmp) @@ -1525,7 +1529,7 @@ proc genVarSection(c: PCtx; n: PNode) = if a.sons[2].kind != nkEmpty: let tmp = c.genx(a.sons[0], {gfAddrOf}) let val = c.genx(a.sons[2]) - c.preventFalseAlias(a.sons[2], opcWrDeref, tmp, 0, val) + c.genAdditionalCopy(a.sons[2], opcWrDeref, tmp, 0, val) c.freeTemp(val) c.freeTemp(tmp) else: diff --git a/tests/vm/tcopy_global_var.nim b/tests/vm/tcopy_global_var.nim new file mode 100644 index 0000000000..eadd27b9ae --- /dev/null +++ b/tests/vm/tcopy_global_var.nim @@ -0,0 +1,30 @@ +discard """ + nimout: "static done" +""" + +# bug #5269 + +proc assertEq[T](arg0, arg1: T): void = + assert arg0 == arg1, $arg0 & " == " & $arg1 + +type + MyType = object + str: string + a: int + +block: + var localValue = MyType(str: "Original strning, (OK)", a: 0) + var valueCopy = localValue + valueCopy.a = 123 + valueCopy.str = "Modified strning, (not OK when in localValue)" + assertEq(localValue.str, "Original strning, (OK)") + assertEq(localValue.a, 0) + +static: + var localValue = MyType(str: "Original strning, (OK)", a: 0) + var valueCopy = localValue + valueCopy.a = 123 + valueCopy.str = "Modified strning, (not OK when in localValue)" + assertEq(localValue.str, "Original strning, (OK)") + assertEq(localValue.a, 0) + echo "static done" From e672208b82494cb105c2c21e0604157724bb9816 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 09:41:47 +0100 Subject: [PATCH 29/50] fixes #5221 --- compiler/vmgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 9460f1809d..125fe8ae08 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1510,7 +1510,7 @@ proc genVarSection(c: PCtx; n: PNode) = #assert(a.sons[0].kind == nkSym) can happen for transformed vars if a.kind == nkVarTuple: for i in 0 .. a.len-3: - setSlot(c, a[i].sym) + if not a[i].sym.isGlobal: setSlot(c, a[i].sym) checkCanEval(c, a[i]) c.gen(lowerTupleUnpacking(a, c.getOwner)) elif a.sons[0].kind == nkSym: From 63822932556c9eb68eff08c8b60fefe518b0d9d2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 11:06:06 +0100 Subject: [PATCH 30/50] niminst: less verbose output in a desparate attempt to make travis green again --- tools/niminst/niminst.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 4478151bec..67f5e2b331 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -507,7 +507,7 @@ proc srcdist(c: var ConfigData) = if not existsDir(getOutputDir(c) / "c_code"): createDir(getOutputDir(c) / "c_code") for x in walkFiles(c.libpath / "lib/*.h"): - echo(getOutputDir(c) / "c_code" / extractFilename(x)) + when false: echo(getOutputDir(c) / "c_code" / extractFilename(x)) copyFile(dest=getOutputDir(c) / "c_code" / extractFilename(x), source=x) var winIndex = -1 var intel32Index = -1 @@ -624,7 +624,7 @@ proc xzDist(c: var ConfigData; windowsZip=false) = proc processFile(destFile, src: string) = let dest = tmpDir / destFile - echo "Copying ", src, " to ", dest + when false: echo "Copying ", src, " to ", dest if not existsFile(src): echo "[Warning] Source file doesn't exist: ", src let destDir = dest.splitFile.dir From d35c561759d98115f75c1d06e2f42c73366d8d09 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 12:10:57 +0100 Subject: [PATCH 31/50] fixes #4996 --- lib/impure/nre.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 626c3fd6bd..dda4b033fb 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -23,6 +23,15 @@ export options ## ## A regular expression library for Nim using PCRE to do the hard work. ## +## **Note**: If you love ``sequtils.toSeq`` we have bad news for you. This +## library doesn't work with it due to documented compiler limitations. As +## a workaround, use this: +## +## .. code-block:: nim +## +## import nre except toSeq +## +## ## Licencing ## --------- ## From b15e8124fe62dca5fb91eefadd1fd0de37de5923 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 15:21:40 +0100 Subject: [PATCH 32/50] fixes #5090 --- compiler/ccgexprs.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index a3144bb3c4..f8b549777d 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1383,7 +1383,9 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) = var a, b: TLoc assert(d.k == locNone) - initLocExpr(p, e.sons[1], a) + var x = e.sons[1] + if x.kind in {nkAddr, nkHiddenAddr}: x = x[0] + initLocExpr(p, x, a) initLocExpr(p, e.sons[2], b) let t = skipTypes(e.sons[1].typ, {tyVar}) let setLenPattern = if not p.module.compileToCpp: From 072d79511f0daf30bdf4ce4f1957b7c58b2df512 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 5 Feb 2017 17:14:49 +0100 Subject: [PATCH 33/50] fixes #5314 --- compiler/ccgexprs.nim | 2 +- tests/async/tasync_in_seq_constr.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/async/tasync_in_seq_constr.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f8b549777d..ade2cb41ff 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1210,7 +1210,7 @@ proc genSeqConstr(p: BProc, t: PNode, d: var TLoc) = proc genArrToSeq(p: BProc, t: PNode, d: var TLoc) = var elem, a, arr: TLoc - if t.kind == nkBracket: + if t.sons[1].kind == nkBracket: t.sons[1].typ = t.typ genSeqConstr(p, t.sons[1], d) return diff --git a/tests/async/tasync_in_seq_constr.nim b/tests/async/tasync_in_seq_constr.nim new file mode 100644 index 0000000000..7d216e352f --- /dev/null +++ b/tests/async/tasync_in_seq_constr.nim @@ -0,0 +1,17 @@ +discard """ + output: '''@[1, 2, 3, 4]''' +""" + +# bug #5314 + +import asyncdispatch + +proc bar(): Future[int] {.async.} = + await sleepAsync(500) + result = 3 + +proc foo(): Future[seq[int]] {.async.} = + await sleepAsync(500) + result = @[1, 2, await bar(), 4] # <--- The bug is here + +echo waitFor foo() From 4c4f6541d956b3d5ee098fd4e3d4ab9ca794fc64 Mon Sep 17 00:00:00 2001 From: nigredo-tori Date: Mon, 6 Feb 2017 14:06:37 +0700 Subject: [PATCH 34/50] Don't prepend project path to absolute filenames passed to setCommand (#5341) --- compiler/scriptconfig.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 245680eecd..9e94f1c19e 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -110,10 +110,13 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string; let arg = a.getString 1 if arg.len > 0: gProjectName = arg + let path = + if gProjectName.isAbsolute: gProjectName + else: gProjectPath / gProjectName try: - gProjectFull = canonicalizePath(gProjectPath / gProjectName) + gProjectFull = canonicalizePath(path) except OSError: - gProjectFull = gProjectName + gProjectFull = path cbconf getCommand: setResult(a, options.command) cbconf switch: From ecdf6045014fdc31b6407d202f188c2243f045c8 Mon Sep 17 00:00:00 2001 From: flyx Date: Mon, 6 Feb 2017 17:31:44 +0100 Subject: [PATCH 35/50] nimweb: Show output of failed external command (#5343) --- tools/nimweb.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 29464f8e33..a082520e02 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -263,8 +263,8 @@ proc findNim(): string = proc exec(cmd: string) = echo(cmd) - let (_, exitCode) = osproc.execCmdEx(cmd) - if exitCode != 0: quit("external program failed") + let (outp, exitCode) = osproc.execCmdEx(cmd) + if exitCode != 0: quit outp proc sexec(cmds: openarray[string]) = ## Serial queue wrapper around exec. From 2a22dc7787935a434a1dc30f9116ee878aecea17 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 6 Feb 2017 17:21:16 +0100 Subject: [PATCH 36/50] nimsuggest improvement: don't die because of illformed ASTs --- compiler/msgs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index e6a2b75a67..49e4fa184a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -12,7 +12,7 @@ import type TMsgKind* = enum - errUnknown, errIllFormedAstX, errInternal, errCannotOpenFile, errGenerated, + errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile, errGenerated, errXCompilerDoesNotSupportCpp, errStringLiteralExpected, errIntLiteralExpected, errInvalidCharacterConstant, errClosingTripleQuoteExpected, errClosingQuoteExpected, @@ -135,8 +135,8 @@ type const MsgKindToStr*: array[TMsgKind, string] = [ errUnknown: "unknown error", - errIllFormedAstX: "illformed AST: $1", errInternal: "internal error: $1", + errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open \'$1\'", errGenerated: "$1", errXCompilerDoesNotSupportCpp: "\'$1\' compiler does not support C++", From 1b9270d08b4f4bcdd26d97988b4ecab7deb99c48 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 6 Feb 2017 22:05:56 +0100 Subject: [PATCH 37/50] Adds bug fix list to 0.16.2 announcement. --- web/news/e031_version_0_16_2.rst | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/web/news/e031_version_0_16_2.rst b/web/news/e031_version_0_16_2.rst index 4d9d99d165..ed73524534 100644 --- a/web/news/e031_version_0_16_2.rst +++ b/web/news/e031_version_0_16_2.rst @@ -25,3 +25,67 @@ Compiler Additions Language Additions ------------------ +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github `_. + +- Fixed "Weird compilation bug" + (`#4884 `_) +- Fixed "Return by arg optimization does not set result to default value" + (`#5098 `_) +- Fixed "upcoming asyncdispatch doesn't remove recv callback if remote side closed socket" + (`#5128 `_) +- Fixed "compiler bug, executable writes into wrong memory" + (`#5218 `_) +- Fixed "Module aliasing fails when multiple modules have the same original name" + (`#5112 `_) +- Fixed "JS: var argument + case expr with arg = bad codegen" + (`#5244 `_) +- Fixed "compiler reject proc's param shadowing inside template" + (`#5225 `_) +- Fixed "const value not accessible in proc" + (`#3434 `_) +- Fixed "Compilation regression 0.13.0 vs 0.16.0 in compile-time evaluation" + (`#5237 `_) +- Fixed "Regression: JS: wrong field-access codegen" + (`#5234 `_) +- Fixed "fixes #5234" + (`#5240 `_) +- Fixed "JS Codegen: duplicated fields in object constructor" + (`#5271 `_) +- Fixed "RFC: improving JavaScript FFI" + (`#4873 `_) +- Fixed "Wrong result type when using bitwise and" + (`#5216 `_) +- Fixed "upcoming.asyncdispatch is prone to memory leaks" + (`#5290 `_) +- Fixed "Using threadvars leads to crash on Windows when threads are created/destroyed" + (`#5301 `_) +- Fixed "Type inferring templates do not work with non-ref types." + (`#4973 `_) +- Fixed "Nimble package list no longer works on lib.html" + (`#5318 `_) +- Fixed "Missing file name and line number in error message" + (`#4992 `_) +- Fixed "ref type can't be converted to var parameter in VM" + (`#5327 `_) +- Fixed "nimweb ignores the value of --parallelBuild" + (`#5328 `_) +- Fixed "Cannot unregister/close AsyncEvent from within its handler" + (`#5331 `_) +- Fixed "name collision with template instanciated generic inline function with inlined iterator specialization used from different modules" + (`#5285 `_) +- Fixed "object in VM does not have value semantic" + (`#5269 `_) +- Fixed "Unstable tuple destructuring behavior in Nim VM" + (`#5221 `_) +- Fixed "nre module breaks os templates" + (`#4996 `_) +- Fixed "Cannot implement distinct seq with setLen" + (`#5090 `_) +- Fixed "await inside array/dict literal produces invalid code" + (`#5314 `_) \ No newline at end of file From 25fb5be0cb7c59a561afc68aa33370799b060859 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 6 Feb 2017 22:06:32 +0100 Subject: [PATCH 38/50] finish tool improvements --- tools/finish.nim | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tools/finish.nim b/tools/finish.nim index eba6ec0d95..4b2cda9991 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -1,12 +1,12 @@ # -------------- post unzip steps --------------------------------------------- -import strutils, os, osproc, browsers +import strutils, os, osproc, streams, browsers const arch = $(sizeof(int)*8) proc downloadMingw() = - openDefaultBrowser("http://nim-lang.org/download/mingw$1.zip" % arch) + openDefaultBrowser("https://nim-lang.org/download/mingw$1.zip" % arch) when defined(windows): import registry @@ -86,18 +86,21 @@ when defined(windows): proc checkGccArch(mingw: string): bool = let gccExe = mingw / r"gcc.exe" if fileExists(gccExe): + const nimCompat = "nim_compat.c" + writeFile(nimCompat, """typedef int + Nim_and_C_compiler_disagree_on_target_architecture[ + $# == sizeof(void*) ? 1 : -1]; + """ % $sizeof(int)) try: - let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut, - poUsePath}).strip - when hostCPU == "i386": - result = (arch.contains("i686-") and not arch.contains("w64")) or - arch == "mingw32" - elif hostCPU == "amd64": - result = arch.contains("x86_64-") or arch.contains("i686-w64-mingw32") - else: - {.error: "Unknown CPU for Windows.".} + let p = startProcess(gccExe, "", ["-c", nimCompat], nil, + {poStdErrToStdOut, poUsePath}) + #echo p.outputStream.readAll() + result = p.waitForExit() == 0 except OSError, IOError: result = false + finally: + removeFile(nimCompat) + removeFile(nimCompat.changeFileExt("o")) proc defaultMingwLocations(): seq[string] = proc probeDir(dir: string; result: var seq[string]) = From 07562d6b99ddea15b83b50c4713ce304e49f46ea Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 6 Feb 2017 22:43:07 +0100 Subject: [PATCH 39/50] website: http to https updates --- compiler/installer.ini | 8 ++++---- copying.txt | 4 ++-- doc/contributing.rst | 2 +- doc/koch.rst | 2 +- tools/website.tmpl | 6 +++--- web/community.rst | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/compiler/installer.ini b/compiler/installer.ini index 2263e030f2..d0e7448256 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -99,10 +99,10 @@ Files: r"tools\start.bat" BinPath: r"bin;dist\mingw\bin;dist" ; Section | dir | zipFile | size hint (in KB) | url | exe start menu entry -Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html" -Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip" -Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" -Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" +Download: r"Documentation|doc|docs.zip|13824|https://nim-lang.org/download/docs-${version}.zip|overview.html" +Download: r"C Compiler (MingW)|dist|mingw.zip|82944|https://nim-lang.org/download/${mingw}.zip" +Download: r"Support DLLs|bin|nim_dlls.zip|479|https://nim-lang.org/download/dlls.zip" +Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads [WinBin] diff --git a/copying.txt b/copying.txt index a6de89dcfe..98b3e568f5 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== -Nim -- a Compiler for Nim. http://nim-lang.org/ +Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2015 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2017 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/doc/contributing.rst b/doc/contributing.rst index 31f04a5e07..ee97f6dc88 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -137,7 +137,7 @@ etc. Note that currently the ``deprecated`` statement does not work well with overloading so for routines the latter variant is better. -`Deprecated `_ +`Deprecated `_ pragma in the manual. diff --git a/doc/koch.rst b/doc/koch.rst index 5fa6179737..ff62b8186d 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -96,7 +96,7 @@ web command The `web`:idx: command converts the documentation in the ``doc`` directory from rst to HTML. It also repeats the same operation but places the result in the ``web/upload`` which can be used to update the website at -http://nim-lang.org. +https://nim-lang.org. By default the documentation will be built in parallel using the number of available CPU cores. If any documentation build sub commands fail, they will diff --git a/tools/website.tmpl b/tools/website.tmpl index 344024ff00..f9b1a219af 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -187,15 +187,15 @@ runForever() diff --git a/web/community.rst b/web/community.rst index 1e4faff913..fefa4c4b64 100644 --- a/web/community.rst +++ b/web/community.rst @@ -6,7 +6,7 @@ Nim's Community Forum ----- - The `Nim forum `_ is the place where most + The `Nim forum `_ is the place where most discussions related to the language happen. It not only includes discussions relating to the design of Nim but also allows for beginners to ask questions relating to Nim. @@ -35,7 +35,7 @@ Nim's Community welcome any questions that you may have! You may also be interested in reading the - `IRC logs `_ which are an archive of all + `IRC logs `_ which are an archive of all of the previous discussions that took place in the IRC channel. From 2e65b48b6298ca3aa1de593d17c50d0d7e2ee709 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 7 Feb 2017 00:49:12 +0100 Subject: [PATCH 40/50] nim.cfg: happy new year --- config/nim.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nim.cfg b/config/nim.cfg index a5c9eeccd2..9374e2b883 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -1,5 +1,5 @@ # Configuration file for the Nim Compiler. -# (c) 2015 Andreas Rumpf +# (c) 2017 Andreas Rumpf # Feel free to edit the default values as you need. From 2d3385c22204091fa5fc2963b5c009ec0182715d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 7 Feb 2017 01:04:38 +0100 Subject: [PATCH 41/50] koch.nim: winrelease without nasty batch files --- compiler/installer.ini | 2 +- koch.nim | 61 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/compiler/installer.ini b/compiler/installer.ini index d0e7448256..ebdbe43e63 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -46,7 +46,7 @@ Start: "doc/html/overview.html" [Other] -Files: "readme.txt;install.txt;contributors.txt;copying.txt" +Files: "readme.txt;copying.txt" Files: "makefile" Files: "koch.nim" Files: "install_nimble.nims" diff --git a/koch.nim b/koch.nim index 25c2c6f068..1d0ece5ffe 100644 --- a/koch.nim +++ b/koch.nim @@ -70,6 +70,8 @@ Web options: build the official docs, use UA-48159761-1 """ +const gaCode = " --googleAnalytics:UA-48159761-1" + proc exe(f: string): string = result = addFileExt(f, ExeExt) when defined(windows): @@ -380,8 +382,61 @@ proc clean(args: string) = # -------------- builds a release --------------------------------------------- +proc patchConfig(lookFor, replaceBy: string) = + const + cfgFile = "config/nim.cfg" + try: + let cfg = readFile(cfgFile) + let newCfg = cfg.replace(lookFor, replaceBy) + if newCfg == cfg: + echo "Could not patch 'config/nim.cfg' [Error]" + echo "Reason: patch substring not found:" + echo lookFor + else: + writeFile(cfgFile, newCfg) + except IOError: + quit "Could not access 'config/nim.cfg' [Error]" + +proc winReleaseArch(arch: string) = + doAssert arch in ["32", "64"] + let cpu = if arch == "32": "i386" else: "amd64" + + template withMingw(path, body) = + const orig = """#gcc.path = r"$nim\dist\mingw\bin"""" + let replacePattern = """gcc.path = r"..\mingw$1\bin" # winrelease""" % arch + patchConfig(orig, replacePattern) + try: + body + finally: + patchConfig(replacePattern, orig) + + withMingw r"..\mingw" & arch & r"\bin": + # Rebuilding koch is necessary because it uses its pointer size to + # determine which mingw link to put in the NSIS installer. + nimexec "c --out:koch_temp --cpu:$# koch" % cpu + exec "koch_temp boot -d:release --cpu:$#" % cpu + exec "koch_temp nsis -d:release" + exec "koch_temp zip -d:release" + + moveFile r"build\nim_$#.exe" % VersionAsString, + r"web\upload\download\nim-$#_x$#.exe" % [VersionAsString, arch] + moveFile r"build\nim-$#.zip" % VersionAsString, + r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch] + proc winRelease() = - exec(r"call ci\nsis_build.bat " & VersionAsString) + # Build -docs file: + when true: + web(gaCode) + withDir "web/upload/" & VersionAsString: + exec "7z a -tzip docs-$#.zip *.html" % VersionAsString + moveFile "web/upload/$1/docs-$1.zip" % VersionAsString, + "web/upload/download/docs-$1.zip" % VersionAsString + when true: + csource("-d:release") + when true: + winReleaseArch "32" + when true: + winReleaseArch "64" # -------------- tests -------------------------------------------------------- @@ -463,10 +518,10 @@ of cmdArgument: of "web": web(op.cmdLineRest) of "doc", "docs": web("--onlyDocs " & op.cmdLineRest) of "json2": web("--json2 " & op.cmdLineRest) - of "website": website(op.cmdLineRest & " --googleAnalytics:UA-48159761-1") + of "website": website(op.cmdLineRest & gaCode) of "web0": # undocumented command for Araq-the-merciful: - web(op.cmdLineRest & " --googleAnalytics:UA-48159761-1") + web(op.cmdLineRest & gaCode) of "pdf": pdf() of "csource", "csources": csource(op.cmdLineRest) of "zip": zip(op.cmdLineRest) From 379bdeca06b42a7151a60ae14617117432ceb03d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 7 Feb 2017 01:32:21 +0100 Subject: [PATCH 42/50] disable NSIS installers, ship with downloader.exe instead --- compiler/installer.ini | 2 ++ koch.nim | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/installer.ini b/compiler/installer.ini index ebdbe43e63..31c6f77281 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -94,6 +94,8 @@ Files: "bin/vccexe.exe" Files: "koch.exe" Files: "finish.exe" +Files: "downloader.exe" + ; Files: "dist/mingw" Files: r"tools\start.bat" BinPath: r"bin;dist\mingw\bin;dist" diff --git a/koch.nim b/koch.nim index 1d0ece5ffe..2b5e052070 100644 --- a/koch.nim +++ b/koch.nim @@ -223,6 +223,8 @@ proc bundleWinTools() = copyExe("tools/finish".exe, "finish".exe) removeFile("tools/finish".exe) nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe") + nimexec("c --cc:vcc --app:gui -o:downloader.exe --noNimblePath " & + "--path:..\ui tools\downloader.nim") proc zip(args: string) = bundleNimbleSrc() @@ -418,8 +420,11 @@ proc winReleaseArch(arch: string) = exec "koch_temp nsis -d:release" exec "koch_temp zip -d:release" - moveFile r"build\nim_$#.exe" % VersionAsString, - r"web\upload\download\nim-$#_x$#.exe" % [VersionAsString, arch] + when false: + # we now disable the NSIS installer as it cannot download from https + # and is broken in so many different ways it's not funny anymore: + moveFile r"build\nim_$#.exe" % VersionAsString, + r"web\upload\download\nim-$#_x$#.exe" % [VersionAsString, arch] moveFile r"build\nim-$#.zip" % VersionAsString, r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch] From a62c60ef870f6d1c26f6c91b93b2ba79bc7c5242 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Feb 2017 08:02:17 +0100 Subject: [PATCH 43/50] koch.nim: don't commit after 1am --- koch.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index 2b5e052070..b1ab07e35e 100644 --- a/koch.nim +++ b/koch.nim @@ -223,8 +223,8 @@ proc bundleWinTools() = copyExe("tools/finish".exe, "finish".exe) removeFile("tools/finish".exe) nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe") - nimexec("c --cc:vcc --app:gui -o:downloader.exe --noNimblePath " & - "--path:..\ui tools\downloader.nim") + nimexec(r"c --cc:vcc --app:gui -o:downloader.exe --noNimblePath " & + r"--path:..\ui tools\downloader.nim") proc zip(args: string) = bundleNimbleSrc() From eb9efed64bcff6fca07b008f6183f84f0560e0de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Feb 2017 09:09:16 +0100 Subject: [PATCH 44/50] downloader tool needs to use https --- koch.nim | 2 +- tools/downloader.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index b1ab07e35e..f21b904ab4 100644 --- a/koch.nim +++ b/koch.nim @@ -223,7 +223,7 @@ proc bundleWinTools() = copyExe("tools/finish".exe, "finish".exe) removeFile("tools/finish".exe) nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe") - nimexec(r"c --cc:vcc --app:gui -o:downloader.exe --noNimblePath " & + nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " & r"--path:..\ui tools\downloader.nim") proc zip(args: string) = diff --git a/tools/downloader.nim b/tools/downloader.nim index dc7b636aa8..511e37f81b 100644 --- a/tools/downloader.nim +++ b/tools/downloader.nim @@ -22,7 +22,7 @@ proc download(pkg: string; c: Controls) {.async.} = client.onProgressChanged = onProgressChanged # XXX give a destination filename instead - let contents = await client.getContent("http://nim-lang.org/download/" & pkg & ".zip") + let contents = await client.getContent("https://nim-lang.org/download/" & pkg & ".zip") let z = "dist" / pkg & ".zip" # XXX make this async somehow: writeFile(z, contents) From d7e312ee135b18d8f95750fb2bc94c80c7e13d04 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Feb 2017 10:43:33 +0100 Subject: [PATCH 45/50] nimsuggest works in macros --- compiler/semexprs.nim | 2 + tools/nimsuggest/tester.nim | 12 +- tools/nimsuggest/tests/twithin_macro.nim | 213 +++++++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 tools/nimsuggest/tests/twithin_macro.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 54a3013221..57674735a0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1052,6 +1052,8 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = # work without now. template/tsymchoicefield doesn't like an early exit # here at all! #if isSymChoice(n.sons[1]): return + when defined(nimsuggest): + if gCmd == cmdIdeTools: suggestExpr(c, n) var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared, checkModule}) if s != nil: diff --git a/tools/nimsuggest/tester.nim b/tools/nimsuggest/tester.nim index c90afe3dbd..156d3ddb97 100644 --- a/tools/nimsuggest/tester.nim +++ b/tools/nimsuggest/tester.nim @@ -161,8 +161,16 @@ proc runTest(filename: string): int = answer.add '\L' if resp != answer and not smartCompare(resp, answer): report.add "\nTest failed: " & filename - report.add "\n Expected: " & resp - report.add "\n But got: " & answer + var hasDiff = false + for i in 0..min(resp.len-1, answer.len-1): + if resp[i] != answer[i]: + report.add "\n Expected: " & resp.substr(i) + report.add "\n But got: " & answer.substr(i) + hasDiff = true + break + if not hasDiff: + report.add "\n Expected: " & resp + report.add "\n But got: " & answer finally: inp.writeLine("quit") inp.flush() diff --git a/tools/nimsuggest/tests/twithin_macro.nim b/tools/nimsuggest/tests/twithin_macro.nim new file mode 100644 index 0000000000..d67984707b --- /dev/null +++ b/tools/nimsuggest/tests/twithin_macro.nim @@ -0,0 +1,213 @@ + +import macros + +macro class*(head, body: untyped): untyped = + # The macro is immediate, since all its parameters are untyped. + # This means, it doesn't resolve identifiers passed to it. + + var typeName, baseName: NimNode + + # flag if object should be exported + var exported: bool + + if head.kind == nnkInfix and head[0].ident == !"of": + # `head` is expression `typeName of baseClass` + # echo head.treeRepr + # -------------------- + # Infix + # Ident !"of" + # Ident !"Animal" + # Ident !"RootObj" + typeName = head[1] + baseName = head[2] + + elif head.kind == nnkInfix and head[0].ident == !"*" and + head[2].kind == nnkPrefix and head[2][0].ident == !"of": + # `head` is expression `typeName* of baseClass` + # echo head.treeRepr + # -------------------- + # Infix + # Ident !"*" + # Ident !"Animal" + # Prefix + # Ident !"of" + # Ident !"RootObj" + typeName = head[1] + baseName = head[2][1] + exported = true + + else: + quit "Invalid node: " & head.lispRepr + + # The following prints out the AST structure: + # + # import macros + # dumptree: + # type X = ref object of Y + # z: int + # -------------------- + # StmtList + # TypeSection + # TypeDef + # Ident !"X" + # Empty + # RefTy + # ObjectTy + # Empty + # OfInherit + # Ident !"Y" + # RecList + # IdentDefs + # Ident !"z" + # Ident !"int" + # Empty + + # create a type section in the result + result = + if exported: + # mark `typeName` with an asterisk + quote do: + type `typeName`* = ref object of `baseName` + else: + quote do: + type `typeName` = ref object of `baseName` + + # echo treeRepr(body) + # -------------------- + # StmtList + # VarSection + # IdentDefs + # Ident !"name" + # Ident !"string" + # Empty + # IdentDefs + # Ident !"age" + # Ident !"int" + # Empty + # MethodDef + # Ident !"vocalize" + # Empty + # Empty + # FormalParams + # Ident !"string" + # Empty + # Empty + # StmtList + # StrLit ... + # MethodDef + # Ident !"age_human_yrs" + # Empty + # Empty + # FormalParams + # Ident !"int" + # Empty + # Empty + # StmtList + # DotExpr + # Ident !"this" + # Ident !"age" + + # var declarations will be turned into object fields + var recList = newNimNode(nnkRecList) + + # expected name of constructor + let ctorName = newIdentNode("new" & $typeName) + + # Iterate over the statements, adding `this: T` + # to the parameters of functions, unless the + # function is a constructor + for node in body.children: + case node.kind: + + of nnkMethodDef, nnkProcDef: + # check if it is the ctor proc + if node.name.kind != nnkAccQuoted and node.name.basename == ctorName: + # specify the return type of the ctor proc + node.params[0] = typeName + else: + # inject `self: T` into the arguments + node.params.insert(1, newIdentDefs(ident("self"), typeName)) + result.add(node) + + of nnkVarSection: + # variables get turned into fields of the type. + for n in node.children: + recList.add(n) + + else: + result.add(node) + + # Inspect the tree structure: + # + # echo result.treeRepr + # -------------------- + # StmtList + # TypeSection + # TypeDef + # Ident !"Animal" + # Empty + # RefTy + # ObjectTy + # Empty + # OfInherit + # Ident !"RootObj" + # Empty <= We want to replace this + # MethodDef + # ... + + result[0][0][2][0][2] = recList + + # Lets inspect the human-readable version of the output + #echo repr(result) + +# --- + +class Animal of RootObj: + var name: string + var age: int + method vocalize: string {.base.} = "..." # use `base` pragma to annonate base methods + method age_human_yrs: int {.base.} = self.age # `this` is injected + proc `$`: string = "animal:" & self.name & ":" & $self.age + +class Dog of Animal: + method vocalize: string = "woof" + method age_human_yrs: int = self.age * 7 + proc `$`: string = "dog:" & self.name & ":" & $self.age + +class Cat of Animal: + method vocalize: string = "meow" + proc `$`: string = "cat:" & self.name & ":" & $self.age + +class Rabbit of Animal: + proc newRabbit(name: string, age: int) = # the constructor doesn't need a return type + result = Rabbit(name: name, age: age) + method vocalize: string = "meep" + proc `$`: string = + self.#[!]# + result = "rabbit:" & self.name & ":" & $self.age + +# --- + +var animals: seq[Animal] = @[] +animals.add(Dog(name: "Sparky", age: 10)) +animals.add(Cat(name: "Mitten", age: 10)) + +for a in animals: + echo a.vocalize() + echo a.age_human_yrs() + +let r = newRabbit("Fluffy", 3) +echo r.vocalize() +echo r.age_human_yrs() +echo r + +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;name;;string;;$file;;166;;6;;"";;100 +sug;;skField;;age;;int;;$file;;167;;6;;"";;100 +sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int{.noSideEffect, gcsafe, locks: 0.};;$file;;169;;9;;"";;100 +sug;;skMacro;;twithin_macro.class;;proc (head: untyped, body: untyped): untyped{.gcsafe, locks: .};;$file;;4;;6;;"Iterates over the children of the NimNode ``n``.";;100 +sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string{.noSideEffect, gcsafe, locks: 0.};;$file;;168;;9;;"";;100 +sug;;skMethod;;twithin_macro.vocalize;;proc (self: Rabbit): string{.noSideEffect, gcsafe, locks: 0.};;$file;;184;;9;;"";;100* +""" From 7a839d7b02770987967c5485cb1ccb33cd45b380 Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Tue, 7 Feb 2017 16:45:59 +0700 Subject: [PATCH 46/50] Move checkCloseError to nativesockets --- lib/pure/nativesockets.nim | 25 +++++++++++++++++++++++++ lib/pure/net.nim | 25 ------------------------- tests/async/tacceptcloserace.nim | 7 +++++++ 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index d51dbd4750..17e23c8e0b 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -620,3 +620,28 @@ proc selectWrite*(writefds: var seq[SocketHandle], when defined(Windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) + +proc checkCloseError*(ret: cint) = + ## Asserts that the return value of close() or closeSocket() syscall + ## does not indicate a programming error (such as invalid descriptor). + ## This must only be used when an error has already occurred and + ## you are performing a cleanup. + ## Otherwise, error handling must be performed as usual. + ## + ## This procedure must be called right after performing the syscall. Example: + ## + ## .. code-block:: nim + ## + ## let ret = someSysCall() + ## if ret != 0: + ## let errcode = osLastError() + ## checkCloseError sock.closeSocket() + ## raise newException(OSError, osErrorMsg(errcode)) + + if ret != 0: + let errcode = osLastError() + when useWinVersion: + doAssert(errcode.int32 notin {WSANOTINITIALISED, WSAENOTSOCK, + WSAEINPROGRESS, WSAEINTR, WSAEWOULDBLOCK}) + else: + doAssert(errcode.int32 notin {EBADF}) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 3699835c21..7f67833582 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -191,31 +191,6 @@ proc isDisconnectionError*(flags: set[SocketFlag], SocketFlag.SafeDisconn in flags and lastError.int32 in {ECONNRESET, EPIPE, ENETRESET} -proc checkCloseError*(ret: cint) = - ## Asserts that the return value of close() or closeSocket() syscall - ## does not indicate a programming error (such as invalid descriptor). - ## This must only be used when an error has already occurred and - ## you are performing a cleanup. - ## Otherwise, error handling must be performed as usual. - ## - ## This procedure must be called right after perfoming the syscall. Example: - ## - ## .. code-block:: nim - ## - ## let ret = someSysCall() - ## if ret != 0: - ## let errcode = osLastError() - ## checkCloseError sock.closeSocket() - ## raise newException(OSError, osErrorMsg(errcode)) - - if ret != 0: - let errcode = osLastError() - when useWinVersion: - doAssert(errcode.int32 notin {WSANOTINITIALISED, WSAENOTSOCK, - WSAEINPROGRESS, WSAEINTR, WSAEWOULDBLOCK}) - else: - doAssert(errcode.int32 notin {EBADF}) - proc toOSFlags*(socketFlags: set[SocketFlag]): cint = ## Converts the flags into the underlying OS representation. for f in socketFlags: diff --git a/tests/async/tacceptcloserace.nim b/tests/async/tacceptcloserace.nim index 899136b426..cbb5b5098f 100644 --- a/tests/async/tacceptcloserace.nim +++ b/tests/async/tacceptcloserace.nim @@ -1,9 +1,16 @@ +discard """ + exitcode: 0 + output: "" +""" + import asyncdispatch, net, os, nativesockets # bug: https://github.com/nim-lang/Nim/issues/5279 proc setupServerSocket(hostname: string, port: Port): AsyncFD = let fd = newNativeSocket() + if fd == osInvalidSocket: + raiseOSError(osLastError()) setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) var aiList = getAddrInfo(hostname, port) if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: From 29f97f8cb1024d66b050a60f0cbffe19c99b3d7a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Feb 2017 11:12:49 +0100 Subject: [PATCH 47/50] koch: boot use hostOs&hostCpu specific nimcache --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index f21b904ab4..825b360a0a 100644 --- a/koch.nim +++ b/koch.nim @@ -323,7 +323,8 @@ proc boot(args: string) = var finalDest = "bin" / "nim".exe # default to use the 'c' command: let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: "" - let smartNimcache = if "release" in args: "nimcache/release" else: "nimcache/debug" + let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & + hostOs & "_" & hostCpu copyExe(findStartNim(), 0.thVersion) for i in 0..2: From b5b9c7d2e2839767fc514d7499b31d45eb732150 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Feb 2017 11:13:11 +0100 Subject: [PATCH 48/50] added crashtester tool --- tools/nimsuggest/crashtester.nim | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tools/nimsuggest/crashtester.nim diff --git a/tools/nimsuggest/crashtester.nim b/tools/nimsuggest/crashtester.nim new file mode 100644 index 0000000000..4b3ba4026b --- /dev/null +++ b/tools/nimsuggest/crashtester.nim @@ -0,0 +1,52 @@ + + +import strutils, os, osproc, streams + +const + DummyEof = "!EOF!" + +proc getPosition(s: string): (int, int) = + result = (1, 1) + var col = 0 + for i in 0.. Date: Tue, 7 Feb 2017 14:59:46 +0100 Subject: [PATCH 49/50] cleaned up accept-close-race fix #5279 --- lib/pure/asyncdispatch.nim | 2 +- lib/pure/nativesockets.nim | 25 ------------------------- lib/upcoming/asyncdispatch.nim | 2 +- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 107e26c0cd..d97214d151 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -772,7 +772,7 @@ when defined(windows) or defined(nimdoc): sizeof(listenSock).SockLen) if setoptRet != 0: let errcode = osLastError() - checkCloseError clientSock.closeSocket() + discard clientSock.closeSocket() failAccept(errcode) else: var localSockaddr, remoteSockaddr: ptr SockAddr diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 17e23c8e0b..d51dbd4750 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -620,28 +620,3 @@ proc selectWrite*(writefds: var seq[SocketHandle], when defined(Windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) - -proc checkCloseError*(ret: cint) = - ## Asserts that the return value of close() or closeSocket() syscall - ## does not indicate a programming error (such as invalid descriptor). - ## This must only be used when an error has already occurred and - ## you are performing a cleanup. - ## Otherwise, error handling must be performed as usual. - ## - ## This procedure must be called right after performing the syscall. Example: - ## - ## .. code-block:: nim - ## - ## let ret = someSysCall() - ## if ret != 0: - ## let errcode = osLastError() - ## checkCloseError sock.closeSocket() - ## raise newException(OSError, osErrorMsg(errcode)) - - if ret != 0: - let errcode = osLastError() - when useWinVersion: - doAssert(errcode.int32 notin {WSANOTINITIALISED, WSAENOTSOCK, - WSAEINPROGRESS, WSAEINTR, WSAEWOULDBLOCK}) - else: - doAssert(errcode.int32 notin {EBADF}) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 74619ab42d..d384cd05e0 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -757,7 +757,7 @@ when defined(windows) or defined(nimdoc): sizeof(listenSock).SockLen) if setoptRet != 0: let errcode = osLastError() - checkCloseError clientSock.closeSocket() + discard clientSock.closeSocket() failAccept(errcode) else: var localSockaddr, remoteSockaddr: ptr SockAddr From 7c15120247f93cc1b729d33f0af592bc8e5e9937 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 7 Feb 2017 18:32:57 +0100 Subject: [PATCH 50/50] Add some tests to httpcore related to #5344. --- lib/pure/httpcore.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 48001ccaa3..d7f720f66f 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -312,3 +312,10 @@ when isMainModule: test.add("Connection", "Test") doAssert test["Connection", 2] == "Test" doAssert "upgrade" in test["Connection"] + + # Bug #5344. + doAssert parseHeader("foobar: ") == ("foobar", @[""]) + let (key, value) = parseHeader("foobar: ") + test = newHttpHeaders() + test[key] = value + doAssert test["foobar"] == "" \ No newline at end of file